diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index c96de48db50..c8ff05e1e29 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -717,6 +717,8 @@ jobs: runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_PPO runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} + - script: L1_Functional_Tests_SingleController + runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Eval runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Other_1 @@ -786,6 +788,8 @@ jobs: runner: ${{ vars.GB200_RUNNER }} - script: L1_Functional_Tests_PPO runner: ${{ vars.GB200_RUNNER }} + - script: L1_Functional_Tests_SingleController + runner: ${{ vars.GB200_RUNNER }} - script: L1_Functional_Tests_Eval runner: ${{ vars.GB200_RUNNER }} - script: L1_Functional_Tests_Other_1 @@ -856,6 +860,8 @@ jobs: runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_PPO runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} + - script: L1_Functional_Tests_SingleController + runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Eval runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Other_1 diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 610a08ab5fe..b6051536d9a 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 610a08ab5fe9f8f5fb5fff36b170429ea67f0f92 +Subproject commit b6051536d9ac569c853200051accf81ad8af0085 diff --git a/docs/design-docs/tq-gym-async-single-controller.md b/docs/design-docs/tq-gym-async-single-controller.md new file mode 100644 index 00000000000..2e9e525be73 --- /dev/null +++ b/docs/design-docs/tq-gym-async-single-controller.md @@ -0,0 +1,467 @@ +# Token Capture v2 in the Async SingleController Pipeline + +Design and implementation plan for running NeMo-Gym rollouts through the +**Token Capture v2** architecture — Gym-owned capture, NeMo-RL as the byte +mover — inside the async SingleController (SC) GRPO pipeline on this branch. + +**Target architecture:** the v2 ownership split (`tq_gym_v2.md` / +`tq_gym_v2_rollout.html` in the prototype checkout). **Reference +implementation / donor code:** the sync-only prototype at +`/lustre/fsw/portfolios/coreai/users/pthombre/gym/RL/` +(branch `pranav/tq_gym_prototype`), which proved the dataflow and the perf +numbers but predates the v2 ownership split — it is quarry, not foundation. + +**Sources reconciled:** + +- This branch (`yukih/sc-entrypoint`): async SC pipeline — + `nemo_rl/algorithms/single_controller.py`, `single_controller_utils/`, + `experience/rollout_manager.py`, `async_utils/replay_buffer.py` + (`TQReplayBuffer`), `async_utils/staleness_sampler.py`, `models/policy/tq_policy.py`. +- v2 design docs: `tq_gym_v2.md` + `tq_gym_v2_rollout.html` (ownership split, + `parent_hint`, stage acks, enriched seal receipts, terminal-hint linearize). +- Prototype (donor): `experience/rollout_writer.py`, + `experience/blackbox_finalizer.py`, `experience/staged_token_source.py`, + `models/generation/vllm/vllm_worker_async.py`, `algorithms/grpo_sync.py`; + Gym pinned as an editable workspace member with the ingress-gate stack. +- Gym upstream: PR [#1967](https://github.com/NVIDIA-NeMo/Gym/pull/1967) + (closed) and the live [#2124](https://github.com/NVIDIA-NeMo/Gym/pull/2124)–#2128 + stack (open; a reduced file-backed capture core — not yet the v2 surface). + +--- + +## 1. Goal + +Replace the async SC's Gym rollout path — where every generated token transits +Gym HTTP responses, a Ray return, and an SC-side tensorize before reaching the +TransferQueue — with the v2 capture pipeline: + +- **Gym owns every decision** about tokens and lineage: envelope parsing, hint + confirmation, parent fallback, delta/mask construction, hash chain, digest, + stage→commit ordering, the registry state machine, rebuild, and + linearization. +- **NeMo-RL implements exactly two protocols and hosts two processes**: a + `TokenSink` that lands bytes in TQ, a `ForestCursor` that transports + registry calls to a Ray actor shell hosting Gym's state machine, plus the + finalizer orchestration (fetch, reconcile policy, publish) and the + `weight_version` value. +- The SC pipeline consumes the result through its existing seams: a buffer + slot per prompt group, canonical rows in the existing `rollout_data` + partition, unchanged train pump. + +Proven upside from the prototype (sync loop, same dataflow): **−46.9 % HTTP +bytes/generated token, −55.7 % HTTP exchanges, −45.3 % terminal Gym→RL +bytes/sample, −4.3 % total step time (p50)**. v2 additionally converts the +`ambiguous_forest` rejection class (sub-agent branches) into trainable rows +via terminal-hint linearization. + +## 2. The v2 contract + +Everything above the line ships in Gym's `nemo_gym/token_capture/` leaf +package (no fastapi, no Ray, no TQ imports); everything below is the entire +surface NeMo-RL writes. + +``` +GYM records.py StagedCallRecord, StageAck, DelegationEnvelope, ParentHint, + ForestCandidate, Reservation, ParentClaim, CommitCoords +GYM hashing.py hash_token_ids, compute_staging_digest, EMPTY_PREFIX_HASH, SCHEMA_VERSION +GYM protocols.py TokenSink.stage(rec) -> StageAck # data plane + ForestCursor.get_candidates/reserve/commit/fail # control plane +GYM forest.py ForestCursorStateMachine (+ CursorConflictError, + DuplicateRequestError, CursorFailedError) — retry-idempotent + reserve, parent validation, growth check, leases, manifest order +GYM capture.py RolloutTokenCapture.begin_call / complete_call — hint confirm, + candidate-scan fallback, delta build, fail-closed ordering +GYM rebuild.py staged deltas -> TokenEntrys -> linearize(policy, terminal_hint) +GYM adapters/vllm.py hook wiring + token extraction; install_capture(...) +GYM gate admission, call_id mint, trajectory tree + coordinate ledger, + parent_hint stamping, stage-ack backfill, seal -> enriched receipt +──────────────────────────────────────────────────────────────────────────── +RL TQTokenSink(dp_client, partition="rollout_staging") # one method: stage() +RL RayForestCursor(registry_actor) # ~20-line transport +RL RolloutForestRegistry (Ray actor SHELL hosting Gym's state machine) +RL finalizer orchestration: row fetch, reconcile policy, publish, placeholders +RL worker setup (once): install_capture(worker, sink=…, cursor=…, + weight_version_fn=lambda: self._rollout_weight_version) +RL weight_version value (trainer state) +``` + +Wire behavior (v2 additions over the prototype): the gate stamps a +`DelegationEnvelope` with a `parent_hint` (~120 B); the worker confirms the +hint with **one hash** and only falls back to the candidate scan on a miss; +the `StageAck` (~100 B) rides the response back so the gate's coordinate +ledger stays current; seal returns an **enriched receipt** with +`terminal_call_id` + a token-free tree summary, letting the finalizer run +`linearize(policy="main_chain_only", terminal_hint=…)`. + +### 2.1 Gap between v2 and what exists today + +| v2 element | Today | +|---|---| +| `nemo_gym/token_capture/` package | Does not exist. The prototype's Gym pin has the ingress gate + a flat manifest (no tree/ledger, no hints, no acks); upstream #2124 is a file-backed capture core without the gate/forest surface. | +| `ForestCursorStateMachine` in Gym | Implemented **in NeMo-RL** (`rollout_writer.py`), same semantics — the direct donor for Gym's `forest.py`. | +| `TQTokenSink` / `RayForestCursor` / `install_capture` | Staging is inlined in `vllm_worker_async.py`; workers call the registry actor directly; wiring is `configure_rollout_writer(...)` pushed from the driver. | +| `parent_hint` / stage acks / enriched receipts | Absent — worker always candidate-scans; `stage_disposition` is inferred at seal; branching rollouts reject as `ambiguous_forest`. | +| `rebuild.py` in Gym | Split across NeMo-RL (`staged_token_source.py`) and Gym (`trajectory/builder.py`). | + +The migration is therefore **two coordinated tracks** (§ 7): Gym lands +`token_capture` (largely by relocating prototype-proven code per the v2 doc's +own 6-step migration), and NeMo-RL builds the framework half against the v2 +protocol names from day one — so nothing is written twice. + +## 3. Current async SC rollout path (what changes) + +``` +_rollout_pump ─► RolloutManager.generate_and_push(prompt) + ├─ tq_buffer.reserve(weight_version) # slot, dispatch order + ├─ AsyncNemoGymRolloutImpl.run_rollout + │ └─ NemoGym.run_rollouts.remote(rows) # tokens echoed back ✗ + └─ tq_buffer.commit(group_id, record, …) + ├─ record_to_train_batch(record) # tensorize on SC ✗ + └─ dp_client.put_samples(N rows) # rollout_data partition +_train_pump ─► sampler.select → logprobs → _advantage_pump → train_microbatch_from_meta +_sync_weights ─► pause dispatches → WeightSynchronizer.sync_weights → resume +``` + +The two ✗ steps disappear: Gym's capture library, running inside the vLLM +worker, lands every token in TQ at generation time through NeMo-RL's +`TQTokenSink`. The SC keeps its reserve/commit slot discipline — `commit` +becomes *"finalize the group's staged rows and record the resulting meta."* + +## 4. Target design + +### 4.1 Data flow + +``` +SC _rollout_pump + └─ generate_and_push(prompt, target_step) + ├─ tq_buffer.reserve(weight_version=v_start) (unchanged) + ├─ mint N rollout_ids, register at gate (new) + ├─ NemoGym.run_rollouts.remote(rows + rollout_ids) (token-free) + │ agent ⇄ gate (tree + ledger, parent_hint) ⇄ vLLM worker + │ Gym capture: begin_call (hint confirm | scan | new root) + │ → cursor.reserve [RayForestCursor → registry shell] + │ complete_call: delta/hash/digest + │ → sink.stage(record) [TQTokenSink → rollout_staging] + │ → cursor.commit(coords) fail-closed + │ StageAck rides the response → gate ledger backfill + ├─ seal each rollout → enriched RolloutReceipt (+ reward) (new) + ├─ finalize group: receipt × manifest reconcile → fetch + Gym + │ re-verify → rebuild → linearize(terminal_hint) → (new) + │ N canonical rows → put_samples(rollout_data, {group_id}_g{i}) + └─ tq_buffer.commit_finalized(group_id, meta, versions) (changed) + +SC _train_pump: unchanged, except advantage/validity handling (§ 4.5) +SC _sync_weights: additionally rotates the workers' weight_version (§ 4.4) +``` + +Two TQ partitions, distinct lifecycles: + +- `rollout_staging` — one delta row per model call, keyed + `{rollout_id}/{call_id}`, written by `TQTokenSink.stage` (the framework's + entire hot-path contribution); cleared by the finalizer after publication, + by eviction, and by TTL. +- `rollout_data` (existing SC partition) — canonical per-sample training rows; + today's schema (`input_ids`, `input_lengths`, `generation_logprobs`, + `token_mask`, `sample_mask`, `prompt_ids_for_adv`, `total_reward`) + **plus `trajectory_valid_mask`**; lifecycle unchanged (select → train → + `clear_samples`). + +### 4.2 NeMo-RL components (the whole framework surface) + +- **`nemo_rl/data_plane/tq_token_sink.py`** — `TQTokenSink`: maps a + `StagedCallRecord` to a TensorDict row + tags and calls `put_samples`. TQ + vocabulary appears nowhere else in the capture path. +- **`nemo_rl/experience/rollout_writer.py`** (slimmed) — `RayForestCursor` + (each method awaits the corresponding registry-actor method) and + `RolloutForestRegistry`, reduced to an actor **shell**: + `self.state = ForestCursorStateMachine(...)` imported from Gym `forest.py`; + every method delegates 1:1. Lease/TTL knobs come from config; semantics, + idempotency, and the exception taxonomy are Gym's. +- **Worker setup** — one call at vLLM async-worker startup: + `install_capture(worker, sink=TQTokenSink(...), cursor=RayForestCursor(...), + weight_version_fn=lambda: self._rollout_weight_version)`. The + driver-pushed `set_rollout_weight_version(int)` remains the mechanism that + updates the value the closure reads. +- **Finalizer** (`nemo_rl/experience/blackbox_finalizer.py`, reworked) — + orchestration only: get manifest from the registry shell, reconcile against + the enriched receipt, fetch rows, call Gym's `rebuild` + verification + functions (the same bytes the worker hashed), call + `linearize(policy="main_chain_only", terminal_hint=receipt.terminal_call_id)`, + publish canonical rows or masked placeholders, clear staging + registry. +- **`TQReplayBuffer.commit_finalized`** and the setup wiring in + `single_controller_utils/setup.py` (§ 6). + +### 4.3 Where finalization runs; group assembly + +Finalization is CPU + TQ I/O work, run per prompt group inside the existing +`generate_and_push` dispatch task via `asyncio.to_thread` — a group's slot +flips ready only when its canonical rows exist, preserving today's ordering. +If finalize latency (~60 ms/rollout measured on sync) shows up in +`exposed_generation`, promote to a small finalizer actor pool behind the same +`finalize_group(group_id, rollout_ids, receipts) -> GroupFinalizeResult` seam. + +Group semantics (GRPO needs N generations per prompt group): + +1. Finalize each of the N rollouts; any rejection (manifest mismatch, digest + mismatch, non-finite logprobs, residual ambiguity) becomes a masked + placeholder row — `trajectory_valid_mask=0`, `sample_mask=0`, reward + zeroed — so the group always publishes exactly N rows and + `shard_meta_for_dp` invariants hold. With terminal-hint linearize, the + dominant v1 rejection class (`ambiguous_forest`) becomes a trained main + chain with verified-but-untrained side branches. +2. `prompt_ids_for_adv` for a placeholder is copied from a verified sibling; + a fully rejected group keeps a constant fallback and zero advantage. +3. Rewards come from the seal receipts, never from token traffic. +4. Optional `min_valid_fraction_per_group`: below it, drop the group entirely + (slot removed, staging cleared, capacity permit released — the SC's + over-sampling machinery already tolerates disappearing groups). + +### 4.4 Weight versions, refit, and staleness — the async-specific design + +Nothing in the v2 docs covers async; these are the new decisions: + +1. **Per-call tagging is the source of truth.** `weight_version_fn` reads + worker state that `_sync_weights` rotates: after + `weight_synchronizer.sync_weights()` and before `_rollout_permitted.set()`, + the SC fans out `set_rollout_weight_version(trainer_version)`. Every + `StagedCallRecord` then carries the true version of the weights that + produced it — strictly better than SC-side start/end stamps. + + > **TODO (T1, deferred): atomic version rotation.** The driver fan-out + > races with the weight swap per worker: calls completing inside the refit + > window can be mis-tagged by ±1 version. Tolerable for the MVP matrix + > (`staleness_window` + `allow`, which absorbs ±1 by design; no worse than + > the legacy async path, which cannot see straddles at all). The fix — + > thread `weight_version` through `WeightSynchronizer.sync_weights()` into + > the worker's update handler so swap and stamp happen in one task, with + > capture sampling the version at `begin_call` and `complete_call` + > (mismatch → stamp older + `wv_straddled` flag) — is a **prerequisite** + > for `strict_on_policy` / `mixed_weight_version_policy: reject`, which + > must not ship on the racy mechanism. +2. **Group staleness = oldest call version.** `finalize_group` computes + `group_min_wv`/`group_max_wv` across all calls of all N rollouts; + `commit_finalized` stores `group_min_wv` as the slot's effective version. + The `StalenessSampler` is unchanged — its window test now uses the + conservative oldest-call version, so a refit-straddling rollout is evicted + exactly when its oldest tokens age out. The reserve-time stamp remains for + dispatch-order/quota accounting only. +3. **Mixed-version groups are a policy, not an error.** New + `async_rl.mixed_weight_version_policy: allow | reject`: + - `allow` (default under `staleness_window`): finalize normally; log + `wv_spread = group_max_wv − group_min_wv`. + - `reject` (forced by `strict_on_policy`): version-spanning rollouts become + placeholders. `strict_on_policy` should also re-enable the drain + (`_inflight_rollouts → 0`) before syncing, making spans impossible; + `reject` is then a safety net. +4. **`generation_logprobs` are behavior-policy logprobs** for the version that + generated each token; the per-token importance-sampling correction in + `ClippedPGLossFn` handles mixed-version sequences, and the staleness window + bounds how far off-policy they can drift. + +### 4.5 Buffer, eviction, and cleanup + +Every path where a group can die must clear **three** stores — the SC slot, +the canonical rows, and the staging rows + registry state: + +- `TQReplayBuffer.remove(..., remove_in_dp=True)` (used by `sampler.evict`) + additionally clears `rollout_staging` keys and calls + `registry.clear_rollout(rid)` for the group's rollouts (slots record their + rollout_ids at reserve time). +- Post-train `clear_samples` is unchanged — staging was already cleared at + publication. +- SC shutdown / cancelled dispatch tasks must `fail_rollout` registered + rollout_ids and clear staged rows (`try/finally`, mirroring the existing + `sem.release()` discipline); otherwise staging leaks until `cursor_ttl_s`. +- Backpressure: `max_buffered_rollouts` bounds groups; staging is additionally + bounded by `max_inflight_prompts × num_generations_per_prompt × + max_rollout_turns` delta rows — size the partition accordingly and keep the + registry's `expire_stale` TTL as the backstop. + +### 4.6 Constraints + +- vLLM async backend first (Gym's `adapters/vllm.py`); an SGLang port is a new + adapter file plus the same `install_capture` call — the SC-side design is + engine-blind by construction. +- Streaming and `n>1` per request are rejected by the adapter (v1 behavior, + unchanged until Gym's adapter supports them). +- `rollout_max_attempts_to_avoid_lp_nan` must be 1 (gate registration is + create-only; sealing is terminal). Non-finite logprobs → placeholder, not + retry. +- Router replay requires `routed_experts` in the staged-record `extras` — + supported by the v2 record shape, deferred until needed. +- Gateway→worker identity rides the `DelegationEnvelope`; the vLLM endpoint + must remain network-isolated (no auth on that hop). + +## 5. Config surface + +```yaml +data_plane: + token_capture: # renamed from the prototype's rollout_writer + enabled: false + # no shadow mode: verification is offline row-equivalence (see § 7 M3) + staging_partition: rollout_staging + finalize_timeout_s: 30.0 + lease_ttl_s: 30.0 # registry shell → Gym state machine + cursor_ttl_s: 3600.0 # also gate registration TTL + linearize_policy: main_chain_only # forwarded to Gym rebuild + +async_rl: + mixed_weight_version_policy: allow | reject # § 4.4 + min_valid_fraction_per_group: 0.0 # § 4.3 (0 = always publish) +``` + +`strict_on_policy` auto-forces `mixed_weight_version_policy: reject` (same +pattern as its existing forcing of staleness/over-sampling). Misconfiguration +(`token_capture.enabled` without `env.should_use_nemo_gym`, or an unsupported +backend) raises at `setup_single_controller` time. + +## 6. Component changes (this repo) + +| Component | Change | +|---|---| +| `data_plane/tq_token_sink.py` (new) | `TQTokenSink` — `StagedCallRecord` → TensorDict row + tags → `put_samples`; the only file that knows TQ on the hot path | +| `experience/rollout_writer.py` (new, slim) | `RayForestCursor` transport; `RolloutForestRegistry` actor shell hosting Gym's `ForestCursorStateMachine` | +| `experience/blackbox_finalizer.py` (new) | Orchestration-only finalizer over Gym `rebuild`/`linearize`; per-group assembly, placeholders, `group_min_wv` | +| `single_controller_utils/setup.py` | Launch registry shell; register staging partition; pass gate config (registration/seal, TTLs, control token) into `spinup_nemo_gym_actor`; hand sink/cursor factories to generation setup | +| `models/generation/vllm/` | Worker startup calls Gym `install_capture(sink, cursor, weight_version_fn)`; keep `set_rollout_weight_version` fan-out; delete any inlined staging logic | +| `experience/rollout_manager.py` | Black-box mode: mint rollout_ids, register → dispatch (token-free) → seal → collect enriched receipts → `finalize_group` → `commit_finalized` | +| `async_utils/replay_buffer.py` | Slots record rollout_ids; `commit_finalized(group_id, meta, group_min_wv, group_max_wv)`; `remove()` clears staging + registry | +| `single_controller.py` | `_sync_weights` rotates worker weight_version; teardown fails/clears in-flight rollouts; metrics: `wv_spread`, hint hit rate, wasted hints, invalid-row rate, finalize latency | +| `environments/nemo_gym.py` | Register/seal control-plane helpers; gate config synthesis; receipts returned with token-free results | +| `data_plane/schema.py`, `interfaces.py`, `single_controller_utils/config.py` | Staging fields (from Gym `SCHEMA_VERSION`), `TokenCaptureConfig`, `AsyncRLConfig` additions; `AdvantageConfig` validity awareness | + +**Gym dependency:** a build containing `nemo_gym/token_capture/` and the v2 +gate (tree + ledger, hints, acks, enriched receipts). Until released, pin the +Gym branch where Track A lands (the prototype pin `6ea5810` is the starting +point; upstream #2124–#2128 should converge into it). + +## 7. Implementation plan + +**Bring-up first.** The MVP runs entirely out of this repo against the +vendored Gym pin: capture logic and the `ForestCursorStateMachine` are ported +from the prototype into NeMo-RL **behind the v2-named seams** (`TokenSink`, +`ForestCursor`, `install_capture`-shaped worker setup, defined locally). +Track A — relocating those internals into Gym's `token_capture` — is deferred +to hardening (T6) and, by construction, changes no SC-facing code when it +lands. The MVP config matrix is restricted to `staleness_window` + +`mixed_weight_version_policy: allow`; `strict_on_policy`, `reject`, and +`force_in_order` raise `NotImplementedError` at setup until T1/T4. + +### MVP (get it running) — one PR, five sign-off-gated stages + +The MVP is developed on a single branch and lands as **one PR**, built in five +stages. Each stage is its own signed-off commit (series), keeps the tree green +in isolation (S1–S3 are dormant behind `token_capture.enabled=false`), and +ends at a **review gate: the stage diff, test results, and any deviations +from this doc are presented for explicit user sign-off before the next stage +begins.** Protocol/record shapes freeze at the S1 gate. + +- **S1 — token_capture primitives.** `nemo_rl/experience/token_capture/` + (protocols, records + id grammar + `staging_key`, hashing, `forest.py` + state machine — all ported from the prototype behind v2 names), + `TQTokenSink`, `RayForestCursor` + registry shell, staging schema, + `TokenCaptureConfig`. Forest/sink unit tests against the protocol boundary. +- **S2 — vLLM worker capture.** `capture.py` (candidate-scan only), + `adapters/vllm.py` `install_capture`, generation fan-outs + (`configure_token_capture`, `set_rollout_weight_version`), + `prepare_token_capture` partition registration. Worker unit tests: + stage→commit ordering, fail-closed, version stamping. +- **S3 — gate control plane + finalizer.** Register/seal helpers, receipts + through `run_rollouts`, NaN-retry hard error; `blackbox_finalizer` + + `staged_token_source`; `finalize_group` (always N rows, placeholders with + sibling `prompt_ids_for_adv`, `group_min_wv`/`group_max_wv`, + `min_valid_fraction_per_group`). +- **S4 — SC integration, direct mode.** Config validation (MVP matrix only — + strict modes raise), setup wiring (§ 6), rollout-manager capture mode, + `commit_finalized` + cleanup in the buffer, `_sync_weights` version + fan-out, teardown sweep, validity-aware advantages (**baseline mean/std + over valid rows only; invalid rows get advantage 0** — unit-tested in this + stage), metrics (finalize p50/p99, invalid rate, `wv_spread`, registry RPC + latency). Gate evidence includes a 2-GPU manual run. +- **S5 — verification, no shadow mode.** Fixed-seed job on legacy and direct + paths, training rows dumped from TQ and diffed offline; reward-curve + comparison; capture-enabled functional test wired into + `L1_Functional_Tests_SingleController.sh`. S5 sign-off = MVP acceptance; + the PR opens after it (or earlier as a draft, decided at the S1 gate). + +### Hardening TODOs (ordered) + +- **T1 — atomic weight-version rotation** (§ 4.4 TODO). Prerequisite for T4. +- **T2 — failure sweep + chaos test.** Registry/gate actor death → fail + affected dispatch tasks, release permits, clear orphaned staging keys; + kill-registry-mid-step test. First, because bring-up debugging kills actors. +- **T3 — finalizer isolation.** Dedicated bounded executor + own DP client so + a TQ stall cannot starve the trainer's `to_thread` calls. +- **T4 — strict modes.** `strict_on_policy` (with drain re-enabled), + `mixed_weight_version_policy: reject`, and `force_in_order` (which matches + on the reserve-time `target_step` only; `group_min_wv` governs only the + window modes). +- **T5 — calls-per-rollout cap.** Gate-side admission limit with a + registry-side backstop — the design's only unbounded resource. +- **T6 — Track A: relocate into Gym** (below) + the rollout-id contract: one + id grammar/validator, gate as sole `call_id` minter (unifying with + upstream's `model_call_id`), `staging_key()` defined in Gym `records.py`, + `DelegationEnvelope` as the only gate→worker carrier. +- **T7 — scale.** Registry sharding by `hash(rollout_id)` (driven by the M2 + latency counter), cross-repo metrics channel (Gym counters riding the seal + receipt), finalizer actor pool, consistent-hash rollout→worker affinity. + +### Track A — Gym: land `token_capture` (deferred to T6; mostly relocation) +1. Move hashing + record dataclasses + `build_staging_delta` from the + prototype into `nemo_gym/token_capture/` (digests byte-identical — the + prototype's shadow/digest tests are the check); move + `ForestCursorStateMachine` + exception taxonomy into `forest.py` with its + unit tests (the NeMo-RL implementation in the prototype's + `rollout_writer.py` is the donor). +2. Define `TokenSink`/`ForestCursor` protocols; wire capture through them. +3. Move the capture algorithm (`_prepare_rollout_request` / + `_stage_rollout_response` bodies) into `RolloutTokenCapture`. +4. Move the engine adapter into `adapters/vllm.py`; expose `install_capture`. +5. Move rebuild (`StagedSnapshotTokenSource` core) into `rebuild.py` + + `linearize(policy, terminal_hint)`. +6. Behavior additions: gate tree + ledger, `parent_hint`, stage acks, enriched + receipts. Ship the adapter conformance suite (golden captures → + byte-identical records/digests). + Steps 1–5 change no behavior and are exactly the package to upstream + (reconciling with the #2124 stack). + +Steps 1–5 change no behavior; the MVP's locally-hosted state machine and +capture logic are the donors, so the relocation is a swap behind the seams +from M1 — no SC-facing code changes. Until step 6 lands, the gate produces +v1 receipts (no hints/acks): capture always takes the candidate-scan fallback +and `linearize` runs without a terminal hint — the seams tolerate the v1 gate, +so nothing downstream waits on step 6; its features (hint hit rate, lower +`ambiguous_forest` rate) simply light up in existing metrics when it ships. + +**Post-MVP validation milestone (with T1–T4 landed):** 1-off async nightly +(`grpo-llama3.1-8b …-async-1off-single-controller`) in direct mode, plus a +perf report reproducing the sync measurements (HTTP bytes/token, step time) +on the async path. Unit tests accompanying T1/T4: sampler evicts by +oldest-call version; refit-straddling rollout finalizes under `allow`, +placeholders under `reject`; fully-rejected group releases capacity. + +## 8. Risks and open questions + +- **Gym release coupling.** `token_capture` becomes hot-path code in every + engine worker on Gym's release cadence; lease/TTL knobs and manifest shape + become Gym API. Mitigations from the v2 doc: `SCHEMA_VERSION` pinning, the + core package stays dependency-free, `adapters/vllm.py` lazily imports vLLM. + Until upstreamed, we carry the Gym branch pin. +- **Upstream convergence.** The open #2124–#2128 stack is a different + (file-backed) shape; Track A must reconcile with it or supersede it — + coordination with the Gym team is the schedule risk — which is why the MVP + runs entirely out of this repo and Track A is deferred to T6, insulated by + the protocol boundary. +- **Finalize latency on the rollout critical path.** ~60 ms/rollout hides + behind generation seconds, but async moves it out from behind training — + measured from M2 via the finalize-latency counter. +- **Placeholder-heavy groups** shift the GRPO baseline; the M2 advantage rule + (baseline over valid rows only) plus `min_valid_fraction_per_group` are the + mitigations. Needs a post-MVP experiment. +- **Checkpointing.** SC checkpointing is itself TODO; staging rows and + registry state are deliberately not checkpointed — in-flight rollouts are + abandoned on restore (registry TTL + create-only registration make this + safe), matching the buffer's restore semantics. +- **Multi-row rollouts.** GRPO grouping and reward attribution for + `all_leaves` linearization is explicitly undesigned in v2; out of scope — + `main_chain_only` trains the main chain and verifies side branches only. diff --git a/docs/design-docs/tq-gym-gate-authoritative-implementation-log.md b/docs/design-docs/tq-gym-gate-authoritative-implementation-log.md new file mode 100644 index 00000000000..a127bc759df --- /dev/null +++ b/docs/design-docs/tq-gym-gate-authoritative-implementation-log.md @@ -0,0 +1,537 @@ +# Implementation Log — Token Capture v3 (Gate Token Custody) + +Tracks stage-by-stage progress of `tq-gym-gate-authoritative.md` (MVP: S1–S5, +one PR, sign-off gates between stages). Branch: `yukih/sc-entrypoint`. + +Standing constraints (user-mandated): + +- **Dormant by default.** `token_capture.enabled=false` is the default; every + legacy codepath must behave exactly as before. Any change that affects + behavior regardless of the flag (bug fixes, submodule pin bumps) is + disclosed explicitly at the stage gate for sign-off. +- Wire shapes (§ 3.4) and the serving rule (§ 3.3) freeze at the S1 gate. + +Environment: dev node with 8×H100 for stage-gate test runs. +Prototype donor checkout: `/lustre/fsw/portfolios/coreai/users/pthombre/gym/RL` +(branch `pranav/tq_gym_prototype` @ `05e0adfa0`, Gym pin `6ea5810`). + +--- + +## S1 — primitives (Gym fork) + buffer surgery (RL repo) + +Status: **SIGNED OFF (user review, 2026-07-28).** Wire shapes and the serving +rule are frozen. Post-review closeout below (subpackage restructure done; +regression evidence recorded as it lands). + +### Gym fork (submodule branch `tq-gate-capture`) + +Base: **upstream PR #2124 head `32b555f04`** (= upstream main @ `fa0c2da3` ++ the token-id-capture core), per § 9.2. S1 work is two commits on top: +`70e43b60` "feat(token-id-capture): S1 gate-authoritative capture primitives" +and `61fbb660` "refactor(token-id-capture): move S1 capture core into +staging/ subpackage" (the post-review restructure; see Open TODOs). + +| Item | Status | Notes | +|---|---|---| +| records.py wire shapes | done | StagedCallRecord / StageResult / CommitCoords / CallRecord / RolloutReceipt / StagedCallSnapshot, `staging_key()`, `SCHEMA_VERSION=1`; reserved `chain_hash`/`cum_hash` fields for H2 | +| protocols.py | done | TokenSink / TokenSource / WeightVersionProvider / CaptureAdapter protocols; `install_capture` signature frozen (body lands in S2 `capture.py`) | +| digest.py | done | `compute_staging_digest`, encoders, `build_staging_delta` — **verified byte-identical to the prototype donor** (`rollout_writer.py`) on golden vectors incl. -0.0/NaN-adjacent cases | +| lineage.py | done | pure `RolloutLineage` (admit/commit/fail/seal → manifest) + create-only `LineageRegistry` with TTL sweep; no leases/hash-claims — the marker names the parent explicitly | +| rebuild.py | done | `snapshots_to_entries` (mask-driven, **verified equal to the prototype's `StagedSnapshotTokenSource.entries()`** on the worked-example forest incl. the c3 fork) + `linearize(main_chain_only, terminal_hint)` | +| conformance kit | done | 4 golden fixtures (worked example § 4.1, single-call, capture-failed, mixed-wv); `run_lineage_conformance` + `run_sink_source_conformance`; goldens frozen | +| purity rule | done | core modules import with no fastapi/ray/torch/TQ/aiohttp (subprocess-import test); `token_id_capture/__init__` resolves the #2124 reader/route exports lazily (PEP 562) so the core stays pure with the public API unchanged | +| tests | done | `tests/unit_tests/test_token_capture_gate_primitives.py` — 24 tests; **44/44 green including #2124's 20-test base suite** (the S3 base sanity check, already green at S1) | + +### NeMo-RL repo + +| Item | Status | Notes | +|---|---|---| +| `nemo_rl/data_plane/tq_token_sink.py` | done | TQTokenSink / TQTokenSource over put_samples/get_samples; 3 jagged + 2 scalar columns per staged row; parent pointers rejoined from the manifest by the finalizer | +| `TokenCaptureConfig` (`single_controller_utils/config.py`) | done | pydantic BaseModel on MasterConfig, `enabled=False` default; staging_partition / on_capture_failure / mixed_weight_version_policy / min_valid_fraction_per_group / TTLs | +| TQReplayBuffer surgery (`async_utils/replay_buffer.py`) | done | `commit_finalized` (slot's effective version = `group_min_wv`), `abort`, `rollout_ids` on slots, staging-aware `remove`; evicted-slot `commit` fix (pre-write check + un-write on mid-write eviction) | +| `_buffer_capacity` leak fix (`single_controller.py:319`) | done | release on dispatch exception, paired with `generate_and_push` aborting the reserved slot so eviction never double-releases | +| Partition pre-registration (`setup.py`) | done | registers `rollout_data` + `rollout_staging` from the driver thread, **gated on `token_capture.enabled`** (the TQ lazy-field controller race) | +| Tests | done | `test_tq_replay_buffer.py` 18/18 (new: token-capture mode + evicted-commit classes); `test_tq_token_sink.py` 7/7 — **conformance kit green against a live TQ simple backend** (byte-exact digests through float32 storage); `test_rollout_manager.py` 8/8 (2 new failure-path tests) | + +### S1-gate checklist (§ 10) + +- **Submodule fork logistics**: local branch `tq-gate-capture` on the vendored + submodule; base = `refs/pull/2124/head` @ `32b555f04` fetched from + `NVIDIA-NeMo/Gym` origin. **Open decision for sign-off**: where the gitlink + should point for CI (fork remote vs. an NVIDIA-NeMo/Gym branch) — the rev + currently exists only locally. +- **Leaf package importable in the worker venv**: `nemo_gym.token_id_capture` + core modules import in the RL venv with zero serving deps (purity test); + worker `py_executable` check to be repeated in S2 when `install_capture` + is wired into the vLLM worker. +- **vLLM prefix-ids + splice validated** (scratchpad + `validate_prefix_ids_vllm.py`, 1×H100): + - Template-level splice (`_replace_prefix_tokens`) for the functional-test + templates **Qwen/Qwen3-0.6B** (SC DP functional test) and + **Qwen/Qwen2.5-1.5B-Instruct** (SC exemplar): exact model prefix + preserved under retokenization drift (Qwen3's history-render strips + `` blocks — retokenization_differs=True — and the splice handles it). + - Live vLLM token-in smoke (Qwen3-0.6B): turn-2 `TokensPrompt` of spliced + exact ids → `out.prompt_token_ids == spliced` (exact prefix conditioning, + 47-token prefix), native generated-id + logprob extraction with no string + parsing and no `/tokenize` round trip. + +### Open TODOs (pre-sign-off) + +- [x] **Group the S1 modules into a subpackage** — done post-sign-off + (2026-07-28), Gym fork commit `61fbb660` (a follow-up commit, not an amend). + `records.py` (staging shapes, split back out; #2124's `records.py` reverts + to base-identical) + `protocols.py` / `digest.py` / `lineage.py` / + `rebuild.py` / `conformance/` now live under + `nemo_gym/token_id_capture/staging/`; `staging/__init__` re-exports the + wire shapes + protocols (the disclosure-5 name collision dissolves into + namespacing); the purity test now globs the subpackage instead of a + hand-maintained list (46/46 green, incl. #2124's 20-test base suite). + RL-side imports (`tq_token_sink.py`, `test_tq_token_sink.py` — 7/7 green + vs live TQ after the move) and the design doc's § 3.0/§ 9.3 paths updated. + S2's `capture.py` lands in `staging/`; `adapters/` and gate hosting stay + outside the purity scope. + +### Deviations / disclosures (for S1 sign-off) + +1. **Gym submodule pin moves `610a08ab` → `61fbb660`** (= #2124 head + S1 + commits; ~50 upstream commits ahead of the old pin). Attempted alternative — + cherry-picking #2124 onto the old pin — required hand-merging its + prerequisite (#1715 observability capture, 34 files / 2.7k lines) into an + upstream-untested combination; rejected in favor of the design's + prescribed base. **This changes Gym behavior with the flag off**; the + flag-off SC gym functional test is the regression evidence to run at the + gate (queued; see below). +2. **`uv.lock` regenerated** for the new pin (aiohttp 3.13.3→3.14.1 floor, + +simple-websocket/toml/websocket-client, starlette/typing-extensions + bumps). Affects all environments regardless of the flag. +3. **Bug fixes active with the flag off** (all disclosed by design §§ 9.1): + evicted-slot `commit` no longer orphans TQ rows; `_buffer_capacity` no + longer leaks on failed dispatch (failed dispatch now also aborts its + reserved slot instead of leaving a phantom entry until staleness eviction). +4. **`#2124` API preserved via lazy exports**: `token_id_capture/__init__` + now resolves reader/route/source names through module `__getattr__` + (required by the § 3.0 purity rule). No call-site changes. +5. **Protocol naming**: the design's `TokenSink`/`TokenSource` protocols + collide with names #2124 already exports; they live in + `token_id_capture/protocols.py` and are imported by module path, not + re-exported from the package root. +6. **Purity enforcement** is a subprocess-import unit test rather than an + import-linter dependency (same guarantee, no new Gym dependency). +7. **Pre-existing test breakage fixed**: `test_rollout_manager.py`'s + `_FakeBuffer` lacked the `target_step` kwarg (broken before this work); + fixed while adding the new failure-path tests. + +### Regression evidence (flag off) + +- `tests/unit/test_config_validation.py` + `test_config_v2.py`: **476 passed**. +- `tests/unit/experience/`: 8/8; `tests/unit/single_controller/test_tq_replay_buffer.py`: 18/18. +- Full `tests/unit/single_controller/` + `tests/unit/experience/` suite + (2026-07-28, `NRL_FORCE_REBUILD_VENVS=true` venv rebuild first): **59 + passed; 10 failures, all pre-existing at branch HEAD** (each reproduced + byte-identically with ALL working-tree changes stashed — committed test + fixtures out of sync with committed branch code, not this work): + - `test_rollout_pump.py::test_rollout_pump_writes_expected_tq_data` — + SC actor init reads `master_config.logger` (`single_controller.py:116`, + committed 2026-06-21) but the test's `MasterConfig.model_construct` + never sets the required `logger` field → actor creation dies → + 0 rows in TQ. (The prior session's "Ray-version mismatch" diagnosis was + the venv symptom; after rebuild this is the real, deterministic cause.) + - 9 × `test_single_controller_setup.py::TestSetup::*` — `KeyError: + 'seed'`: `setup_single_controller` calls `set_seed(grpo_config["seed"])` + (commit `888cb8eeb`) but the test's `_make_master_config` has no + `grpo.seed`. + Flagged for the user at the gate (fixable as a test-fixture patch, but + left untouched to keep the stage diff clean). +- SC functional tests (flag off, dev node GPUs) as pin-bump regression + evidence: + - `tests/functional/grpo_dp_single_controller.sh` (Qwen3-0.6B, 2 GPUs, + Megatron + async vLLM + TQ): **PASS** (2026-07-28) — all 5 metric + checks green (`gen_kl_error` max 6.0e-4 < 2e-3; probs-ratio clamps + exactly 1.0). Caveat: required `NRL_FORCE_REBUILD_VENVS=true` — the + container's prebaked `/opt/ray_venvs` are stale vs the branch lock + (Ray 2.54.0 vs 2.55.1; `nvidia-resiliency-ext` 0.6.0.dev33 < 0.6.0's + minimum). Both mismatches pre-date the S1 uv.lock regen (Ray 2.55.1 + was already pinned at branch HEAD); environmental, not this work. + - `tests/functional/grpo_async_gym_single_controller.sh` (SC + NeMo-Gym + workplace-assistant, Qwen3-0.6B, 2 GPUs, 10 steps): **PASS** + (2026-07-28) — metric checks green (`median(gen_kl_error)`=0.041 < 1.3; + `max(reward)`=0.5 > 0). Same caveat: needs `NRL_FORCE_REBUILD_VENVS=true` + on this node (stale prebaked NemoGym-actor venv, Ray 2.54.0). + **This is the flag-off pin-bump evidence for the Gym submodule move + `610a08ab` → the tq-gate-capture branch** (disclosure 1): the legacy + token-echo path through the new pin trains correctly. + +## S2 — capture core + vLLM adapter + worker hosting + +Status: **code + tests complete (2026-07-28); awaiting user sign-off at the +S2 gate.** + +### Gym fork (submodule branch `tq-gate-capture`) + +One commit on top of the S1 pair: `51b8092e` "feat(token-id-capture): S2 +engine-blind capture core + vLLM adapter". + +| Item | Status | Notes | +|---|---|---| +| `staging/capture.py` | done | `RolloutTokenCapture.begin_call/complete_call` — engine-blind record + digest build; **fail-closed ordering**: staged coords exist only after `sink.stage` reports bytes durable, every capture failure (bad delta, sink rejection/exception, extraction error) degrades to `capture_failed` coords without breaking the served completion; weight version stamped at `begin_call` (generation-start semantics); streaming rejected (`StreamingUnsupportedError`); double-complete is a loud caller bug; `complete_call_from_response` drives the adapter; `install_capture` working body via the `CaptureHost` one-method seam (instance also returned) | +| `staging/protocols.py` | done | S1-frozen `install_capture` signature now delegates to the capture core (was `NotImplementedError`); same callable re-exported from `staging/__init__` | +| `adapters/vllm.py` | done | engine-specific only, **no vllm imports** (duck-typed payloads): `enter_prefix` via the worker's existing `required_prefix_token_ids` field; `replace_prefix_tokens` relocated **verbatim** from `nemo_rl/models/generation/vllm/vllm_worker_async.py`; native extraction off the final chat payload — message token fields or `choice.logprobs.content` `token_id:` entries (in-process; no second `/tokenize`); one-choice guard; `extract_prompt_ids` reads the hookup-attached engine prompt (vLLM's OpenAI response doesn't carry it) | +| tests | done | `tests/unit_tests/test_token_capture_s2_worker.py` — 29 tests: mock adapter + mock sink ordering matrix, install wiring, extraction shapes, splice goldens incl. the § 4.1-style retokenization-drift example; **75/75 green** (S2 + S1 primitives + #2124 base suite; the purity glob picked up `capture.py` automatically) | + +### NeMo-RL repo + +| Item | Status | Notes | +|---|---|---| +| `vllm_worker_async.py` hosting | done | `install_token_capture` (CaptureHost seam), `setup_token_capture(dp_cfg, staging_partition)` fan-out target (in-worker DP client + `TQTokenSink` + the single `install_capture` call with `VLLMCaptureAdapter`; model-owner ranks only), `_rollout_weight_version` attribute + `set_rollout_weight_version`. All dormant until the fan-out runs. | +| `vllm_generation.py` fan-outs | done | `setup_token_capture` (asserts async engine) + `set_rollout_weight_version`, standard `run_all_workers_single_data` DP-leader pattern | +| SC `_sync_weights` rotation | done | flag-gated `set_rollout_weight_version(self._trainer_version)` fan-out beside the existing `RolloutManager.set_weight_version` (§ 9.1 lists this under S4; pulled forward as it completes the S2 version-stamping story — disclosed) | +| `PY_EXECUTABLES.VLLM_GYM` | done | `--extra vllm --extra nemo_gym` worker env for capture-enabled runs (constant only; the flag-gated registry override for `VllmAsyncGenerationWorker` is S4 setup wiring) | +| pyrefly | done | `tq_token_sink.py` added to `project-includes`; `nemo_gym.*` added to `replace-imports-with-any` (editable finder hook unresolvable, same pattern as vllm/megatron) | +| Tests | done | `tests/unit/models/generation/test_vllm_token_capture_hosting.py` — 6 tests (`--nemo-gym-only`): install wiring w/ vLLM adapter, non-model-owner skip, live version stamping through the install closure into staged records, both fan-outs incl. async-engine guard | + +### S2 checks (carried from the S1 gate) + +- **Leaf package importable in the worker venv**: `uv run --locked --extra + vllm --extra nemo_gym` resolves and imports + `nemo_gym.token_id_capture.staging` + `adapters.vllm` beside vllm 0.20.0 + (the prebaked `--extra vllm`-only venv does *not* contain `nemo_gym` — + hence `VLLM_GYM`, switched in at setup only when capture is enabled). +- **Splice relocation equivalence** (scratchpad + `validate_splice_relocation.py`): Gym's `replace_prefix_tokens` produces + byte-identical output to the RL worker's `_replace_prefix_tokens` on the + S1-gate templates — Qwen3-0.6B (retokenization_differs=True, the + ``-strip drift case) and Qwen2.5-1.5B-Instruct — plus the + no-prefix path. The RL original stays in place until S3 hosts the Gym + copy on the request path, so both existing callers are untouched. + +### Deviations / disclosures (for S2 sign-off) + +1. **`install_capture` returns the capture instance** (S1 signature said + `-> None`): additive; the `CaptureHost` seam remains the primary wiring. +2. **SC `_sync_weights` fan-out pulled forward from S4** (flag-gated, + dormant): completes per-call version stamping end-to-end in S2. +3. **`pyrefly.toml` edits** are active regardless of the flag (type-checker + config only; no runtime effect). +4. **Worker request-path integration deferred to S3** by design: the gate + owns the gate→worker context wire shape (serving rule § 3.3), so + `begin_call`/`complete_call` are not yet called from the HTTP handler — + S2 delivers the hosting seam, the capture core, and the adapter. + +## S3 — Gym gate + +Status: **code + tests complete (2026-07-28); awaiting user sign-off at the +S3 gate.** (S2 sign-off was given verbally — "In case S2 is done can you +start S3" — with the S2 gate summary below still standing for review.) + +### Gym fork (submodule branch `tq-gate-capture`) + +One commit: `05986b04` "feat(token-id-capture): S3 gate — lineage hosting, +prefix serving, marker plumbing, control plane". No rebase was needed — the +fork already sits on #2124's head (`32b555f04`), and its 20-test base suite +stays green. + +| Item | Status | Notes | +|---|---|---| +| `token_id_capture/memory_store.py` | done | `MemoryRolloutTokenBuffer` — per-rollout **delta forest** (parent-linked deltas; forks share prefixes through the chain instead of duplicating cumulative sequences; O(total committed tokens)); create-only register, chain-walk `cumulative_ids`, `drop` at seal/fail/TTL. Ids only, never logprobs | +| `token_id_capture/gate.py` | done | `RolloutCaptureGate` hosting the S1 `lineage.py` machine — **no new lineage logic**: registration (create-only), `find_marker` (deepest `ng_call_id` names the parent; sub-agent forks resolve to interior nodes), `message_fingerprint` (normalized role / text content with `` stripped / tool-call name+args / tool linkage; capture carriers and token fields excluded — pinned by conformance tests), the § 3.3 serving rule in `prepare_call` (token-in only for known marker + matching fingerprint; else text-mode new root with `fallback_reason`), `ingest_coords` = authoritative commit (buffer extend + fingerprint record + marker release; `capture_failed` poisons and releases nothing), `seal_rollout` → token-free receipt + full state drop, `fail_rollout` / `expire_stale` TTL, § 8 metrics counters. `RolloutGateConfig` (enabled=False) | +| `token_id_capture/control_routes.py` | done | `PUT /ng-control/rollouts/{id}` (create-only, 409 on dup), `POST .../seal` (receipt; 404 after drop), `POST .../fail` (idempotent), `GET /ng-control/metrics`; `RolloutControlClient` for the framework side (aiohttp via `server_utils.request`, deferred import) | +| `openai_utils.py` | done | `CallMarkerMixin` / `CallMarkerTypedDictMixin` + `WithMarker` variants of the five response items, the assistant chat param, and the chat response message, mirrored on the `ForTraining` pattern; unions extended | +| `responses_converter.py` | done | outbound: marker attaches to the last content-bearing output item (`RESPONSES_TO_MARKER`, same carrier as the token arrays); inbound: an echoed marker item survives conversion onto the flushed assistant chat message. Both directions are presence-driven — no config, dormant when no gate mints markers | +| `responses_api_models/vllm_model/app.py` | done | flag-gated gate hosting: `prepare_call` after `_preprocess_chat_completion_create_params` (both sides of the fingerprint see one normalization pipeline); exact prefix into the worker's existing `required_prefix_token_ids` splice seam + `ng_capture` call context; engine logprob/token-id request fields set gate-side (worker extracts natively; the gate strips `logprobs` from the response so they never reach the agent); coords popped off the response = the commit — a **missing-coords response is committed as `capture_failed`** (rollout poisoned loudly, completion still served); marker attach; sha256(rollout_id) → client **affinity** (stateless, no map to leak); control routes installed in `setup_webserver`; backend errors/context-length shortcuts fail the admitted call (no marker → no children); setup-time `ValueError` when gate + legacy token echo are both enabled | +| tests | done | `tests/unit_tests/test_token_capture_s3_gate.py` — 21 tests: **all 4 S1 golden call sequences replayed through the gate produce receipts byte-identical to the direct `RolloutLineage` drive** (fixture→gate call-id renaming only); fallback matrix (marker stripped / history edited / unknown marker / reasoning-strip fingerprint stability); duplicate + wrong-rollout coords rejection; capture-failed poisoning; seal-drops-state + TTL; buffer fork prefixes; control-route round trips; converter marker echo round trip. `responses_api_models/vllm_model/tests/test_token_capture_gate_app.py` — 5 server e2e tests over HTTP with a fake capture-enabled worker (register → 2-turn conversation with exact prefix service and prev_len chaining → seal receipt; missing-coords poisoning; edited-history two-root fallback; dormant server exposes nothing; config guard). **Full Gym suite 1507 passed / 30 skipped** incl. all prior capture suites and the 72 existing vllm_model app tests | + +### Deviations / disclosures (for S3 sign-off) + +1. **Buffer interface**: the in-memory buffer exposes a delta-chain interface + rather than implementing #2124's `TokenCaptureStore` (JSONL `TokenEntry` + append/read) — prefix serving needs parent-linked deltas, not per-call + full snapshots. The JSONL store remains untouched as the debug/persistence + backend (H1). +2. **Marker carriers are presence-driven, not config-gated**, in the + converter and typed models: markers can only exist when a gate minted + them, so the legacy path is byte-identical with the flag off (union + additions verified against the full 1507-test suite). +3. **Rollout affinity is a stable hash** (sha256(rollout_id) mod clients), + not the prototype's sticky map — stateless, so nothing leaks when + rollouts outlive sessions. Design § 9.2 asked for "rollout affinity in + `_resolve_client`"; the mechanism choice is disclosed here. +4. **Gate-side engine fields**: in gate mode the gate (not + `return_token_id_information`) sets `logprobs=True, top_logprobs=0, + return_tokens_as_token_ids=True` on the worker request — the S2 adapter's + extraction shapes need them until native extraction is wired deeper + (worker-side stripping + coords attach is the S4 RL hookup). +5. **`fail_rollout` control route is idempotent** (returns `failed: false` + after seal/TTL instead of erroring) — a cancelled dispatch double-fail + must not crash teardown (§ 7 cleanup). +6. Functional-test side effect: `ng_prepare_data` rewrote + `workplace_assistant` metrics JSONs in the submodule working tree during + the flag-off evidence runs; reverted, not committed. + +### Regression evidence (flag off, S3 pin) + +- `tests/functional/grpo_async_gym_single_controller.sh` re-run against the + S3 pin (`05986b04`, gate code present but `token_capture_gate.enabled` + defaulting false): **PASS** (2026-07-28) — `median(gen_kl_error)`=0.038 + < 1.3, `max(reward)`=0.5 > 0. The legacy token-echo path through the + edited `app.py`/converter is behaviorally unchanged. +- Full Gym unit suite at the S3 commit: 1507 passed / 30 skipped. + +Not in S3 (lands in S4 with the RL wiring): NeMo-RL's use of +`RolloutControlClient` (register/seal/fail from `environments/nemo_gym.py`), +rollout-id minting + metadata plumbing from the SC, worker-side coords +attach/strip (the serving hookup that pairs with S2's `RolloutTokenCapture`), +receipts through `run_rollouts`, and the finalizer. + +## S4 — receipts, finalizer, SC integration + +Status: **complete (2026-07-28) — code, unit tests, and live capture-enabled +gate evidence (below) all green; awaiting user sign-off at the combined +S2–S4 gate.** (S3 sign-off pending — the user asked to "Finish S4"; S2–S4 +are presented together at the gate.) + +All NeMo-RL-side; no Gym fork changes in this stage. + +| Item | Status | Notes | +|---|---|---| +| `nemo_rl/experience/blackbox_finalizer.py` (new) | done | orchestration only: `finalize_rollout` — receipt → `TQTokenSource.fetch` by manifest keys → § 5 verification (digest recompute over fetched float32 values, mask ∈ {0,1}, finite logprobs, `prev_len + delta_len == cum_len`, weight-version tag equality) → Gym `linearize(main_chain_only, terminal_hint)`; `finalize_group` — always N rows (`{group_id}_g{i}` == the gate-registered rollout ids), placeholders (`sample_mask=0`, `prompt_ids_for_adv` from a valid sibling), `group_min_wv`/`group_max_wv` (fallback wv for all-placeholder groups), `min_valid_fraction_per_group` group drop, `mixed_weight_version_policy` allow/reject, publish via `pack_payload` → `put_samples`, then clear the group's staged rows (finalizer is the staging partition's only reader) | +| worker request path (`vllm_worker_async.py`) | done | the S2/S3 pairing: `_begin_request_capture` at `preprocess_chat` (both render paths — post-splice ids in token-in mode, full render in text mode), `_finish_request_capture` in the chat endpoint (stage → coords ride `ng_commit_coords`; logprobs stripped so the worker→gate hop is token-light § 3.2), `_abort_request_capture` on every endpoint error path; `ng_capture` field on `NeMoRLChatCompletionRequest` | +| `environments/nemo_gym.py` | done | `token_capture` on `NemoGymConfig`; `_spinup` injects the gate config through the `policy_model.responses_api_models.vllm_model` global-config override block (`token_capture_gate.enabled=true`, `return_token_id_information=false`) and **hard-errors unless `rollout_max_attempts_to_avoid_lp_nan == 1`** (NaN-retry would re-register create-only ids); control-plane helpers (`register_rollouts`/`fail_rollouts`/`gate_metrics` + seal) via Gym's `ServerClient` resolving the `policy_model` server by name; receipt-mode `run_rollouts` registers before dispatch, seals per completed row, and returns token-free results (the legacy token walk + contiguity assert never runs — the gate owns that guarantee) | +| `experience/rollout_manager.py` | done | capture dispatch `_generate_and_finalize`: mints `{group_id}_g{i}` (sample ids == rollout ids), reserves the slot with them, threads them into row `metadata.ng_rollout_id`, finalizes via `asyncio.to_thread`, commits via `commit_finalized`; failure path aborts the slot **and** best-effort-fails the gate registrations; receipt-mode `Completion` (token-free; receipt in `env_extras`); receipt-derived token metrics; receipts excluded from the wandb table | +| `algorithms/advantage_estimator.py` + SC | done | `GRPOAdvantageEstimator.compute_advantage(valid_mask=...)` replaces the hardwired `torch.ones_like` in `calculate_baseline_and_std_per_prompt`; the SC advantage pump passes `sample_mask` (validity folded, no new train field); other estimators absorb the kwarg | +| `single_controller_utils/setup.py` | done | MVP-matrix validation (requires NeMo-Gym path + vllm + async engine, loud `ValueError`/`NotImplementedError`); `VLLM_GYM` registry override for `VllmAsyncGenerationWorker` **before** generation builds; `setup_token_capture` + `set_rollout_weight_version(0)` fan-outs after partition pre-registration; `BlackboxFinalizer` built and threaded into `RolloutManager`; gate config into `spinup_nemo_gym_actor` | +| exemplar YAML | done | documented `token_capture` block in `examples/configs/grpo_math_1B_single_controller.yaml` (defaults live on `TokenCaptureConfig`); config-validation suite green | +| pyrefly | done | `blackbox_finalizer.py` added to `project-includes`; no new errors | + +### Tests (all green) + +- `tests/unit/data_plane/test_blackbox_finalizer.py` — 5 tests vs a **live TQ + simple backend**: the S1 worked-example golden row reproduces byte-exact + through staging + finalize; the rejection matrix (missing receipt, + poisoned, empty manifest, identity mismatch, missing rows, digest + corruption); mixed-wv allow/reject; N-row publish with sibling-prompt + placeholder + staging cleanup; `min_valid_fraction_per_group` drop. +- `tests/unit/models/generation/test_vllm_token_capture_hosting.py` — +4 + request-path tests (10 total): begin→finish round trip (stage before + coords, logprobs stripped, state drained), token-in `prev_len` chaining, + no-op off the capture path, abort semantics. +- `tests/unit/experience/test_rollout_manager.py` — +3 receipt-mode tests + (13 total): id minting/threading end-to-end, `commit_finalized` carry, + dropped-group abort, failed-dispatch abort + gate `fail_rollouts`. +- `tests/unit/algorithms/test_advantage_validity.py` — 2 tests: invalid rows + excluded from the per-prompt baseline; `None` keeps legacy behavior. +- Flag-off regression: `tests/unit/single_controller/` + + `tests/unit/experience/` + `test_config_validation.py`: **522 passed** + (same two pre-existing branch-HEAD failures excluded, documented at S1). + +### Deviations / disclosures (for S4 sign-off) + +1. **Finalizer runs in the dispatch task** via `asyncio.to_thread` (design's + MVP placement); finalize latency rides the rollout dispatch, not the + train pump. +2. **Gate config injection** uses the `policy_model` global-config override + block (the mechanism env yamls already use) instead of new + `global_config.py` keys — no Gym-side config change needed. +3. **Legacy test fixtures updated**: `_make_manager`/hand-built managers in + `test_rollout_manager.py` gained the new `_finalizer`/`_env_handles` + attributes (the legacy `run_rollout` impl call stays byte-identical — + verified by the pre-existing flow tests). +4. **`gate_metrics` control endpoint** is exposed via the NemoGym actor but + not yet logged per train step (receipt-derived rollout metrics + + `finalize/*` metrics land in `rollout_metrics`); wiring + `token_in_rate` into the SC logger is S5 work with the § 8 metrics pass. +5. **`commit_finalized` staging_keys are empty** by design: the finalizer + clears staged rows right after publish, so eviction has nothing extra to + clear (the design's staging-aware `remove` remains for abnormal paths). + +### Gate evidence (2-GPU capture-enabled run) + +- `grpo_async_gym_single_controller.sh ++token_capture.enabled=true` + (2026-07-28, dev node, 2×H100, 10 steps): **PASS** — both metric checks + green: `median(gen_kl_error)`=0.0375 < 1.3 (statistically identical to + the flag-off S3-pin run's 0.038 — the gate→worker→TQ→finalizer path + reproduces legacy token fidelity), `max(reward)`=0.5 > 0. Every train + step ran with `global_valid_seqs=8.0` — all rows finalizer-verified + valid, zero placeholder rows trained. +- Observed failure-path exercise (loud, handled, § 7 semantics): 3 of the + ~40 dispatched groups were cancelled by the SC mid-flight; their agents' + in-flight calls then hit the gate after `fail_rollouts` dropped the + create-only registrations → `UnknownRolloutError` from + `gate.prepare_call` → HTTP 500 to the (already-dead) rollout, and one + group's late `seal` correctly got 404 and was absorbed by the + `seal(...) failed` warning path. No leaks into training: none of these + groups produced rows, and all trained steps were full-valid. +- Environment caveat (unchanged from S1): first attempt failed on the + node's stale prebaked `/opt/ray_venvs` (vllm 0.17.1/Ray 2.54.0 vs the + lock's 0.20.0/2.55.1 → `ModuleNotFoundError: + vllm.entrypoints.serve.render`); the recorded PASS is the rerun with + `NRL_FORCE_REBUILD_VENVS=true`. + +## S5 — verification + +Status: **in progress (2026-07-28).** S4 signed off ("Commit S4 start S5") +and committed as RL `6b32665ca`. Work items (§ 10 S5): fixed-seed +legacy-vs-capture row diff, reward-curve comparison, per-call HTTP bytes +vs the echo path (+ `token_in_rate` into the SC logger, deferred from S4), +chaos smoke (kill gate mid-step), capture-enabled functional test in +`L1_Functional_Tests_SingleController.sh` (blocked on the gitlink/CI +decision). S5 sign-off = MVP acceptance. + +### Row diff: legacy vs capture (2026-07-28) + +Method: env-gated row dump (`nemo_rl/experience/row_dump.py`, +`NRL_SC_DUMP_TRAIN_ROWS=`, no-op unless set) hooked at both canonical +publish sites; identical direct-invocation runs (same node/placement/data) +differing only in `token_capture.enabled`; offline matcher keyed by +`(weight_version, prompt_ids_for_adv)`. + +Constraints discovered (documented, not fixable in-scope): + +- **No per-request sampling determinism exists** (engine seed is + placement-derived; `SamplingParams` carries no seed; the Gym path + rejects `top_k` overrides at `rollout_manager.py:452`; dataset-level + `temperature` is overwritten by the generation config at + `rollout_manager.py:474`). +- `policy.generation.temperature=0` (greedy) reaches the engine but NaNs + the loss at iteration 1 on **both** paths identically (vLLM's degenerate + greedy logprobs → NaN grad norm) — so the byte diff is a one-step + (wv=0) comparison, and full-length curves run at temperature 1. +- vLLM is cross-run nondeterministic under continuous batching (logit + jitter with batch composition), bounding what any cross-run diff can + show. + +Results: + +- Temp-1 pair (10 steps, 80 rows/run): all 40 group keys matched 1:1, + **prompt prefixes byte-identical in every row**; generated suffixes + diverge (sampling, as expected). +- Greedy wv=0 pair (identical weights): **7/8 rows byte-identical in ids + and masks end-to-end** across two entirely different pipelines + (echo-splice-tensorize vs gate-TQ-finalizer). The 1/8 divergence is a + mid-generation argmax flip at a near-tie (candidate logprobs −1.01 vs + −0.97) — engine jitter, not pipeline drift. Logprob deltas on + identical-token rows: exactly 0.0 on several rows, ≤ 0.147 max + elsewhere (batch-composition numerics; a transformation bug would be + systematic, not zero-on-some-rows). +- Step-1 rows are all single-call at this sequence budget; cross-run + multi-turn splice fidelity is not separately re-proven here — it rests + on the S1 live token-in smoke, S3 gate e2e prefix chaining, the per-row + digest verification live in every capture run, and the capture run's + gen_kl_error (0.0375) matching legacy (0.038). + +### Reward-curve comparison (2026-07-28, temp-1 pair, 10 steps) + +- `train/reward` per step: **identical 10/10** — + `[0, 0, 0.25, 0, 0.25, 0.25, 0.25, 0.5, 0, 0]` on both paths (seeded + dataset order; same prompts per step; same reward outcomes). +- `train/gen_kl_error`: same band — legacy median 0.0379 (max 0.068), + capture median 0.0358 (max 0.046). +- `train/global_valid_seqs` = 8.0 every step on both paths. + +### Wire metrics (2026-07-28) + +- `gate/*` metrics wired into the SC logger (the S4 deferral): per-step + fetch of the gate's cumulative § 8 counters through a new + `RolloutManager.gate_metrics()` passthrough, logged with derived + `gate/token_in_rate`; fetch failures are swallowed loudly (metrics never + kill a step). Live reading recorded with the chaos run below. +- Per-call token-carrier bytes, computed from the temp-1 run's actual rows + (JSON as on the wire): legacy echo attachment + (`prompt_token_ids`+`generation_token_ids`+`generation_log_probs`) mean + **2,540 B/call** (8.6 B/token at ~296 tok/call) vs capture marker + **35 B/call** — **−98.6 %** on the gate→agent carrier for this + single-turn workload; multi-turn re-echo compounds the legacy side per + turn. The full HTTP-level perf report (bytes/token, step time, gate + latency) remains post-MVP validation per § 10. + +### Instrumented perf A/B (2026-07-28, user-requested; pulled forward from +### post-MVP validation) + +Method: env-gated ASGI byte counters on every HTTP server (Gym +`HttpByteCounterMiddleware` in `server_utils.py`, `NG_HTTP_BYTES_DIR`; RL +mirror `nemo_rl/utils/http_byte_counter.py` on the vLLM worker app, +`NRL_HTTP_BYTES_DIR`); identical 10-step runs at 1024 seq budget differing +only in the flag; per-hop aggregation + timing from TB metrics. + +| Metric | Legacy | Capture | Δ | +|---|---|---|---| +| HTTP bytes / trained token | 107.3 B | 69.1 B | **−35.6 %** | +| Total HTTP bytes (10 steps) | 2.52 MB | 1.64 MB | −35 % | +| `total_step_time` median | 4.48 s | 4.07 s | **−9.0 %** | +| `exposed_generation` median | 1.99 s | 1.65 s | **−16.8 %** | +| `valid_tokens/s/GPU` median | 30.8 | 34.7 | **+12.5 %** | + +Per-hop highlights: the worker's `/tokenize` route disappears entirely +(77 calls → 0); worker `/v1/chat/completions` response bytes 0.34 → 0.19 MB +(logprob echo gone); gate `/v1/responses` responses 0.28 → 0.13 MB (token +arrays → markers); even the verifier hop shrinks (0.37 → 0.23 MB in) since +agent histories no longer carry token arrays. Health parity: same KL band +(0.0376 vs 0.0413 median), identical `max_reward` 0.5, all rows valid. + +Caveats: this workload remains single-call per rollout even at 1024 +(gate counters: 80 registered/sealed, 0 token_in, 0 fallbacks — all roots), +so the echo's per-turn compounding and `token_in_rate` are not exercised — +the prototype's −46.9 % bytes/token multi-turn number remains the +reference; 10-step timing on a 0.6B model is directional, not a benchmark. + +Reading of the numbers (assessment): + +- **The bytes reduction is structural, not statistical** — it comes from + routes/payloads that categorically no longer exist (`/tokenize` gone, + logprob echo gone, token arrays → 35 B markers), and this single-call + workload is capture's *worst case*: legacy never paid its per-turn + history re-echo (roughly quadratic in turns) while capture stays + O(generated tokens). −35.6 % is a floor that grows with agentic depth + and context length. +- **The timing wins are plausibly real but not yet bankable**: one run + pair, n=10 medians, 0.6B model — variance on this stack can be + ±5–10 %. In their favor: a concrete mechanism (one fewer HTTP + round-trip per call + ~40 % smaller payloads on the generation critical + path) and the gain concentrating in `exposed_generation` (−16.8 %) + exactly where the mechanism predicts. Expect the relative timing win to + compress on large models (GPU decode dominates) while the bytes win + grows. +- **Capture is at minimum perf-neutral while buying provenance**: the + gate, staging writes, digest verification, and finalizer all sit in the + measured path and the capture run is faster, with identical training + health — the design's custody guarantees carry no perf tax. +- To make the timing claim quotable: 3–5 repeated pairs (variance bars) + and a genuinely multi-turn workload (untrimmed tools, larger budget), + which is also the run that measures a real `token_in_rate` — earmarked + as the post-MVP perf report (§ 10). + +### Chaos smoke: gate killed mid-step (2026-07-28) + +Method: capture run, `SIGKILL` to the verified `policy_model` (gate) +process after step 3, 3-minute observation, then teardown. (Four earlier +attempts were invalidated by harness bugs — orphaned wrapper teardown, +GPU-squatting orphan EngineCore, a pgrep pattern that couldn't match the +gate, a task timeout — all documented in the session log; none were +capture-path defects.) + +Verdict vs the § 7 failure model: + +- **No corruption, no crash**: the SC actor and run stayed alive; step + count froze at 3 — nothing trained on bad data after the kill, and the + three completed steps were healthy. +- **FINDING (S5 → H1): gate death is a silent stall, not the promised + loud failure.** The NemoGym actor's control-plane `register_rollouts` + call to the dead gate sat in Gym's `server_utils.request` retry loop — + observed at **retry=375+ over the full window** (`ClientOSError`, + unbounded for connection errors) — so the dispatch never failed, the § 7 + fail-path (abort slot → `fail_rollouts` → placeholders) never engaged, + and the run would stall indefinitely. § 7's "SC dispatch timeout" row + presumes a timeout that is not wired for control-plane calls. + Recommended fix (H1 scope, where the design already places the failure + sweep + kill-gate CI test): bound control-plane retries/time + (`RolloutControlClient` request timeout), so gate death surfaces as + failed dispatches + placeholders + staging TTL, per the § 7 table. +- Staging-TTL sweep not observable in a 3-minute window (TTL 3600 s); + covered by unit tests. + +S5 finding fixed en route: agent `/run` 500s raise +`aiohttp.ClientResponseError` out of `run_rollouts`, and Ray cannot pickle +its `CIMultiDictProxy` headers — the SC saw a masking `TypeError` instead +of the real error. Fixed in `environments/nemo_gym.py` (catch + re-raise +picklable `RuntimeError`). **Active with the flag off** (any legacy agent +500 hit the same masking); disclosed for the S5 gate. diff --git a/docs/design-docs/tq-gym-gate-authoritative-setup.md b/docs/design-docs/tq-gym-gate-authoritative-setup.md new file mode 100644 index 00000000000..caddfc3b1f4 --- /dev/null +++ b/docs/design-docs/tq-gym-gate-authoritative-setup.md @@ -0,0 +1,126 @@ +# Token Capture (Gate-Authoritative): Setup and Try-It Guide + +How to set up and run the gate-authoritative token-in/token-out capture +pipeline from this branch. For the design itself see +[tq-gym-gate-authoritative.md](tq-gym-gate-authoritative.md); for the +stage-by-stage evidence see +[tq-gym-gate-authoritative-implementation-log.md](tq-gym-gate-authoritative-implementation-log.md). + +## What you get + +With `token_capture.enabled=true`, NeMo-Gym rollouts in the async +SingleController GRPO path run token-in/token-out: the Gym gate holds each +rollout's token lineage, vLLM workers stage per-call token deltas + logprobs +directly to the TransferQueue, and agent-facing messages plus the Ray return +become token-free. With `enabled=false` (the default) every legacy codepath +behaves exactly as before — the feature is dormant. + +## Prerequisites + +- The requirements of the async SingleController + NeMo-Gym path: the + feature only engages with `env.should_use_nemo_gym=true` and the async + vLLM generation backend. +- 2 GPUs for the smoke test below. +- `HF_TOKEN` exported (the functional test downloads the workplace-assistant + dataset from Hugging Face). + +## 1. Clone with submodules + +The feature spans this repo **and** a pinned NeMo-Gym fork branch. The +submodule gitlink points at commit `e3b3eac6` on +[`pthombre/tq-gate-capture`](https://github.com/NVIDIA-NeMo/Gym/tree/pthombre/tq-gate-capture) +of the public NVIDIA-NeMo/Gym repo (upstream main + a pinned rev of +[PR #2124](https://github.com/NVIDIA-NeMo/Gym/pull/2124) + the gate work), +so the standard recursive clone resolves it with no extra remotes: + +```bash +git clone --recurse-submodules git@github.com:NVIDIA-NeMo/RL.git +cd RL +git checkout pthombre/tq-gym-gate-capture +git submodule update --init --recursive +``` + +Verify the pin: `git submodule status 3rdparty/Gym-workspace/Gym` should +show `e3b3eac6...`. If it shows a `+` or a fetch error, re-run +`git submodule update --init --recursive` from the repo root. + +Environment setup is otherwise unchanged from +[installation](../about/installation.md) — the Gym fork is an editable uv +workspace member, so `uv run` picks it up automatically. + +## 2. Configuration + +All knobs live under `token_capture:` in the master config; defaults are on +`TokenCaptureConfig` +(`nemo_rl/algorithms/single_controller_utils/config.py`) and the exemplar +block is in `examples/configs/grpo_math_1B_single_controller.yaml`: + +```yaml +token_capture: + enabled: false # the only switch you must flip + staging_partition: "rollout_staging" + on_capture_failure: "continue" # continue: placeholder row | abort: fail rollout + mixed_weight_version_policy: "allow" + min_valid_fraction_per_group: null + registration_ttl_s: 3600.0 + staging_ttl_s: 3600.0 +``` + +Enable it on any SC + NeMo-Gym recipe with `++token_capture.enabled=true`. +Setup validation will reject configurations that enable capture without the +NeMo-Gym path or with `rollout_max_attempts_to_avoid_lp_nan != 1`. + +## 3. Smoke test (2 GPUs) + +The same SC + Gym functional test CI runs (see +`tests/functional/L1_Functional_Tests_SingleController.sh`): + +```bash +export HF_TOKEN=... +uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh \ + ++token_capture.enabled=true +``` + +This prepares the workplace-assistant dataset, runs a short GRPO training +job through the gate, and asserts on the resulting metrics. Run it once +without the override first if you want a legacy-path baseline from the same +tree. + +## 4. Larger runs + +`swe/` contains the SWE-bench token-capture vs. legacy perf A/B: launch +tooling (`launch_swe_ab.sh`, `make_capture_config.py`), the runbook +(`SWE_RUN.md`), the pinned launch record (`EXPERIMENT_LAUNCH.md`), and +`aggregate_perf.py` for the comparison. `examples/swe_bench/` holds the +underlying async GRPO SWE recipe and launcher. + +## 5. What to watch + +Per-train-step `gate/*` metrics land in the SC logger (wandb/tensorboard): + +- `token_in_rate` — fraction of model calls served token-in (the happy + path). Drops indicate marker stripping or history edits by the agent; + the run stays correct (text-mode fallback) but wasteful. +- `fallback_rate` by cause, `capture_failure_rate`, + `digest_verify_failures`, `invalid_row_rate`, finalize latency, + `wv_spread`. + +Debug switches (env-gated, off by default): + +- `NRL_SC_DUMP_TRAIN_ROWS=` — dump canonical training rows at publish + time for legacy-vs-capture row diffs. +- `NRL_HTTP_BYTES_DIR` / `NG_HTTP_BYTES_DIR` — per-call HTTP byte counters + on the RL vLLM worker / Gym middleware respectively (the headline + bytes-per-token comparison vs. the token-echo path). + +## Troubleshooting + +- **Submodule fetch fails**: the gitlink must resolve to `e3b3eac6` on + NVIDIA-NeMo/Gym; check network access to github.com and re-run + `git submodule update --init --recursive`. +- **Everything falls back to text mode** (`token_in_rate` ≈ 0): the agent + or a proxy is stripping the `ng_call_id` marker from assistant messages, + or rewriting history above it. Correct but slow — see design doc § 3.3. +- **Placeholder-heavy groups**: check `capture_failure_rate` (worker-side + staging failures poison rollouts under `on_capture_failure: continue`) + and gate TTL expiries in the gate logs. diff --git a/docs/design-docs/tq-gym-gate-authoritative.md b/docs/design-docs/tq-gym-gate-authoritative.md new file mode 100644 index 00000000000..27e8d7064fe --- /dev/null +++ b/docs/design-docs/tq-gym-gate-authoritative.md @@ -0,0 +1,697 @@ +# Token Capture v3: Gate Token Custody (Token-In / Token-Out) + +Design and implementation plan for running NeMo-Gym rollouts through a +**token-in/token-out** capture pipeline in the async SingleController (SC) +GRPO pipeline. The Gym gate (the model server every agent calls) becomes the +**custodian of token lineage**: it holds each rollout's cumulative token +buffer, serves exact token prefixes to vLLM workers, and returns token-free +receipts. Workers stage per-call token deltas + logprobs directly to the +TransferQueue (TQ); tokens make exactly one heavy hop, at generation time. + +This supersedes two earlier drafts: + +- the **v2 doc** (`tq-gym-async-single-controller.md`): two trackers (gate + message tree + RL-side registry actor), two Ray RPCs per model call; +- the **hash-lineage draft** of this doc: single gate tracker, but lineage + verified after the fact by worker-side hash confirmation. Review found its + claim-confirm was not computable for non-root parents and its candidate + set excluded forks; more importantly, a code survey (§ 2) showed the + hash machinery solves a problem the codebase does not have. + +The v2 doc remains the reference for inherited material where cited +(weight-version design § 6, parts of the failure model). + +--- + +## 1. Goal and design principle + +Replace the async SC's Gym rollout path — where every generated token +transits Gym HTTP responses (twice per turn, growing with history), a Ray +return, and an SC-side tensorize — with a pipeline where: + +- **Tokens rest at exactly two places**: the gate's per-rollout buffer + (in-flight custody) and TQ (durable staging + canonical rows). +- **Tokens move on exactly one heavy hop**: worker → TQ, once per model + call, carrying delta ids + logprobs + extras. +- **Every other hop is token-light**: gate→worker carries prefix ids + (replacing the text prompt, comparable size); worker→gate carries delta + ids only (~4 B/token, no logprobs); agent-facing messages and the Ray + return are token-free. +- **The integration is framework- and backend-portable.** Gym defines the + protocols and owns lineage (§ 3.0); an RL framework integrates by + implementing four small contracts (sink, source, weight-version provider, + wiring); an inference engine integrates via one adapter module. + NeMo-RL/TQ and vLLM are the first providers, not the design. + +### 1.1 The core principle: relocation, not invention + +The survey of this branch (2026-07-27) established that **token-in/token-out +already runs in production here** — routed through the most expensive +possible path: + +1. Gym's model server forces `logprobs=True, return_tokens_as_token_ids`, + string-parses generated ids from logprob entries, recovers prompt ids via + a second `/tokenize` HTTP call, and attaches + `prompt_token_ids`/`generation_token_ids`/`generation_log_probs` to the + response message (`responses_api_models/vllm_model/app.py:497-551`, + `nemo_gym/responses_converter.py:362-374`). +2. The agent echoes those arrays back inside its message history every turn. +3. The vLLM worker scrapes them (`model_post_init`, + `vllm_worker_async.py:505-517`) and splices the model's exact sampled ids + in front of the freshly rendered suffix (`_replace_prefix_tokens`, + `vllm_worker_async.py:52`). +4. The Ray return then carries all tokens again (`message_log` tensors + + decoded strings in `full_result`, `environments/nemo_gym.py:319-422`). + +Exact prefix conditioning, lineage identification, and template splicing are +therefore **proven code**. This design changes only custody: the cumulative +buffer moves from "echoed through the agent" to "held at the gate"; the +delta+logprobs move from "attached to HTTP responses and the Ray return" to +"staged once to TQ". Correctness properties are preserved *by construction* +(the model is conditioned on the same bytes as today), not re-verified after +the fact. + +### 1.2 Design alternatives + +| Design | Lineage mechanism | Token custody | Verdict | +|---|---|---|---| +| v2 doc | RL registry actor, hash-verified | worker→TQ | Works; two trackers; 2 RPCs/call | +| Hash-lineage draft | gate tree of hashes, worker hash-confirm | worker→TQ | Confirm not computable at depth ≥ 2 without extra wire fields; forks misresolve; verifies what token-in guarantees | +| **This doc** | **gate token buffer + message marker** | **gate (in-flight) + TQ** | Chosen: lineage explicit, zero verification on hot path | +| Bytes-through-gate | gate holds tokens + extras | gate | Rejected: logprobs/`routed_experts` (~KB/token) transit Python HTTP | +| Status quo | token echo through agent | agent messages + Ray return | The measured bytes problem | + +The hash machinery is not deleted from the universe — it returns in the +hardening phase (§ 10, H2) as finalize-time tamper evidence and as the +mechanism that later relaxes the strict serving rule (§ 3.3). + +## 2. Current state (what the survey established) + +Facts the plan depends on, with sources: + +- **SC path**: `_rollout_pump` → `RolloutManager.generate_and_push` + (`rollout_manager.py:644`) → `reserve` slot → `run_rollouts.remote` + (tokens ride the return) → `TQReplayBuffer.commit` tensorizes + puts N + rows (`replay_buffer.py:612-660`). `RolloutManager` runs *inside* the SC + actor; the only Ray boundary is `run_rollouts`. +- **No weight-version signal reaches vLLM workers** today; the only + fan-out is `RolloutManager.set_weight_version` + (`single_controller.py:591`). The `_sync_weights` drain is commented out + (`single_controller.py:571-584`). +- **vLLM workers are data-plane-unaware**: zero `data_plane` imports under + `models/generation/`. Template to copy: `sync_rollout_actor.py:128-130`. +- **`rollout_data` is never `register_partition`-ed**; lazy field + registration under concurrent puts is a documented TQ controller race + (`adapters/transfer_queue.py:449-461`). +- **Gym pin**: submodule `3rdparty/Gym-workspace/Gym` @ `610a08a` + (editable uv workspace member). It has **no gate and no capture + package** — in particular it predates upstream PR + [#2124](https://github.com/NVIDIA-NeMo/Gym/pull/2124) + (`nemo_gym/token_id_capture/`), which is a **required base** for the + Gym-side work (§ 9.2). The gate donor code (admission middleware, + `RolloutRegistry`, control router) lives in the prototype checkout's Gym + pin (`/lustre/fsw/.../gym/RL`, Gym @ `6ea5810`, + `nemo_gym/observability/capture_gate.py`). +- **Rollout identity today**: NeMo-RL passes no id into Gym (`_rowidx` + re-sort only); Gym↔vLLM affinity is a session cookie → sticky + round-robin (`vllm_model/app.py:576-584`). +- **Validity hook exists**: `calculate_baseline_and_std_per_prompt` + accepts a `valid_mask` currently hardwired to ones + (`advantage_estimator.py:69`). +- **Prototype donors** (`/lustre/fsw/.../gym/RL`): staging delta builder + + three-column TensorDict write, `compute_staging_digest` + (float32-bit-pattern scheme), hash-free mask-driven rebuild + (`StagedSnapshotTokenSource.entries()`), finalizer reconcile/verify + skeleton, gate admission + call-id mint + register/seal control API. + +## 3. The contract + +### 3.0 Protocol architecture: Gym defines, frameworks provide + +The integration varies along two axes (RL framework's storage/training; +inference backend) and is invariant along one (lineage custody, wire +schema, serving rule). The code mirrors that split: + +- **Gym owns the invariant, concretely**: the gate, the wire records + (single definition), the digest, the rebuild/linearize semantics, the + control routes, and the protocol definitions below. All of it lives in + the `nemo_gym/token_id_capture/` **leaf package** (grown from #2124), + under a hard purity rule — **no fastapi, no ray, no torch, no TQ + imports** in the core modules, enforced by an import-linter test — so + the package is importable inside any framework's worker process. +- **Gym also ships the backend adapters** (`adapters/vllm.py`, later + `adapters/sglang.py`) implementing a `CaptureAdapter` protocol: how to + enter prefix ids into an engine, splice the suffix, and extract + generated ids + logprobs. The gate itself is engine-blind. +- **The RL framework provides four small implementations** against Gym's + `protocols.py`: + +```python +# Defined in GYM (token_id_capture/staging/protocols.py); frameworks implement. +class TokenSink(Protocol): # WHERE deltas go (NeMo-RL impl: TQTokenSink) + def stage(self, record: StagedCallRecord) -> StageResult: ... + +class TokenSource(Protocol): # finalizer's read-back (NeMo-RL: TQ get_samples) + def fetch(self, staging_keys: list[str]) -> list[StagedCallSnapshot]: ... + +class WeightVersionProvider(Protocol): # trainer state, framework-owned + def __call__(self) -> int: ... + +# plus one wiring call at worker startup: +install_capture(serving_layer, sink=..., weight_version_fn=...) +``` + +Everything training-side (finalizer orchestration, placeholders, group +staleness, advantages) remains framework code — Gym has no opinion there. +Two rules make multi-framework work in practice: **staging keys are opaque +to Gym** (TQ keys, file paths, redis keys are all valid — the receipt +manifest is the only join between lineage and storage), and **conformance +is tested, not trusted** — Gym ships golden fixtures (call sequences → +byte-exact records/digests/manifests/linearized rows) that every framework +and adapter runs in its CI. `SCHEMA_VERSION` is negotiated at rollout +registration, so version skew fails at register time, not at finalize. + +**The whole contract at a glance** — everything above the line ships in +Gym; everything below is the entire surface an RL framework writes: + +``` +GYM nemo_gym/token_id_capture/ (leaf package; purity rule) + records.py StagedCallRecord, CommitCoords, RolloutReceipt, + CallRecord, staging_key(), SCHEMA_VERSION + protocols.py TokenSink, TokenSource, WeightVersionProvider, + CaptureAdapter; install_capture(...) + digest.py compute_staging_digest, encoders + lineage.py RolloutLineage — pure per-rollout state machine: + admit/commit/fail/seal -> manifest (no HTTP, no store; + separately tested; gate hosting stays thin) + capture.py RolloutTokenCapture.begin_call / complete_call — + engine-blind: record + digest build, fail-closed + stage->respond ordering, coords assembly + rebuild.py snapshots -> entries -> linearize(policy, terminal_hint) + adapters/vllm.py engine-specific ONLY: suffix splice, id+logprob + extraction, serving-layer hookup (sglang.py later) + store.py / memory_store.py #2124 store iface; in-memory buffer + conformance/ golden fixtures + installable test kit + gate thin hosting of lineage.py: admission, call_id mint, + marker resolve + fingerprint, serving rule, prefix + serving, coords ingestion, seal -> receipt; + control routes + RolloutControlClient +────────────────────────────────────────────────────────────────────────── +RL TQTokenSink / TQTokenSource # TQ impls of the protocols +RL blackbox_finalizer.py # orchestration over Gym rebuild +RL worker setup (once): install_capture(serving_layer, sink=…, + weight_version_fn=lambda: self._rollout_weight_version) +RL weight_version value (trainer state) + set_rollout_weight_version fan-out +RL control-plane calls via Gym's RolloutControlClient; buffer/setup wiring +``` + +### 3.1 Identity: rollout ids and the call marker + +- The SC mints rollout ids `{group_id}_g{i}` (matching existing TQ sample + ids, `payload.py:115`) and registers them at the gate (create-only) + before dispatch. Ids ride `responses_create_params.metadata` — the + zero-agent-change carrier already used for side-channel params + (`vllm_model/app.py:288-300`). +- The gate mints a `call_id` per admitted model call. +- **The marker.** When the gate forwards a completion to the agent, it + attaches `ng_call_id: ` to the assistant message — replacing + today's token-array attachment, same carrier, ~10 B instead of KBs. The + agent echoes history verbatim (this is how the token echo works today), + so the next request's messages carry the parent pointer **explicitly**. + Lineage is a dictionary lookup, not a content-matching tree walk: forks + (a sub-agent inheriting turn-1 history carries turn-1's marker) and + identical siblings (distinct markers) are resolved exactly. + +### 3.2 Wire changes per hop + +| Hop | Today | This design | +|---|---|---| +| agent → gate | messages + echoed token arrays (grows/turn) | messages + markers (token-free) | +| gate → worker | text prompt (+ echoed ids in messages) | `prefix_ids` + suffix messages via `extra_body`; or text mode (fallback) | +| worker → TQ | — | **the only heavy hop**: delta ids + logprobs + masks + extras, once/call | +| worker → gate | text + logprob block (ids string-parsed) + 2nd `/tokenize` call | text + delta ids (~4 B/token) + `CommitCoords` (~100 B); no `/tokenize` | +| gate → agent | text + token arrays | text + marker | +| gate → SC (`run_rollouts` return) | full token tensors + decoded strings | `RolloutReceipt` (~100 B/call, token-free) | +| finalizer ↔ TQ | — | staged rows in, canonical rows out (off hot path) | + +### 3.3 The serving rule (what makes zero-hash correct) + +> **Serve token-in only when the incoming request carries a unique, known +> marker AND the message history up to the marker matches the gate's +> recorded fingerprint for that call. Anything else — no marker, unknown +> marker, edited history — falls back to text mode: the worker renders the +> full conversation from scratch and the call is captured as a new root +> (full ids staged, `parent=None`).** + +The gate keeps a compact fingerprint (hash of normalized messages) per +committed call to detect history edits above the marker. A fallback is +wasteful (duplicated prefix storage, cold KV cache) but perfectly correct: +the model trains on exactly the bytes it saw. Silent wrong-prefix service is +structurally impossible — the gate serves only bytes whose provenance it +verified, or serves nothing. `token_in_rate` (§ 8) measures how often the +happy path holds. + +### 3.4 Records + +**`StagedCallRecord`** (worker → TQ, key `"{rollout_id}/{call_id}"`): +`token_ids_delta`, `token_mask_delta` (0.0 carried prompt / 1.0 generated), +`generation_logprobs_delta`, optional extras (`routed_experts`); tags: +`rollout_id, call_id, parent_call_id, prev_len, new_len, weight_version, +digest, schema_version`. + +**`CommitCoords`** (worker → gate, rides the response): `call_id, +parent_call_id, delta_len, cum_len, digest, staging_key, weight_version, +disposition: staged | capture_failed`, plus the delta ids for the gate's +buffer. + +**`RolloutReceipt`** (gate → SC at seal, token-free): `rollout_id, reward, +terminal_call_id, manifest: list[(call_id, parent_call_id|None, delta_len, +cum_len, digest, staging_key, weight_version, mode: token_in|text)], +schema_version`. + +All record shapes are defined **once**, in Gym's +`nemo_gym/token_id_capture/staging/records.py` (§ 3.0); frameworks import them +from the leaf package. Wire shapes freeze at the S1 gate. Hash fields +(`chain_hash`, `cum_hash`) are **reserved optional fields** so the +hardening layer (H2) is additive. + +### 3.5 Fail-closed ordering (per call) + +``` +gate: admit (marker → parent | fallback), mint call_id, + forward prefix_ids + suffix (or text) +worker: splice (or render), generate, extract ids+logprobs natively +worker: sink.stage(record) # bytes durable BEFORE ack + ok -> coords{staged} + delta ids ride the response + fail -> coords{capture_failed} +gate: ingest coords = authoritative commit; extend token buffer; + attach marker; forward text to agent +``` + +A child request cannot exist before its parent's bytes are durable and +committed, because the marker the child needs rides the response the gate +releases only after ingesting coords. Capture failure does not break the +agent: the completion still returns; the gate marks the rollout +capture-poisoned; the finalizer produces a placeholder. +`token_capture.on_capture_failure: continue | abort` (default `continue`). + +## 4. Data flow + +``` +SC _rollout_pump + └─ generate_and_push(prompt, target_step) + ├─ tq_buffer.reserve(weight_version=v_start) (unchanged) + ├─ mint N rollout_ids, register at gate (create-only) (new) + ├─ NemoGym.run_rollouts.remote(rows + rollout_ids) (token-free) + │ + │ per model call: + │ agent ─(messages + markers)─► GATE + │ marker → parent lookup; fingerprint check; + │ mint call_id; prefix_ids from token buffer + │ GATE ─(prefix_ids + suffix | text)─► vLLM WORKER + │ splice (_replace_prefix_tokens) | full render + │ generate; extract ids+logprobs natively + │ sink.stage(record) ──► TQ rollout_staging ◄── the only + │ delta ids + coords ride the response heavy hop + │ GATE ingests coords = COMMIT; buffer += delta; + │ marker on assistant message; text to agent + │ + ├─ seal each rollout → RolloutReceipt; gate drops buffer + state + ├─ finalize group: fetch rows by manifest keys → digest check → + │ mask-driven rebuild → linearize(main_chain_only, + │ terminal_hint) → N rows (placeholders as needed) → + │ put_samples(rollout_data) + └─ tq_buffer.commit_finalized(group_id, meta, group_min_wv, group_max_wv) + +SC _train_pump: unchanged; baseline over valid rows (§ 7) +SC _sync_weights: rotates worker weight_version (§ 6) +``` + +Two TQ partitions: `rollout_staging` (delta rows, cleared by finalizer / +eviction / TTL) and the existing `rollout_data` (canonical rows). Both are +pre-registered at setup (§ 2, controller race). + +### 4.1 Worked example + +Group `g7`, rollout `g7_r0`, wv 4 throughout. **c1**: no marker → text mode, +new root; worker renders `[10..14]` (3 prompt + 2 generated), stages +`g7_r0/c1`, coords + delta ids back; gate buffers, marks the assistant +message `ng_call_id=c1`. **c2** (tool result): messages carry `c1`'s marker +→ fingerprint ok → token-in: `prefix_ids=[10..14]` + suffix `[tool:"391"]`; +worker splices `[20,21,22]`, generates `[23,24]`; stages delta of 5; +`parent=c1`. **c3** (sub-agent from turn-1 history): carries `c1`'s marker → +token-in from an *interior* node — a fork, resolved exactly. **c4** +(framework rewrote a message): fingerprint miss → text mode, new root; 9 ids +staged, `parent=None`. Seal → receipt manifest `[(c1,∅,5),(c2,c1,+5), +(c3,c1,+4),(c4,∅,9)]`, terminal `c2`, reward 1.0. Finalize: fetch 4 rows, +digest-check, rebuild main chain c1→c2 → one canonical row (10 ids, 4 +trainable), c3/c4 verified-untrained; publish N rows; clear staging. + +## 5. What the finalizer verifies (and what it doesn't) + +Per row: digest recomputation (`compute_staging_digest` over ids + mask + +logprob bit patterns — catches TQ corruption and key mixups), shape/mask/ +finite-logprob checks, `prev_len + delta_len == cum_len`, weight-version tag +equality. Rebuild is the prototype's **hash-free, mask-driven** +`StagedSnapshotTokenSource.entries()`; linearization +`main_chain_only` with `terminal_hint`. Any rejection → masked placeholder +(always N rows; `prompt_ids_for_adv` copied from a valid sibling; +`min_valid_fraction_per_group` optionally drops the group). + +Not verified in the MVP: cross-row chain integrity (an adversarial reorder +of manifest entries that also fixes up lengths). The gate is inside the +trust boundary (network-isolated gate→worker hop, as v2 § 4.6); chain +hashes return as tamper evidence in H2. + +## 6. Weight versions, refit, staleness + +Inherited from v2 § 4.4: per-call tagging via the worker's +`_rollout_weight_version` (new attribute; set at the end of both refit +paths and via a new `set_rollout_weight_version` fan-out from +`_sync_weights`); group staleness = oldest call version (`group_min_wv`), +stored as the slot's effective version (`commit_finalized`); +`mixed_weight_version_policy: allow | reject`; the T1 atomic-rotation TODO +unchanged and still prerequisite for strict modes; verl's work-preserving +`wait` option on the hardening list. MVP matrix: `staleness_window` + +`allow` only; strict modes raise `NotImplementedError`. Note the SC drain +is currently commented out (`single_controller.py:571-584`) — straddles are +normal and absorbed by `group_min_wv` conservatism. + +## 7. Failure model + +| Failure | Consequence | Recovery | +|---|---|---| +| Worker stages, dies before responding | Gate call-timeout → call failed → no marker released → rollout cannot continue → placeholder | Staging TTL sweeps the orphan row | +| Worker responds, gate dies | Rollout dies (gate mediates it); no receipt | Staging TTL; SC dispatch timeout | +| Gate dies between seal and receipt delivery | Receipt rides the `run_rollouts` return; if lost, rollout unrecoverable → placeholder + TTL | Optional receipt persistence (H1) | +| Coords lost (response dropped) | Call-timeout → failed; agent never got completion → no child exists | TTL | +| Agent strips/edits markers | Fingerprint or marker miss → text fallback, new root | Correct but wasteful; visible as `token_in_rate` drop | +| SC dispatch cancelled / shutdown | `try/finally`: `fail_rollout` + clear staged keys by prefix + release buffer permit | Registration TTL backstop | +| NaN-logprob batch retry | Would re-register create-only ids | `rollout_max_attempts_to_avoid_lp_nan == 1` enforced at setup | + +Cleanup is two stores: the SC slot (+ its TQ rows, staging + canonical) and +the gate's per-rollout state (self-clears at seal / `fail_rollout` / +registration TTL). Slots record their rollout_ids at reserve time; +`TQReplayBuffer.remove(remove_in_dp=True)` clears both partitions. + +## 8. Metrics + +`token_in_rate` (marker hit), `fallback_rate` by cause (no-marker / +fingerprint-miss / unknown-marker), `capture_failure_rate`, +`digest_verify_failures`, `invalid_row_rate`, finalize p50/p99, `wv_spread`, +gate admission→commit latency, receipts lost, staging partition size, +per-call HTTP bytes (the headline number vs. the echo path). + +## 9. Component changes + +### 9.0 Runtime placement at a glance + +Who runs where, and the one-line ownership rule for each home: + +| Runtime home | Components | Owns | +|---|---|---| +| **SingleController Ray actor** (NeMo-RL) | `_rollout_pump`/`_train_pump`/`_sync_weights`; `RolloutManager` (mints rollout_ids); `TQReplayBuffer` (`reserve`/`commit_finalized`/`remove`); `BlackboxFinalizer` (runs in the dispatch task via `asyncio.to_thread` for the MVP); `StalenessSampler`; advantage pump | **Training assembly**: group semantics, slots, receipts → N canonical rows, staleness, cleanup | +| **NemoGym Ray actor** (NeMo-RL file wrapping Gym) | server spin-up; token-free `run_rollouts` + receipt unpacking; gate control-plane client (register/seal/`fail_rollout`) | The Ray↔HTTP boundary; no token logic | +| **Gym model server = the gate** (submodule fork = main + #2124 + gate work) | thin hosting of the pure **`lineage.py`** state machine; admission + call_id mint + TTLs; per-rollout **token buffer** (in-memory `token_id_capture` store impl); `ng_call_id` marker attach/resolve + history fingerprint; `prefix_ids` serving + rollout→worker affinity; coords ingestion = commit; seal → receipt | **Lineage custody**: everything that requires understanding *messages*. Holds token ids in flight; never logprobs; never writes TQ | +| **Gym agent + resources servers** | — | **Nothing new** (design goal: 27 agent impls untouched; markers ride messages opaquely; rewards via `/verify` as today) | +| **vLLM worker Ray actor** (NeMo-RL, hosting Gym's capture) | in-process HTTP server; **Gym `capture.py`** (engine-blind: record/digest build, fail-closed ordering, coords) + **Gym `adapters/vllm.py`** (splice, extraction) via `install_capture`; NeMo-RL provides `TQTokenSink.stage()` (the only heavy hop), the DP client, and `_rollout_weight_version` | **Token production + durability**: capture logic is Gym's; storage and hosting are NeMo-RL's; bytes durable before ack | +| **TransferQueue** | `rollout_staging` (finalizer is the only reader) + `rollout_data` (train pump is the only reader) | Bytes at rest | + +The wire records, digest, rebuild semantics, and protocols are defined +**once**, in Gym's dependency-free `token_id_capture` leaf package +(§ 3.0), and imported by both the gate and the framework's worker/finalizer +— importable anywhere because the core modules carry no heavy dependencies +(enforced in Gym CI). + +### 9.1 NeMo-RL + +There is **no** `nemo_rl/experience/token_capture/` package — records, +digest, protocols, rebuild, and capture logic live in Gym (§ 3.0, § 9.2). +NeMo-RL writes only the provider implementations and training assembly: + +| Component | Change | +|---|---| +| `nemo_rl/data_plane/tq_token_sink.py` (new) | implements Gym's `TokenSink` and `TokenSource` protocols over TQ (`put_samples` / `get_samples`); the only hot-path file that knows TQ | +| `nemo_rl/experience/blackbox_finalizer.py` (new) | orchestration only: fetch via `TokenSource`, call Gym's verify/`rebuild`/`linearize`, always-N placeholders, `group_min_wv`, publish to `rollout_data` | +| `algorithms/async_utils/replay_buffer.py` | `commit_finalized`; slots record rollout_ids; `remove` clears staging; `abort(group_id)`; fix `commit` on evicted slots (`:657`) | +| `algorithms/single_controller.py` | release `_buffer_capacity` in dispatch `finally` (`:319`); `set_rollout_weight_version` fan-out at `:591`; teardown sweep; metrics | +| `algorithms/single_controller_utils/setup.py` | pre-register `rollout_staging` + `rollout_data`; gate config into `spinup_nemo_gym_actor`; sink factory + `dp_cfg` to generation setup | +| `algorithms/single_controller_utils/config.py` | `TokenCaptureConfig` on `MasterConfig`; `AsyncRLConfig` additions | +| `models/generation/vllm/vllm_worker_async.py` | **hosting only**: one `install_capture(serving_layer, sink=TQTokenSink(...), weight_version_fn=...)` call at startup; `setup_token_capture(dp_cfg)` fan-out target building the in-worker DP client; `_rollout_weight_version` attribute. Capture logic moves to Gym: fail-closed ordering + record build in `capture.py`; prefix-in, splice (`_replace_prefix_tokens` relocates), extraction in `adapters/vllm.py` | +| `models/generation/vllm/vllm_generation.py` | `setup_token_capture` / `set_rollout_weight_version` fan-outs (existing `run_all_workers_single_data` pattern) | +| `experience/rollout_manager.py` | mint rollout_ids (thread `group_id` into `run_rollout`); receipt mode in `_result_to_completion` / `_compute_rollout_metrics` (incl. removing tokens from the wandb table, `:575`) | +| `environments/nemo_gym.py` | register/seal/fail control-plane helpers; receipt-mode `_postprocess` (drop token walk + contiguity assert `:329-388` — the gate owns that guarantee now); fix `run_rollouts` return annotation; NaN-retry hard error | +| `algorithms/advantage_estimator.py` | pass real validity into `calculate_baseline_and_std_per_prompt` (`:69`); validity folds into `sample_mask` (no new train field) | + +### 9.2 Gym (vendored submodule, fork branch **based on #2124**) + +The Gym implementation builds **on top of upstream PR #2124** (token id +capture core, `nemo_gym/token_id_capture/`), not beside it. The fork branch +is upstream main + #2124 (pinned to a specific rev of that PR) + the gate +work. What #2124 supplies and how it is used: + +- `records.py` `TokenEntry` (rollout id + server call id + prompt/gen ids + + logprobs + message content) → the gate's per-call buffer entry; this + design adopts its rollout/call-id grammar rather than minting a parallel + one. +- `sink.py` (records a `TokenEntry` from the finished model-server + response, streaming-safe) → the coords-ingestion point: extended to read + the worker's delta ids + `CommitCoords` and extend the rollout buffer. +- `store.py` (`TokenCaptureStore`, per-rollout JSONL, per-file locking) → + becomes the store interface behind the gate buffer: a new in-memory + implementation serves the hot path; the JSONL store is retained as an + optional debug/persistence backend and for H1 receipt persistence. +- `config.py` (`token_id_capture_enabled` + directory) and the + `base_responses_api_model.py` integration (record + install routes + + re-stream) → the on/off switch and wiring the gate extends. +- `routes.py` (`GET /ng-capture/tokens/{rollout_id}`) → retained; useful + as a debug read path beside TQ staging. + +The package is grown into the **integration SDK** under the § 3.0 purity +rule (core modules dependency-free, import-linter-enforced), so any +framework's worker can import it: + +| Component | Change | +|---|---| +| `nemo_gym/token_id_capture/` (from #2124) | base package: records/store/sink/config/routes as above; in-memory store impl added | +| `token_id_capture/staging/records.py` (new) | **single definition** of all wire shapes (§ 3.4), beside #2124's `TokenEntry` (whose `records.py` is untouched); `SCHEMA_VERSION` | +| `token_id_capture/staging/protocols.py` (new) | `TokenSink`, `TokenSource`, `WeightVersionProvider`, `CaptureAdapter`; `install_capture` entrypoint | +| `token_id_capture/staging/digest.py` (new) | `compute_staging_digest` + encoders (prototype `rollout_writer.py:1014` port; golden vectors) | +| `token_id_capture/staging/rebuild.py` (new) | pure functions: snapshots → entries → `linearize(policy, terminal_hint)` — identical training-row semantics for every framework (prototype `staged_token_source.py` core) | +| `token_id_capture/staging/lineage.py` (new) | `RolloutLineage` — pure per-rollout state machine (admit/commit/fail/seal → manifest); no HTTP, no store; unit-tested standalone at S1 | +| `token_id_capture/staging/capture.py` (new) | `RolloutTokenCapture.begin_call`/`complete_call` — engine-blind: record + digest build, fail-closed stage→respond ordering, coords assembly; tested against mock adapter + mock sink | +| `token_id_capture/adapters/vllm.py` (new) | engine-specific only: prefix-in entry, suffix splice (relocated `_replace_prefix_tokens`), native id+logprob extraction, serving-layer hookup. `adapters/sglang.py` is a later drop-in that inherits `capture.py`'s ordering for free | +| `token_id_capture/staging/conformance/` (new) | golden fixtures + installable test kit run by every framework/adapter CI | +| gate hosting (port from prototype `capture_gate.py`, `rollout_registry.py`) | **thin hosting of `lineage.py`**: admission + call-id mint, marker resolve + fingerprint, serving rule, coords ingestion, seal → receipt, `fail_rollout`/TTL; control router; Gym also ships the `RolloutControlClient` frameworks call | +| `responses_api_models/vllm_model/app.py` | per-rollout **token buffer**; marker → parent lookup + fingerprint check in `_preprocess_chat_completion_create_params` (`:259`); `prefix_ids` into `extra_body`; coords ingestion replacing the logprob-scrape + `/tokenize` block (`:497-551`); rollout affinity in `_resolve_client` (`:576`) | +| `nemo_gym/responses_converter.py` | attach `ng_call_id` marker instead of token arrays (`:362-374`) | +| rollout id plumbing | `responses_create_params.metadata` carrier end-to-end; no agent changes | + +**Gym dependency:** a fork branch of the submodule = upstream main + +**#2124 (pinned rev)** + the gate work; the gitlink must point at a rev +fetchable by CI (decision at the S1 gate: fork remote vs. NVIDIA-NeMo/Gym +branch). Because #2124 is adopted as the base, H4 upstreaming reduces to +contributing the gate/TQ layers and reconciling with the *rest* of the +stack (#2125–#2128). + +### 9.3 File manifest + +New files, **Gym fork** (base = main + #2124; all under the § 3.0 purity +rule except `adapters/` and the gate/server wiring): + +| File | Defines | Stage | +|---|---|---| +| `nemo_gym/token_id_capture/staging/records.py` | single wire schema (§ 3.4): `StagedCallRecord`/`CommitCoords`/receipt/manifest; `SCHEMA_VERSION` (#2124's `records.py` untouched) | S1 | +| `nemo_gym/token_id_capture/staging/protocols.py` | `TokenSink`, `TokenSource`, `WeightVersionProvider`, `CaptureAdapter`, `install_capture` | S1 | +| `nemo_gym/token_id_capture/staging/digest.py` | `compute_staging_digest` + encoders (prototype port; golden vectors) | S1 | +| `nemo_gym/token_id_capture/staging/lineage.py` | `RolloutLineage` pure state machine: admit/commit/fail/seal → manifest (no HTTP, no store) | S1 | +| `nemo_gym/token_id_capture/staging/rebuild.py` | snapshots → entries → `linearize(policy, terminal_hint)` (pure; prototype `staged_token_source.py` core) | S1 | +| `nemo_gym/token_id_capture/staging/conformance/` | golden fixtures + installable conformance kit | S1 | +| `nemo_gym/token_id_capture/staging/capture.py` | `RolloutTokenCapture` — engine-blind capture: record/digest build, fail-closed stage→respond ordering, coords assembly | S2 | +| `nemo_gym/token_id_capture/adapters/vllm.py` | engine-specific: prefix-in entry, suffix splice (relocated `_replace_prefix_tokens`), id+logprob extraction, serving-layer hookup | S2 | +| `nemo_gym/token_id_capture/memory_store.py` | in-memory rollout token buffer behind #2124's store interface | S3 | +| `nemo_gym/token_id_capture/gate.py` | thin hosting of `lineage.py`: admission, call_id mint, marker resolution + fingerprint, serving rule, receipt assembly | S3 | +| `nemo_gym/token_id_capture/control_routes.py` | register/seal/`fail_rollout` control API + `RolloutControlClient` | S3 | + +Modified, Gym fork: `responses_api_models/vllm_model/app.py` +(marker lookup + `prefix_ids` in `_preprocess…`; coords ingestion replacing +the logprob-scrape + `/tokenize`; affinity in `_resolve_client`; S3), +`nemo_gym/responses_converter.py` (marker instead of token arrays; S3), +`nemo_gym/base_responses_api_model.py` (install gate + routes; S3), +`nemo_gym/global_config.py` (config keys; S3) — details in § 9.2. + +New files, **NeMo-RL** (provider implementations + training assembly only): + +| File | Defines | Stage | +|---|---|---| +| `nemo_rl/data_plane/tq_token_sink.py` | `TQTokenSink` / `TQTokenSource` implementing Gym's protocols over `put_samples`/`get_samples` | S1 | +| `nemo_rl/experience/blackbox_finalizer.py` | orchestration over Gym verify/`rebuild`/`linearize`; `finalize_group` (always N rows, placeholders, `group_min_wv`) | S4 | + +Modified, NeMo-RL: `algorithms/async_utils/replay_buffer.py` (S1), +`algorithms/single_controller_utils/config.py` (S1), `.../setup.py` +(S1+S4), `algorithms/single_controller.py` (S1 fix + S4), +`models/generation/vllm/vllm_worker_async.py` (S2, hosting only), +`models/generation/vllm/vllm_generation.py` (S2, fan-outs), +`environments/nemo_gym.py` (S3-RL+S4), `experience/rollout_manager.py` +(S4), `algorithms/advantage_estimator.py` (S4), exemplar YAML +`examples/configs/grpo_math_1B_single_controller.yaml` (S4) — details in +§ 9.1. + +Placement rule: protocols, records, digest, rebuild, and capture logic are +defined once in Gym's leaf package and imported everywhere (venv check at +the S1 gate); NeMo-RL contributes storage implementations, hosting, and +training assembly. Another RL framework integrates by re-implementing only +the two NeMo-RL "new files" rows; another backend by one `adapters/` file. + +## 10. Implementation plan + +### MVP — one PR, five sign-off-gated stages + +Single branch, one PR; each stage a signed-off commit series keeping the +tree green (S1–S3 dormant behind `token_capture.enabled=false`); each ends +at a review gate (stage diff, test results, deviations presented for +explicit sign-off). **Wire shapes (§ 3.4) and the serving rule (§ 3.3) +freeze at the S1 gate.** + +- **S1 — primitives (Gym fork) + buffer surgery (RL repo).** Gym fork: + `token_id_capture` records (single wire schema), `protocols.py`, + `digest.py` (golden vectors vs. prototype), **`lineage.py`** (pure state + machine), `rebuild.py`, conformance kit, import-linter purity test. RL: + `TQTokenSink`/`TQTokenSource` implementing the protocols; + `TokenCaptureConfig`; `TQReplayBuffer` `commit_finalized` / rollout_ids + on slots / staging-aware `remove` / `abort` / evicted-slot fix; + `_buffer_capacity` leak fix; partition pre-registration. Unit tests: + records/digest vectors; **lineage state machine standalone** (admit/ + commit/fail/seal transitions, manifest ordering, TTL/fail paths, fork + topologies) — lineage logic is fully tested two stages before it + touches HTTP; conformance kit green on the TQ implementations; buffer + ops incl. eviction/cleanup. + **S1-gate checklist:** submodule fork logistics decided; the leaf + package importable in the vLLM worker venv; vLLM prefix-ids path + + splice validated for the functional-test templates. +- **S2 — capture core + vLLM adapter (Gym fork) + worker hosting (RL + repo).** Gym, two layers: **`capture.py`** — engine-blind + `RolloutTokenCapture` (record + digest build, fail-closed stage→respond + ordering, coords + delta ids on the response, non-streaming assert), + tested against a **mock adapter + mock sink** so the ordering matrix is + backend-independent; **`adapters/vllm.py`** — engine-specific only: + `extra_body` prefix-in feeding the relocated `_replace_prefix_tokens` + splice, native id+logprob extraction, serving-layer hookup, with its own + per-template splice goldens. RL: `install_capture` call at worker + startup, DP client + `setup_token_capture` fan-out, + `_rollout_weight_version` + `set_rollout_weight_version` fan-out; + hosting/fan-out/version-stamping tests. +- **S3 — Gym gate (submodule fork, based on #2124).** Rebase the fork onto + upstream main + #2124 (pinned rev; run its 20-test capture suite as the + base sanity check). **Host the S1 `lineage.py` machine in the gate** — + no new lineage logic lands here, only hosting: in-memory store impl, + admission + call-id mint ports (prototype donors), marker attach/resolve + + fingerprint, the serving rule, prefix serving, coords ingestion via + the #2124 sink seam, rollout affinity, token-free receipts, TTLs, + control routes + `RolloutControlClient`. Conformance tests: the S1 + golden call sequences replayed **through the gate** → manifests + byte-identical to S1's direct-drive results; marker-stripped and + history-edited fallbacks; duplicate-coords / wrong-rollout rejection; + timeout; #2124 suite stays green. +- **S4 — receipts, finalizer, SC integration.** Receipt-mode + `run_rollouts` (tokens removed from message_log, `full_result`, wandb + table); `blackbox_finalizer` + `finalize_group` (always N rows, + placeholders, `group_min_wv`/`group_max_wv`, + `min_valid_fraction_per_group`); validity-aware baseline (unit-tested); + config validation (MVP matrix only); setup wiring; `commit_finalized` + + cleanup; teardown sweep; NaN-retry hard error; metrics (§ 8). Gate + evidence: 2-GPU manual run with a sub-agent fork → two-root manifest → + trained main chain. +- **S5 — verification.** Fixed-seed job on legacy and capture paths, + training rows dumped from TQ and diffed offline (token-in should be + byte-identical where the legacy echo path drifts); reward-curve + comparison; per-call HTTP bytes measured vs. the echo path; chaos smoke + (kill gate mid-step → placeholders + TTL, no leaks); capture-enabled + functional test in `L1_Functional_Tests_SingleController.sh`. + S5 sign-off = MVP acceptance. + +### Hardening (ordered) + +- **H1 — failure sweep + chaos.** Gate death mid-step, coords loss, + receipt loss; optional receipt persistence if S5 measures meaningful + loss; kill-gate CI test. First, because bring-up debugging kills actors. +- **H2 — hash layer.** `cum_hash`/`chain_hash` fill the reserved record + fields: chain re-verification at finalize (tamper evidence) and + worker-side prefix confirm — which relaxes the strict serving rule + (uncertain marker → one hash check instead of a full-render new root). +- **H3 — atomic weight-version rotation** (v2 § 4.4 TODO) then **strict + modes**: `strict_on_policy` (drain re-enabled), `reject`, `wait` + (verl's dropless pattern), `force_in_order`. +- **H4 — upstream into Gym proper.** With the § 3.0 structure the code is + already in its final home: upstreaming is merging the fork branch onto + the merged #2124 and reconciling with the remainder of the stack (#2125 + trajectory builder, #2126 delivery/scoping, #2127 on-policy pin, #2128 + example); publish the conformance kit as the multi-framework contract. +- **H5 — perf + scale.** Incremental suffix tokenization (drop the double + render); finalizer isolation/pool; admission caps (calls/rollout, buffer + bytes/rollout — the design's unbounded resources); gate scale-out with + rollout affinity; SGLang adapter. + +**Post-MVP validation:** 1-off async nightly in capture mode; perf report +reproducing the sync prototype measurements (HTTP bytes/token, step time) +plus `token_in_rate` and gate latency. + +## 11. Risks + +- **Gym submodule fork on the MVP critical path.** Mitigations: donor gate + code is proven (prototype `6ea5810`); the fork is ours; the gitlink/CI + question is forced at the S1 gate, not discovered late. +- **#2124 is an open PR.** Basing the fork on it means rebase churn if it + changes before merging. Mitigations: pin the fork to a specific #2124 + rev; keep gate code in separate modules touching the base package only + through its public seams (records/store/sink/config), so a rebase is + mechanical; track the PR during S3. +- **Protocols in the fork raise iteration friction**: every schema/protocol + change during bring-up touches the Gym fork even for RL-only work. + Mitigated by the fork being a local editable workspace member (no + publish cycle) and by freezing shapes at the S1 gate. +- **Leaf-package availability in worker venvs.** The purity rule makes the + import safe, but each framework's worker environment must actually + contain `nemo_gym` (or a separately published leaf distribution). + Checked at the S1 gate for NeMo-RL's vLLM `py_executable`. +- **Marker survival across agent frameworks.** The marker uses the exact + carrier today's token echo uses, so it survives wherever the current + pipeline works; an agent that strips unknown fields degrades to text + fallback — correct but wasteful. `token_in_rate` is first-class from S4. +- **Gate as stateful hot-path service** (per-rollout token buffers, KBs–MBs + × in-flight rollouts). Buffer bytes are bounded by H5 caps; state + self-clears at seal/TTL; crash blast radius measured in S5 chaos; and + the lineage state machine is a pure, separately-tested class + (`lineage.py`, S1), so gate hosting stays thin. +- **No cross-row chain integrity in the MVP** (§ 5): accepted; the gate is + inside the existing trust boundary; H2 restores tamper evidence. +- **Fingerprint definition** (message normalization for equality) must be + pinned in S3 conformance tests — too strict inflates fallbacks, too loose + misses edits. Fallback-on-mismatch keeps both errors safe. +- **Finalize latency, placeholder-heavy groups, checkpointing**: unchanged + from v2 § 8 (in-flight rollouts abandoned on restore; TTL + create-only + registration make this safe). + +## Appendix A — prior art + +- **verl**: token-in/token-out rationale ("apply_chat_template to final + history makes PPO not converge"); TQ meta-passing; `global_steps` spans + + `drop|wait` staleness policies. No forest — linear trajectories only. +- **slime** (commit `ea9819f`): message-tree lineage with prefix-match + + rewrite-merge; consistent-hash session→engine affinity + (`X-SMG-Routing-Key`) mirrored by this design's rollout affinity. The + earlier drafts adopted its mount-point walk as a lineage hint; this + design replaces content inference entirely with the explicit marker — + the failure modes slime papers over (ambiguity, rewrites) become + explicit fallbacks here. +- **Gym #2124–#2128**: token-id capture stack for external-agent training + (issue #1824). **#2124 (capture core: records/store/sink/config/routes) + is the adopted base for this design's Gym work** (§ 9.2); it has no + gate/lineage/prefix-serving surface — that is what this design adds on + top. #2125–#2128 (trajectory builder, delivery/scoping, on-policy pin, + example) are reconciled at H4. +- **The sync prototype** (`/lustre/fsw/.../gym/RL`, branch + `pranav/tq_gym_prototype`): proved the staging dataflow and the perf + numbers (−46.9 % HTTP bytes/token, −55.7 % exchanges, −4.3 % step time + p50); donor for the digest scheme, staging sink, finalizer skeleton, and + the gate's admission/control plane. diff --git a/docs/design-docs/tq-gym-tokidcap-migration-log.md b/docs/design-docs/tq-gym-tokidcap-migration-log.md new file mode 100644 index 00000000000..f7893770878 --- /dev/null +++ b/docs/design-docs/tq-gym-tokidcap-migration-log.md @@ -0,0 +1,122 @@ +# Tokidcap Migration — Implementation Log + +Working log for executing [tq-gym-tokidcap-migration.md](tq-gym-tokidcap-migration.md) +(re-basing the gate-authoritative token-capture work onto the upstream Gym +token-id-capture stack). Companion to the MVP-era +[tq-gym-gate-authoritative-implementation-log.md](tq-gym-gate-authoritative-implementation-log.md), +which remains the record for the fork-based S1–S5 work. + +Conventions: every change lands here with its **status** +(`planned` → `in-progress` → `done ()` / `dropped ()`), the seam +or finding it serves, and any divergence from the plan doc. Un-gated behavior +changes (active with `token_capture.enabled=false`) get an explicit +**DISCLOSURE** marker. + +## Base facts (pinned) + +- Upstream stack: linear chain #2190→#2124→#2125→#2126→#2180→#2181→#2182, + verified 2026-07-31 in the submodule clone; **stack top = `81ac2736`** + (#2182, "require a bearer token on the token read route"). +- New Gym branch: `tq-tokidcap-capture`, cut from `81ac2736` (exact rev, + §6 finding G). Old fork branch `tq-gate-capture` (`e3b3eac6`, base + `fa0c2da3`) is left untouched as the cherry-pick donor. +- RL branch: work proceeds on `yukih/sc-entrypoint`; the §9b.1a companion-base + question (nano-SWE branch `3fcc69666`, owner Zhiyu Li) is an open + coordination point — capture commits are kept clean for later cherry-pick. +- Upstream asks: none landed as of 2026-07-31; every seam uses its §4 + workaround, contained in our modules. Ask outcomes shrink the diff later. + +## Workaround ledger (asks not landed) + +| Seam | Workaround in effect | Where | +|---|---|---| +| #2124-c1 install_token_sink honored | gate installs its manifest-observer sink by construction (we own the gate module); no base patch needed for worker-locus MVP | gate.py | +| #2124-c2 mark_incomplete on protocol | capture-poison flows through our CommitCoords/lineage, not the base sink protocol | staging/lineage.py | +| #2124-c3 commit_entry split | coords ingestion implemented in our gate module, feeding `RolloutLineage.record` on the base `LineageIndex` directly | gate.py | +| #2124-c4 schema_version | version carried in our staging records/receipts only | staging/records.py | +| #2126-c2 opaque rollout_id key | contained fork edit: accept an opaque `_ng_rollout_id` run-body key in `rollout_correlation.py` (shape-identical to the ask; collapses when it lands) — done (`1bdd7cdc`) | rollout_correlation.py | +| (new, to post as an ask) run-wide agent opt-in | `token_id_capture_all_agents` global key treats every agent as `token_id_capture=true` — the SC cannot enumerate agent servers configured via `config_paths` — done (`b6051536`) | base_responses_api_agent.py | +| #2180-c1 set_lineage_index | gate constructs a capacity-sized `LineageIndex` and replaces `sink._LINEAGE` before first request; attr name pinned in the bump checklist below | gate.py | +| #2180-c2 capacity config + eviction metric | comes free with the above (we construct the instance); eviction counter polled into gate metrics | gate.py | +| #2180-c3 resolver seam | not needed for MVP (no marker); Stage-2 exit kept in mind | — | +| #2181-c1 ng_capture hook | attached in `vllm_model/app.py` chat path (file we already edit for the gate); the highest-churn-risk edit, kept minimal | vllm_model/app.py | +| #2181-c2 required_prefix_token_ids contract | consumed as-is by the worker splice (proven code); rename risk noted in bump checklist | adapters/vllm.py | + +**Bump checklist** (things a stack rebase can silently break): `sink._LINEAGE` +attr name; `required_prefix_token_ids` field name; `model_call_id` mint site +(`base_responses_api_model.py` `_CaptureMiddleware`); `run_builder` / +`BuildNotes.unresolved_retries` shape; `/ng-rollout/` prefix regex. + +## Commit series (Gym branch `tq-tokidcap-capture`) + +Per plan §9b.2; each commit = a future stack PR. Status updated as they land. + +| # | Commit | Status | Notes | +|---|---|---|---| +| 1 | staging wire schema + digest (`staging/records.py`, `digest.py`, `protocols.py`; sink protocols renamed `StagingSink`/`StagingSource`) | done (`79540d2e`) | digest golden vectors unchanged; purity + records/digest tests in `test_token_capture_staging_core.py` (10 green) | +| 2 | terminal-aware linearize over `run_builder` (thin `staging/rebuild.py`) | done (`3d813fee`) | `snapshots_to_entries` + `LinearizedRow` kept; manifest walk deleted; terminal hint overrides token-mass pick (test proves the fork case); `unresolved_retries` → `RebuildError` → placeholder; 18 tests green | +| 3 | engine-blind capture core + vLLM adapter (`staging/capture.py`, `adapters/vllm.py`) | done (`232c7a43`) | module is identity-agnostic; rekey lands at call sites; 47 tests green | +| 4 | gate hosting, prefix serving, control plane (`gate.py`, `staging/lineage.py`, `control_routes.py` + bearer auth; seam edits in `vllm_model/app.py`) | done (`e1a9b4e7`) | identity switch complete (URL-prefix + `model_call_id`, marker plumbing gone); gate hosts capacity-sized eviction-counting `LineageIndex` via `sink._LINEAGE` replacement; coords ingestion feeds `LineageIndex.record`; bearer auth default-required; bounded client deadlines; conformance kit byte-exact through the `run_builder` linearize; e2e suite rehosted (ambiguity, auth, unknown-id rejection, flag-off dormancy); 199 tests green | +| 5 | observability (byte-counter middleware cherry-pick, unattributed-call counter; fallback-by-cause + eviction counter landed in commit 4) | done (`04597a3a`) | one adaptation: fork's `Dict` annotation → builtin `dict` (base dropped the import) | + +Deleted relative to the fork (base owns them): flat capture core +(`records`/`sink`/`store`/`config`/`routes`/`reader`/`source` + lazy +`__init__`), `memory_store.py`, most of `rebuild.py`, all marker plumbing +(`openai_utils.py` / `responses_converter.py` `*WithMarker` classes, +`find_marker`, `NG_CALL_ID_FIELD` message stamping), every fork edit to +`base_responses_api_model.py`. + +## RL-side changes (branch `yukih/sc-entrypoint`) + +| Change | Status | Notes | +|---|---|---| +| Re-pin submodule to `tq-tokidcap-capture` | in-progress | gitlink commits with the RL alignment; NRL_FORCE_REBUILD_VENVS on first run after | +| Identity carrier switch (drop `metadata["ng_rollout_id"]`; `_ng_rollout_id` run-body key → `/ng-rollout/`) | done (uncommitted) | rollout_manager `_build_inputs`, nemo_gym.py; verified `run_examples` posts rows verbatim so the key reaches the agent | +| Per-agent correlation opt-in | done (uncommitted) | SC sets `token_id_capture_all_agents=true` (Gym `b6051536`) | +| Gate hosting config: lineage capacity (derived: rollouts = 2×in-flight, tokens = rollouts×max seq len), bearer token (minted per run), capture dir (`/gym_token_capture`, the #2124-c1 workaround) | done (uncommitted) | TokenCaptureConfig + setup.py derivation + nemo_gym.py injection + exemplar YAML | +| Bounded control-plane deadline (`control_timeout_s`, default 60 s) | done (uncommitted) | `asyncio.wait_for` around every `_control` call (S5 silent-stall finding, H1 pulled into Stage 1) | +| Control-plane bearer auth on the client | done (uncommitted) | Authorization header on every `_control` call | +| Rekey staging keys to `{rollout_id}/{model_call_id}` | done (no code change) | keys minted Gym-side; worker/finalizer identity-agnostic | +| Finalizer: rebuild via `run_builder` wrapper, terminal hint + `unresolved_retries` → placeholder | done (no code change) | new Gym `linearize` keeps the signature; `unresolved_retries` surfaces as `RebuildError` → existing `rebuild_failed` placeholder path | +| Fix dead `staging_ttl_s` config | deferred | still unread (as in the fork MVP); H1 scope with the failure sweep | +| `uv.lock` regeneration for the new pin | done (uncommitted) | the stack top dropped Gym's `docs` dependency-group → lock update required; regenerated with uv 0.11.6 (the version family that wrote revision 3) for a minimal 934-line-deletion diff — uv 0.12 rewrites the whole lock to revision 4, avoid it for this repo | + +Note: the SC seals without an explicit `terminal_call_id` (it cannot know it); +the receipt's terminal defaults to the last-committed call (chronological), +which the finalizer's terminal-aware selection consumes. A background +sub-agent that commits after the main conversation's final call would win +the hint — accepted for Stage 1, revisit if the A/B row diff surfaces it. + +## Test gates + +- [ ] Gym: base capture suite + full unit suite green at every commit +- [ ] Gym: `staging/` purity test (subprocess import, no fastapi/ray/torch) +- [ ] Gym: conformance kit + gate e2e rehosted on stack request path +- [ ] Gym: flag-off byte-identity (capture disabled ⇒ legacy path unchanged) +- [ ] RL: capture unit tests (`--nemo-gym-only`) green on new pin +- [ ] RL: flag-off 2-GPU functional (`grpo_async_gym_single_controller.sh`) — + pin-bump regression +- [ ] RL: capture-enabled functional; fixed-seed A/B row diff; chaos smoke + +## Disclosures (active with the flag off) + +(none yet — all Gym-side behavior changes are behind `token_capture_gate.enabled` +or `NG_HTTP_BYTES_DIR`; the `_resolve_client` signature gained an optional +`rollout_id=None` parameter, a no-op when unset) + +## Divergences from the plan doc + +- **Gate activation requires the base token capture enabled with a real + capture dir** (`token_id_capture_enabled=true` + `token_id_capture_dir`) — + the #2124-c1 "activation without a capture dir" ask is not landed, and the + capture middleware only mints `model_call_id`/sets the capture context when + a token store exists. Cost: the base `capture_tokens` no-ops on the gate + path (worker strips token fields before the response), so the dir stays + ~empty; the RL launcher points it at the run's log dir. +- **Editing a user turn no longer breaks lineage.** The fork's fingerprint + covered the full history; the base's covers model-authored turns only, so + a harness that rewrites user/tool content keeps its chain (by design + upstream). The e2e "edited history" test now edits the assistant turn. +- **Fallback cause names** are `no_history` / `no_match` / `ambiguous` + (plan §8 sketched `no_prefix`/`no_match`/`ambiguity`/`multi_worker`; + `multi_worker` is not distinguishable at the gate and is deferred). diff --git a/docs/index.md b/docs/index.md index 92f2989eb20..ed0c0292cba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -325,6 +325,10 @@ design-docs/training-backends.md design-docs/sequence-packing-and-dynamic-batching.md design-docs/env-vars.md design-docs/nemo-gym-integration.md +design-docs/tq-gym-async-single-controller.md +design-docs/tq-gym-gate-authoritative.md +design-docs/tq-gym-gate-authoritative-setup.md +design-docs/tq-gym-gate-authoritative-implementation-log.md ``` ```{toctree} diff --git a/examples/configs/grpo_math_1B_single_controller.yaml b/examples/configs/grpo_math_1B_single_controller.yaml new file mode 100644 index 00000000000..db87aef172e --- /dev/null +++ b/examples/configs/grpo_math_1B_single_controller.yaml @@ -0,0 +1,395 @@ +# GRPO via SingleController (async-RL) — mirrors grpo_math_1B.yaml with +# data_plane.enabled=true and a top-level async_rl: section holding the +# SC-specific runtime knobs. +grpo: + num_prompts_per_step: 32 + num_generations_per_prompt: 16 + max_rollout_turns: 1 + max_num_epochs: 1 + max_num_steps: 1000000 + normalize_rewards: true + use_leave_one_out_baseline: true + val_period: 10 + val_at_start: false + val_at_end: false + overlong_filtering: false + advantage_clip_low: null + advantage_clip_high: null + max_val_samples: 256 + val_batch_size: 256 + seed: 42 + use_dynamic_sampling: false + dynamic_sampling_max_gen_batches: 10 + batch_multiplier: 1 + reward_shaping: + enabled: false + overlong_buffer_length: 128 + overlong_buffer_penalty: 1 + max_response_length: ${policy.max_total_sequence_length} + stop_properly_penalty_coef: null + + adv_estimator: + name: "grpo" + normalize_rewards: ${grpo.normalize_rewards} + use_leave_one_out_baseline: ${grpo.use_leave_one_out_baseline} + minus_baseline: true + reward_scaling: + enabled: false + source_min: 0.0 + source_max: 1.0 + target_min: 0.0 + target_max: 1.0 + seq_logprob_error_threshold: null + invalid_tool_call_advantage: null + malformed_thinking_advantage: null + + async_grpo: + enabled: true + max_trajectory_age_steps: 1 + in_flight_weight_updates: false + recompute_kv_cache_after_weight_updates: false + +loss_fn: + reference_policy_kl_penalty: 0.01 + reference_policy_kl_type: "k3" + kl_input_clamp_value: 20.0 + kl_output_clamp_value: 10.0 + ratio_clip_min: 0.2 + ratio_clip_max: 0.2 + ratio_clip_c: null + use_on_policy_kl_approximation: false + use_importance_sampling_correction: false + truncated_importance_sampling_type: null + truncated_importance_sampling_ratio: null + truncated_importance_sampling_ratio_min: null + sequence_level_importance_ratios: false + token_level_loss: true + force_on_policy_ratio: false + use_kl_in_reward: false + disable_ppo_ratio: false + positive_example_nll_weight: 0.0 + +checkpointing: + enabled: true + checkpoint_dir: "results/grpo-single-controller" + metric_name: "val:accuracy" + higher_is_better: true + keep_top_k: 3 + save_period: 10 + checkpoint_must_save_by: null + model_save_format: "safetensors" + save_consolidated: false + save_optimizer: true + +policy: + model_name: "Qwen/Qwen2.5-1.5B" + tokenizer: + name: ${policy.model_name} + chat_template_kwargs: null + hf_config_overrides: {} + train_global_batch_size: 512 + train_micro_batch_size: 4 + generation_batch_size: 32 + logprob_batch_size: ${policy.train_micro_batch_size} + max_total_sequence_length: 512 + precision: "bfloat16" + logprob_chunk_size: null + offload_optimizer_for_logprob: false + + dtensor_cfg: + _v2: true + enabled: false + cpu_offload: False + sequence_parallel: false + activation_checkpointing: false + tensor_parallel_size: 1 + context_parallel_size: 1 + custom_parallel_plan: null + automodel_kwargs: {} + lora_cfg: + enabled: False + target_modules: [] + exclude_modules: [] + match_all_linear: true + dim: 8 + alpha: 32 + dropout: 0.0 + dropout_position: "post" + lora_A_init: "xavier" + use_triton: true + + megatron_cfg: + enabled: true + force_reconvert_from_hf: False + empty_unused_memory_level: 1 + activation_checkpointing: false + recompute_granularity: "full" + recompute_modules: null + tensor_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + expert_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + num_layers_in_first_pipeline_stage: null + num_layers_in_last_pipeline_stage: null + context_parallel_size: 1 + pipeline_dtype: ${policy.precision} + sequence_parallel: false + freeze_moe_router: true + moe_router_dtype: "fp64" + moe_router_load_balancing_type: "none" + moe_router_bias_update_rate: 0.0 + moe_permute_fusion: true + apply_rope_fusion: True + bias_activation_fusion: True + defer_fp32_logits: False + moe_per_layer_logging: False + moe_enable_deepep: false + moe_token_dispatcher_type: "alltoall" + moe_shared_expert_overlap: false + gradient_accumulation_fusion: false + use_fused_weighted_squared_relu: false + peft: + enabled: false + target_modules: [] + exclude_modules: [] + dim: 8 + alpha: 32 + dropout: 0.0 + dropout_position: "post" + lora_A_init_method: "xavier" + lora_B_init_method: "zero" + a2a_experimental: false + lora_dtype: None + optimizer: + optimizer: "adam" + lr: 5.0e-6 + min_lr: 5.0e-7 + weight_decay: 0.01 + bf16: true + fp16: false + params_dtype: "float32" + adam_beta1: 0.9 + adam_beta2: 0.999 + adam_eps: 1e-8 + sgd_momentum: 0.9 + use_distributed_optimizer: true + use_precision_aware_optimizer: true + clip_grad: ${policy.max_grad_norm} + optimizer_cpu_offload: false + optimizer_offload_fraction: 0.0 + scheduler: + start_weight_decay: ${policy.megatron_cfg.optimizer.weight_decay} + end_weight_decay: ${policy.megatron_cfg.optimizer.weight_decay} + weight_decay_incr_style: "constant" + lr_decay_style: "constant" + lr_decay_iters: 1000 + lr_warmup_iters: 13 + lr_warmup_init: 5.0e-7 + distributed_data_parallel_config: + grad_reduce_in_fp32: false + overlap_grad_reduce: true + overlap_param_gather: true + use_custom_fsdp: false + data_parallel_sharding_strategy: "optim_grads_params" + fp8_cfg: + enabled: false + fp8: "e4m3" + fp8_recipe: "blockwise" + fp8_param: false + env_vars: null + + draft: + enabled: false + model_name: null + loss_weight: 0.1 + num_layers: null + aux_layer_indices: null + + dynamic_batching: + enabled: False + train_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.train_micro_batch_size}} + logprob_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.logprob_batch_size}} + sequence_length_round: 64 + + sequence_packing: + enabled: True + train_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.train_micro_batch_size}} + logprob_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.logprob_batch_size}} + algorithm: "modified_first_fit_decreasing" + sequence_length_round: 64 + + make_sequence_length_divisible_by: ${policy.dtensor_cfg.tensor_parallel_size} + max_grad_norm: 1.0 + + optimizer: + name: "torch.optim.AdamW" + kwargs: + lr: 5.0e-6 + weight_decay: 0.01 + betas: [0.9, 0.999] + eps: 1e-8 + + scheduler: + - name: "torch.optim.lr_scheduler.LinearLR" + kwargs: + start_factor: 0.1 + end_factor: 1.0 + total_iters: 50 + - name: "torch.optim.lr_scheduler.ConstantLR" + kwargs: + factor: 1.0 + total_iters: 10000000000 + - milestones: [50] + + generation: + port_range_low: 11001 + port_range_high: 15000 + backend: "vllm" + max_new_tokens: ${policy.max_total_sequence_length} + temperature: 1.0 + top_p: 1.0 + top_k: null + stop_token_ids: null + stop_strings: null + mcore_generation_config: + buffer_size_gb: 10 + num_cuda_graphs: 4 + block_size_tokens: 256 + use_cuda_graphs_for_non_decode_steps: true + enable_chunked_prefill: true + unified_memory_level: 0 + max_tokens: 16384 + vllm_cfg: + async_engine: true + precision: ${policy.precision} + kv_cache_dtype: "auto" + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + expert_parallel_size: 1 + gpu_memory_utilization: 0.6 + max_model_len: ${policy.max_total_sequence_length} + enforce_eager: False + use_tqdm: true + use_deep_gemm: False + num_last_layers_in_bf16: 0 + num_first_layers_in_bf16: 0 + enable_vllm_metrics_logger: true + vllm_metrics_logger_interval: 0.5 + vllm_kwargs: {} + colocated: + enabled: false + resources: + gpus_per_node: 1 + num_nodes: 1 + +data: + max_input_seq_length: ${policy.max_total_sequence_length} + shuffle: true + num_workers: 1 + use_multiple_dataloader: false + train: + dataset_name: OpenMathInstruct-2 + split_validation_size: 0.05 + seed: ${grpo.seed} + validation: null + default: + prompt_file: "examples/prompts/cot.txt" + system_prompt_file: null + processor: "math_hf_data_processor" + env_name: "math" + +env: + math: + num_workers: 8 + math_verify_impl: "hf_math_verify" + +logger: + log_dir: "logs" + num_val_samples_to_print: 0 + wandb_enabled: false + tensorboard_enabled: false + mlflow_enabled: false + swanlab_enabled: false + monitor_gpus: true + wandb: + project: "grpo-dev" + name: "grpo-single-controller-dev" + swanlab: + project: "grpo-dev" + name: "grpo-single-controller-dev" + tensorboard: {} + mlflow: + experiment_name: "grpo-dev" + run_name: "grpo-single-controller-dev" + tracking_uri: "http://localhost:5000" + gpu_monitoring: + collection_interval: 10 + flush_interval: 10 + +# TransferQueue data plane — required by the SingleController path. +data_plane: + enabled: true + impl: transfer_queue + backend: "simple" + storage_capacity: 1000000 + num_storage_units: 2 + claim_meta_poll_interval_s: 0.5 + global_segment_size: 549755813888 + local_buffer_size: 68719476736 + +# SC-specific async-RL runtime knobs. +# One training step consumes grpo.num_prompts_per_step prompt groups. +async_rl: + batch_selection_strategy: "strict_on_policy" # or "staleness_window" + max_weight_staleness_versions: 1 + min_prompt_groups_per_batch: 2 + max_inflight_prompts: 8 + # When over_sampling=false this must equal + # grpo.num_prompts_per_step * (max_weight_staleness_versions + 1). + max_buffered_rollouts: 8 + # True : over-generates and wastes rollouts that age past the staleness window; + # False: enforces per-weight-version dispatch quota. + over_sampling: true + +# Gate-authoritative token capture (token-in/token-out via NeMo-Gym; see +# docs/design-docs/tq-gym-gate-authoritative.md). Dormant by default: with +# enabled=false every legacy codepath behaves exactly as before. Requires the +# NeMo-Gym rollout path (env.should_use_nemo_gym=true) with the async vLLM +# backend; defaults live on TokenCaptureConfig +# (nemo_rl/algorithms/single_controller_utils/config.py). +token_capture: + enabled: false + # TQ partition holding per-call staged token deltas (finalizer-cleared). + staging_partition: "rollout_staging" + # continue: a failed worker-side stage poisons the rollout (placeholder row); + # abort: fails the whole rollout at the gate. + on_capture_failure: "continue" + # allow: train groups whose calls span a refit (staleness = group's oldest + # call version); reject: placeholder such rollouts. + mixed_weight_version_policy: "allow" + # Drop the whole group when fewer than this fraction of its rollouts + # produced valid rows (null keeps every group). + min_valid_fraction_per_group: null + # Gate-side cleanup backstops (seconds). + registration_ttl_s: 3600.0 + staging_ttl_s: 3600.0 + # Gym LineageIndex capacity (each in-flight rollout's cumulative tokens + # live in the gate process). null = derived from the training config: + # rollouts = 2 x max in-flight, tokens = rollouts x max sequence length. + # Size explicitly for agentic workloads with deep per-rollout call trees. + lineage_max_rollouts: null + lineage_max_tokens: null + # Bearer token for the gate's /ng-control/* routes. null = minted per run. + control_auth_token: null + # Hard deadline per control-plane call (gate death must surface as failed + # dispatches + placeholders, not a silent retry stall). + control_timeout_s: 60.0 + # Directory for the Gym base capture layer the gate rides on (stays + # essentially empty on the gate path). null = /gym_token_capture. + capture_dir: null + +cluster: + gpus_per_node: 2 + num_nodes: 1 + master_port_range_low: 25000 + master_port_range_high: 28000 diff --git a/examples/configs/recipes/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.yaml b/examples/configs/recipes/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.yaml new file mode 100644 index 00000000000..ee3f8b97acd --- /dev/null +++ b/examples/configs/recipes/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.yaml @@ -0,0 +1,33 @@ +# SingleController variant of grpo-llama3.1-8b-instruct-2n8g-async-1off.yaml. +# Same training topology and async-1off strategy; the SC path routes +# everything through the TransferQueue data plane + SingleControllerActor. +# staleness_window + over_sampling=false replicate the per-version dispatch +# quota of the original async-grpo path. +defaults: ./performance/grpo-llama3.1-8b-instruct-2n8g-async-1off.yaml + +logger: + log_dir: logs/grpo-llama3.1-8b-instruct-2n8g-async-1off-sc + wandb: + name: grpo-llama3.1-8b-instruct-2n8g-async-1off-sc + +checkpointing: + checkpoint_dir: results/grpo-llama3.1-8b-instruct-2n8g-async-1off-sc + +# TransferQueue data plane is mandatory for the SingleController path. +data_plane: + enabled: true + +# SC async-RL runtime knobs. +async_rl: + batch_selection_strategy: staleness_window + # Matches grpo.async_grpo.max_trajectory_age_steps=1. + max_weight_staleness_versions: 1 + # One training step consumes grpo.num_prompts_per_step (=64) prompt groups. + min_prompt_groups_per_batch: 64 + max_inflight_prompts: ${grpo.num_prompts_per_step} # match grpo-llama3.1-8b-instruct-2n8g-async-1off + # over_sampling=false requires + # max_buffered_rollouts == grpo.num_prompts_per_step * (max_weight_staleness_versions + 1) + # 64 * (1 + 1) = 128 + max_buffered_rollouts: 128 + over_sampling: false + force_in_order: true diff --git a/examples/configs/recipes/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.yaml b/examples/configs/recipes/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.yaml new file mode 100644 index 00000000000..949bd31f11b --- /dev/null +++ b/examples/configs/recipes/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.yaml @@ -0,0 +1,49 @@ +# SingleController + Megatron variant of grpo-qwen2.5-math-1.5b-instruct-1n8g. +# SC currently only supports non-colocated generation; the 8 GPUs on the node +# are split 4 (train) + 4 (inference). The SC path routes everything through +# the TransferQueue data plane + SingleControllerActor. strict_on_policy +# auto-sets max_weight_staleness_versions=0 and over_sampling=False, enforcing +# a strict per-version dispatch quota. +defaults: ./grpo-qwen2.5-math-1.5b-instruct-1n8g-fsdp2tp1.v3.yaml + +logger: + log_dir: logs/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-sc + wandb: + name: grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-sc + +checkpointing: + checkpoint_dir: results/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-sc + +# TransferQueue data plane is mandatory for the SingleController path. +data_plane: + enabled: true + +# SC async-RL runtime knobs. +async_rl: + batch_selection_strategy: strict_on_policy + # One training step consumes grpo.num_prompts_per_step (=32) prompt groups. + min_prompt_groups_per_batch: 32 + max_inflight_prompts: 64 + # strict_on_policy auto-sets max_weight_staleness_versions=0 and + # over_sampling=False. over_sampling=false requires + # max_buffered_rollouts == grpo.num_prompts_per_step * (max_weight_staleness_versions + 1) + # 32 * (0 + 1) = 32 + max_buffered_rollouts: 32 + over_sampling: false + +policy: + dtensor_cfg: + enabled: false + megatron_cfg: + enabled: true + scheduler: + lr_warmup_iters: 50 + generation: + vllm_cfg: + async_engine: true + colocated: + enabled: false + resources: + # 4 GPUs for inference; remaining 4 GPUs on the node go to training. + gpus_per_node: 4 + num_nodes: 1 diff --git a/examples/configs/recipes/llm/performance/grpo-llama3.1-8b-instruct-2n8g.yaml b/examples/configs/recipes/llm/performance/grpo-llama3.1-8b-instruct-2n8g.yaml index d965796558b..6ffd52c96a3 100644 --- a/examples/configs/recipes/llm/performance/grpo-llama3.1-8b-instruct-2n8g.yaml +++ b/examples/configs/recipes/llm/performance/grpo-llama3.1-8b-instruct-2n8g.yaml @@ -11,6 +11,7 @@ policy: model_name: meta-llama/Llama-3.1-8B-Instruct tokenizer: name: meta-llama/Llama-3.1-8B-Instruct + train_global_batch_size: 2048 train_micro_batch_size: 1 logprob_batch_size: 2 max_total_sequence_length: 4096 diff --git a/examples/nemo_gym/run_distillation_nemo_gym.py b/examples/nemo_gym/run_distillation_nemo_gym.py index 7af1e4de6d7..1fdcab6c68f 100644 --- a/examples/nemo_gym/run_distillation_nemo_gym.py +++ b/examples/nemo_gym/run_distillation_nemo_gym.py @@ -32,9 +32,7 @@ from nemo_rl.algorithms.utils import get_tokenizer from nemo_rl.data.utils import setup_response_data from nemo_rl.distributed.virtual_cluster import init_ray -from nemo_rl.environments.nemo_gym import ( - setup_nemo_gym_config, -) +from nemo_rl.environments.nemo_gym import setup_nemo_gym_config from nemo_rl.models.generation import configure_generation_config from nemo_rl.utils.config import ( load_config, diff --git a/examples/nemo_gym/run_grpo_nemo_gym.py b/examples/nemo_gym/run_grpo_nemo_gym.py index 25a2c184934..2557348a342 100644 --- a/examples/nemo_gym/run_grpo_nemo_gym.py +++ b/examples/nemo_gym/run_grpo_nemo_gym.py @@ -40,9 +40,7 @@ from nemo_rl.algorithms.utils import get_tokenizer from nemo_rl.data.utils import setup_response_data from nemo_rl.distributed.virtual_cluster import init_ray -from nemo_rl.environments.nemo_gym import ( - setup_nemo_gym_config, -) +from nemo_rl.environments.nemo_gym import setup_nemo_gym_config from nemo_rl.experience.rollouts import run_async_nemo_gym_rollout from nemo_rl.models.generation import configure_generation_config from nemo_rl.utils.config import ( diff --git a/examples/run_grpo_single_controller.py b/examples/run_grpo_single_controller.py new file mode 100644 index 00000000000..bda2fe54999 --- /dev/null +++ b/examples/run_grpo_single_controller.py @@ -0,0 +1,136 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Async GRPO launcher driven by the SingleController actor. + +Builds the full SC bundle driver-side via setup_single_controller and hands it +to SingleControllerActor. Mirrors run_grpo.py for config loading so the same YAML +files apply. data_plane.enabled=true is mandatory. +""" + +import argparse +import os +import pprint +import sys + +import ray +from omegaconf import OmegaConf + +from nemo_rl.algorithms.single_controller import SingleControllerActor +from nemo_rl.algorithms.single_controller_utils import ( + MasterConfig, + setup_single_controller, +) +from nemo_rl.algorithms.utils import get_tokenizer +from nemo_rl.distributed.virtual_cluster import init_ray +from nemo_rl.environments.nemo_gym import setup_nemo_gym_config +from nemo_rl.models.generation import configure_generation_config +from nemo_rl.utils.config import ( + load_config, + parse_hydra_overrides, + register_omegaconf_resolvers, +) +from nemo_rl.utils.logger import get_next_experiment_dir + +# Drop examples/ from sys.path so examples/nemo_gym/ (no __init__.py) doesn't +# shadow the real nemo_gym package as a namespace package. +current_dir = os.path.dirname(os.path.abspath(__file__)) +while current_dir in sys.path: + sys.path.remove(current_dir) + + +def parse_args() -> tuple[argparse.Namespace, list[str]]: + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Run async GRPO training via SingleController" + ) + parser.add_argument( + "--config", type=str, default=None, help="Path to YAML config file" + ) + args, overrides = parser.parse_known_args() + return args, overrides + + +def main() -> None: + """Main entry point.""" + register_omegaconf_resolvers() + args, overrides = parse_args() + + if not args.config: + args.config = os.path.join( + os.path.dirname(__file__), + "configs", + "grpo_math_1B_single_controller.yaml", + ) + + config = load_config(args.config) + print(f"Loaded configuration from: {args.config}") + + if overrides: + print(f"Overrides: {overrides}") + config = parse_hydra_overrides(config, overrides) + + config = OmegaConf.to_container(config, resolve=True) + config = MasterConfig(**config) + print("Applied CLI overrides") + + dp_cfg = config.data_plane + if not dp_cfg.get("enabled", False): + raise ValueError( + "run_grpo_single_controller requires data_plane.enabled=true. " + "Use examples/run_grpo.py for the legacy / sync paths." + ) + + print("Final config:") + pprint.pprint(config) + + config.logger["log_dir"] = get_next_experiment_dir(config.logger["log_dir"]) + print(f"📊 Using log directory: {config.logger['log_dir']}") + if config.checkpointing["enabled"]: + print( + f"📊 Using checkpoint directory: {config.checkpointing['checkpoint_dir']}" + ) + + init_ray() + + tokenizer = get_tokenizer(config.policy["tokenizer"]) + assert config.policy["generation"] is not None, ( + "A generation config is required for SC-driven async GRPO" + ) + has_refit_draft_weights = bool(config.policy["draft"]["enabled"]) + config.policy["generation"] = configure_generation_config( + config.policy["generation"], + tokenizer, + has_refit_draft_weights=has_refit_draft_weights, + ) + + # NeMo-Gym specific config setup. + if bool(config.env.get("should_use_nemo_gym")): + setup_nemo_gym_config(config, tokenizer) + + bundle = setup_single_controller(config, tokenizer) + + print("🚀 Launching SingleControllerActor") + sc = SingleControllerActor.remote(master_config=config, bundle=bundle) + result = ray.get(sc.run.remote()) + print(f"SC run complete: {result}") + + # Drain env actors before vLLM shutdown to avoid race-condition 500s on + # in-flight requests. + for handle in bundle.env_handles.values(): + ray.get(handle.shutdown.remote()) + + +if __name__ == "__main__": + main() diff --git a/examples/swe_bench/README.md b/examples/swe_bench/README.md new file mode 100644 index 00000000000..3f30f5f4520 --- /dev/null +++ b/examples/swe_bench/README.md @@ -0,0 +1,85 @@ +# Async GRPO on SWE-bench (Qwen3-30B-A3B) + +Launcher and recipe for agentic RL on SWE-bench via NeMo-Gym/OpenHands: +16 nodes by default (8 training + 8 generation, non-colocated), async GRPO +with staleness window 1. + +| File | Purpose | +|---|---| +| `run_grpo_qwen3_30b_async_swe.sh` | SLURM launcher (submits `ray.sub`) | +| `grpo_qwen3_30b_async_swe.yaml` | Recipe config | + +The launcher supports two entrypoints: + +- `SC_MODE=1` (default): `examples/run_grpo_single_controller.py` — + single-controller with the TransferQueue data plane. +- `SC_MODE=0`: `examples/nemo_gym/run_grpo_nemo_gym.py` — classic async GRPO + (async behavior comes from the yaml's `grpo.async_grpo` block). + +## Prerequisites + +1. A NeMo-RL checkout (or code snapshot) containing `ray.sub` at its root — + the launcher submits from the root it lives under. +2. An enroot container image of a recent NeMo-RL nightly. +3. The SWE train dataset (`.jsonl`). +4. The SWE sandbox images (apptainer/singularity `.sif` files for the + swe-bench / sweap instances). **Edit the `container_formatter` lists in the + yaml** to point at your local copies (they ship as `/path/to/...` + placeholders). +5. A HF checkpoint to train from. + +## Quick start + +Run from the repo root: + +```bash +ACCOUNT= \ +CONTAINER=/path/to/nemo-rl-nightly.sqsh \ +MODEL_PATH=/path/to/hf_checkpoint \ +TRAIN_DATA_PATH=/path/to/swe_train.jsonl \ +EXTRA_MOUNTS=/path/to/shared_fs:/path/to/shared_fs \ +bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh +``` + +`EXTRA_MOUNTS` must make the model / data / `.sif` locations visible inside +the container (the default mounts only cover the repo tree and the gym +source). Add `DRY_RUN=1` to print the sbatch command and config without +submitting. + +Secrets (`WANDB_API_KEY`, `HUGGINGFACE_TOKEN`, ...) are read from the calling +environment and never stored in this directory. The recommended pattern is a +small personal wrapper script that exports your site-specific paths, cluster +account, and secrets, then calls this launcher. + +## Common variations + +```bash +# Classic (non-single-controller) entrypoint +SC_MODE=0 bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh + +# Different training tensor parallelism +TP=2 bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh + +# streaming-2 dispatch semantics (over-generation + out-of-order consumption; +# default is streaming-1: strict repro of classic async_grpo age-1 dispatch) +OVER_SAMPLING=true FORCE_IN_ORDER=false \ +bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh + +# Finer intra-step dispatch: start training once 2 prompt groups are ready +# instead of waiting for the full batch (SC only; PPS=8 by default) +MIN_PROMPT_GROUPS=2 bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh + +# Smaller / shorter run +NUM_NODES=8 NUM_GEN_NODES=4 TIME=2:0:0 MAX_NUM_STEPS=5 \ +bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh +``` + +The full knob list (parallelism, batch sizes, staleness, agent turn limits, +caches, ...) is documented in the launcher's header comment. + +## Monitoring + +The submit banner prints the experiment name, W&B target, and log location. +The driver log lands at `/logs/slurm/-logs/ray-driver.log`; +training progress appears there as `train step ...` / `step_metrics=...` +lines and in W&B under `train/reward` and `timing/train/*`. diff --git a/examples/swe_bench/grpo_qwen3_30b_async_swe.yaml b/examples/swe_bench/grpo_qwen3_30b_async_swe.yaml new file mode 100644 index 00000000000..aa774156245 --- /dev/null +++ b/examples/swe_bench/grpo_qwen3_30b_async_swe.yaml @@ -0,0 +1,436 @@ +# ============================================================================ +# Async GRPO SWE RL Training: Qwen3-30B-A3B-Thinking-2507 +# +# Model: Qwen3-30B-A3B-Thinking-2507 (MoE, 30B total / 3B active, thinking) +# Train data: R2E-Gym (r2e-gym subset, 4518 samples) +# Eval data: SWE-bench Verified +# Mode: Async GRPO with non-colocated generation +# Entry: examples/nemo_gym/run_grpo_nemo_gym.py +# Env: swe_agents (OpenHands agent, singularity sandbox) +# +# Based on: bihu/nemo-rl-qwen-swe/grpo_qwen3_30b_thinking_swe.yaml +# Gym: main branch (nemo-rl-async-swe repo) +# ============================================================================ + +checkpointing: + enabled: true + checkpoint_dir: "results/grpo-qwen3-30b-thinking-swe-rl" + metric_name: "train:total_reward/mean" + higher_is_better: true + keep_top_k: 100 + save_period: 5 + checkpoint_must_save_by: "00:03:35:00" + model_save_format: "safetensors" + save_consolidated: false + save_optimizer: true + +grpo: + num_prompts_per_step: 16 + num_generations_per_prompt: 16 + num_val_generations_per_prompt: 1 + max_rollout_turns: 1 + max_num_epochs: 100 + max_num_steps: 1000000 + normalize_rewards: true + use_leave_one_out_baseline: true + advantage_clip_low: -100 + advantage_clip_high: 100 + val_period: 10 + val_at_start: false + val_at_end: false + overlong_filtering: true + max_val_samples: null + val_batch_size: 256 + seed: 42 + invalid_tool_call_strategy: "" + + use_dynamic_sampling: false + dynamic_sampling_max_gen_batches: 10 + batch_multiplier: 1 + + penalize_invalid_tool_call: true + invalid_tool_call_advantage: -5.0 + penalize_malformed_thinking: true + malformed_thinking_advantage: -5.0 + + reward_shaping: + enabled: false + overlong_buffer_length: 128 + overlong_buffer_penalty: 1 + max_response_length: ${policy.max_total_sequence_length} + stop_properly_penalty_coef: null + reward_scaling: + enabled: false + source_min: 0.0 + source_max: 1.0 + target_min: 0.0 + target_max: 1.0 + + async_grpo: + enabled: true + max_trajectory_age_steps: 1 + in_flight_weight_updates: true + recompute_kv_cache_after_weight_updates: false + + seq_logprob_error_threshold: 2 + +loss_fn: + reference_policy_kl_penalty: 0.0 + reference_policy_kl_type: "k3" + kl_input_clamp_value: null + kl_output_clamp_value: null + ratio_clip_min: 0.2 + ratio_clip_max: 0.28 + ratio_clip_c: null + use_on_policy_kl_approximation: true + use_importance_sampling_correction: true + truncated_importance_sampling_ratio: 5.0 + truncated_importance_sampling_ratio_min: null + truncated_importance_sampling_type: tis + sequence_level_importance_ratios: false + token_level_loss: true + force_on_policy_ratio: true + use_kl_in_reward: false + +policy: + model_name: "Qwen/Qwen3-30B-A3B-Thinking-2507" + tokenizer: + name: ${policy.model_name} + chat_template_kwargs: + enable_thinking: true + hf_config_overrides: {} + train_global_batch_size: 256 + train_micro_batch_size: 1 + generation_batch_size: 64 + logprob_batch_size: 1 + max_total_sequence_length: 131072 + precision: "bfloat16" + logprob_chunk_size: 2048 + offload_optimizer_for_logprob: false + + dtensor_cfg: + _v2: true + enabled: false + cpu_offload: False + sequence_parallel: false + activation_checkpointing: false + tensor_parallel_size: 1 + context_parallel_size: 1 + custom_parallel_plan: null + + megatron_cfg: + enabled: true + gradient_accumulation_fusion: false + empty_unused_memory_level: 1 + activation_checkpointing: true + tensor_model_parallel_size: 2 + expert_tensor_parallel_size: 1 + expert_model_parallel_size: 8 + pipeline_model_parallel_size: 2 + num_layers_in_first_pipeline_stage: null + num_layers_in_last_pipeline_stage: null + context_parallel_size: 4 + pipeline_dtype: ${policy.precision} + sequence_parallel: true + freeze_moe_router: true + moe_router_dtype: "fp32" + moe_router_load_balancing_type: "none" + moe_router_bias_update_rate: 1.0e-3 + moe_permute_fusion: true + moe_enable_deepep: false + moe_token_dispatcher_type: "alltoall" + moe_aux_loss_coeff: 0.0 + moe_router_enable_expert_bias: true + moe_shared_expert_overlap: false + apply_rope_fusion: True + bias_activation_fusion: False + defer_fp32_logits: True + moe_per_layer_logging: True + + optimizer: + optimizer: "adam" + lr: 1.0e-6 + min_lr: 1.0e-6 + weight_decay: 0.0 + bf16: true + fp16: false + params_dtype: "float32" + adam_beta1: 0.9 + adam_beta2: 0.999 + adam_eps: 1e-8 + sgd_momentum: 0.9 + use_distributed_optimizer: true + use_precision_aware_optimizer: true + clip_grad: ${policy.max_grad_norm} + optimizer_cpu_offload: false + optimizer_offload_fraction: 0.0 + + scheduler: + start_weight_decay: ${policy.megatron_cfg.optimizer.weight_decay} + end_weight_decay: ${policy.megatron_cfg.optimizer.weight_decay} + weight_decay_incr_style: "constant" + lr_decay_style: "constant" + lr_decay_iters: 1000000 + lr_warmup_iters: 0 + lr_warmup_init: 0 + + distributed_data_parallel_config: + grad_reduce_in_fp32: false + overlap_grad_reduce: false + overlap_param_gather: false + use_custom_fsdp: false + data_parallel_sharding_strategy: "optim_grads_params" + + mtp_loss_scaling_factor: 0.0 + mtp_use_repeated_layer: false + mtp_num_layers: 0 + mtp_detach_heads: false + + fp8_cfg: + enabled: false + fp8: "e4m3" + fp8_recipe: "blockwise" + fp8_param: false + + env_vars: null + + dynamic_batching: + enabled: False + train_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.train_micro_batch_size}} + logprob_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.logprob_batch_size}} + sequence_length_round: 64 + + sequence_packing: + enabled: True + train_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.train_micro_batch_size}} + logprob_mb_tokens: ${mul:${policy.max_total_sequence_length}, ${policy.logprob_batch_size}} + algorithm: "modified_first_fit_decreasing" + sequence_length_round: 64 + + make_sequence_length_divisible_by: 8 + max_grad_norm: 1.0 + + optimizer: null + scheduler: null + + generation: + port_range_low: 11001 + port_range_high: 15000 + backend: "vllm" + max_new_tokens: ${policy.max_total_sequence_length} + temperature: 1.0 + top_p: 1.0 + top_k: null + stop_token_ids: null + stop_strings: null + vllm_cfg: + enable_prefix_caching: true + async_engine: true + precision: ${policy.precision} + kv_cache_dtype: "auto" + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + expert_parallel_size: 1 + gpu_memory_utilization: 0.8 + max_model_len: ${policy.max_total_sequence_length} + enforce_eager: False + enforce_monotonicity: false + use_deep_gemm: False + num_last_layers_in_bf16: 0 + num_first_layers_in_bf16: 0 + enable_vllm_metrics_logger: true + vllm_metrics_logger_interval: 0.5 + expose_http_server: true + skip_tokenizer_init: false + enable_thinking: true + http_server_serving_chat_kwargs: + enable_auto_tools: true + tool_parser: hermes + reasoning_parser: deepseek_r1 + chat_template: | + {%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} + {%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} + {%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endfor %} + {%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n\n' }} + {%- endif %} + default_chat_template_kwargs: + enable_thinking: true + truncate_history_thinking: false + + vllm_kwargs: + mamba_ssm_cache_dtype: "float32" + # The default flashinfer MoE backend (trtllm_bf16_moe) uses a fused/quantized + # expert weight layout that does NOT match the megatron weight-refit broadcast, + # producing garbage generations (and can hang broadcast_weights_for_collective). + # triton uses the standard expert layout the refit broadcast expects. + # Mirrors ruit's grpo_qwen3_30b_async_swe_hsg.yaml on the swe2-scale-gen branch. + moe_backend: triton + compilation_config: + cudagraph_capture_sizes: [1,2,4,8,16,32,64] + + colocated: + enabled: false + resources: + gpus_per_node: 8 + num_nodes: 4 + +data: + max_input_seq_length: null + shuffle: false + num_workers: 1 + use_multiple_dataloader: false + train: + # Site-specific; the launcher always overrides this via ++data.train.data_path. + data_path: "/path/to/swe_train_dataset.jsonl" + validation: + # Site-specific; the launcher always overrides this via ++data.validation.data_path. + data_path: "/path/to/swe_val_dataset.jsonl" + default: + dataset_name: NemoGymDataset + env_name: "nemo_gym" + prompt_file: null + system_prompt_file: null + processor: "nemo_gym_data_processor" + +env: + should_use_nemo_gym: true + should_log_nemo_gym_responses: false + nemo_gym: + skip_venv_if_present: true + port_range_low: 15001 + port_range_high: 20000 + config_paths: + - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml + - responses_api_agents/swe_agents/configs/swebench_openhands_training.yaml + swe_agents_train: + responses_api_agents: + swe_agents: + agent_max_turns: 100 + concurrency: 768 + swebench_agent_timeout: 3600 + run_with_mixed_prompts: true + dataset_path: ${data.train.data_path} + # Site-specific: apptainer/singularity .sif sandbox images for the SWE + # datasets, tried in order per instance. Point these at your local + # copies of the swe-bench / sweap image sets. + container_formatter: + - "/path/to/swe-bench-images/swebench_sweb.eval.x86_64.{instance_id}.sif" + - "/path/to/sweap-images/sweap.{instance_id}.sif" + - "/path/to/swe-bench-images/namanjain12_{instance_id}.sif" + - "/path/to/sweap-images/sweap.{instance_id}.sif" + - "/path/to/swe-bench-images/swebench_sweb.eval.x86_64{instance_id}.sif" + swe_agents_val: + responses_api_agents: + swe_agents: + agent_max_turns: 200 + concurrency: 768 + swebench_agent_timeout: 3600 + dataset_path: ${data.validation.data_path} + # Site-specific: see the note on swe_agents_train.container_formatter. + container_formatter: + - "/path/to/swe-bench-images/swebench_sweb.eval.x86_64.{instance_id}.sif" + - "/path/to/sweap-images/sweap.{instance_id}.sif" + - "/path/to/swe-bench-images/namanjain12_{instance_id}.sif" + - "/path/to/sweap-images/sweap.{instance_id}.sif" + - "/path/to/swe-bench-images/swebench_sweb.eval.x86_64{instance_id}.sif" + use_absolute_ip: true + +logger: + log_dir: "logs" + num_val_samples_to_print: 0 + wandb_enabled: true + tensorboard_enabled: false + mlflow_enabled: false + monitor_gpus: true + swanlab_enabled: false + wandb: + project: "ruit-nemo-rl" + name: "qwen3-30b-thinking-swe-rl" + tensorboard: {} + mlflow: + experiment_name: "qwen3-30b-thinking-swe-rl" + run_name: "qwen3-30b-thinking-swe-rl" + gpu_monitoring: + collection_interval: 10 + flush_interval: 10 + +cluster: + gpus_per_node: 8 + num_nodes: 16 diff --git a/examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh b/examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh new file mode 100644 index 00000000000..182274a3c5a --- /dev/null +++ b/examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh @@ -0,0 +1,560 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +# Async GRPO on SWE-bench (NeMo-Gym / OpenHands) — Qwen3-30B-A3B, SLURM launcher. +# +# Submits a 16-node (8 train + 8 generation, non-colocated) async GRPO run via +# ray.sub. Supports both entrypoints: +# SC_MODE=1 (default) examples/run_grpo_single_controller.py +# (single-controller + TransferQueue data plane) +# SC_MODE=0 examples/nemo_gym/run_grpo_nemo_gym.py +# (classic async GRPO; async comes from the yaml's +# grpo.async_grpo block) +# +# This file is environment-agnostic: no secrets and no user-specific paths. +# Site-specific values come from the environment (see REQUIRED below) — wrap +# this script with your own launcher that exports them (see +# test_assets/SWE/grpo_swe_tests.sh for an example wrapper). +# +# --------------------------------------------------------------------------- +# REQUIRED environment: +# ACCOUNT SLURM account +# CONTAINER enroot .sqsh/.squashfs image (a recent NeMo-RL nightly; +# must postdate the TransferQueue pyproject dependency) +# MODEL_PATH HF checkpoint dir to train from +# TRAIN_DATA_PATH SWE train jsonl +# +# Common optional environment (see defaults inline for the full list): +# PARTITION (batch), NUM_NODES (16), NUM_GEN_NODES (8), TIME (4:0:0) +# VAL_DATA_PATH (=TRAIN_DATA_PATH), CONFIG_FILE (yaml next to this script) +# TP/EP/CP/PP/VLLM_TP, SEQLEN, PPS/GPP/GBS, LR, MAX_NUM_STEPS +# SC_MODE (1), MIN_PROMPT_GROUPS (=PPS) +# OVER_SAMPLING (false), FORCE_IN_ORDER (true) +# streaming-1 (default): OVER_SAMPLING=false FORCE_IN_ORDER=true +# — no over-generation, each step consumes the groups dispatched +# for it; 1:1 repro of the classic async_grpo (age=1) dispatch. +# streaming-2: OVER_SAMPLING=true FORCE_IN_ORDER=false +# — generation keeps producing, steps consume any groups within +# the staleness window; stale groups get evicted (wasted). +# PERSISTENT_CACHE compile/uv cache root on shared fs (~/.cache/... default) +# GYM_VENV_DIR /tmp/nemo_gym_venvs default; /opt/gym_venvs if your image +# has the gym server venvs baked in +# MOUNTS / EXTRA_MOUNTS container mounts (make model/data paths visible!) +# WANDB_API_KEY + WANDB_PROJ + EXP_SUFFIX logging & naming +# HUGGINGFACE_TOKEN / GITHUB_TOKEN / GITLAB_TOKEN passed through if set +# REAPER_COMMENT SLURM --comment payload (cluster-specific; empty default) +# DRY_RUN=1 print everything, submit nothing +# +# Usage (from a repo/snapshot root that contains ray.sub): +# ACCOUNT=... CONTAINER=... MODEL_PATH=... TRAIN_DATA_PATH=... \ +# bash examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh +# ============================================================================= + +set -e + +# --------------------------------------------------------------------------- +# Locate the tree to submit from: this script lives at examples/swe_bench/ +# inside the repo (or a code snapshot); ray.sub must exist at its root. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUN_DIR="${RUN_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +if [ ! -f "${RUN_DIR}/ray.sub" ]; then + echo "Error: ${RUN_DIR}/ray.sub not found — RUN_DIR must be a NeMo-RL checkout/snapshot root." >&2 + exit 1 +fi +RUN_COMMIT="unknown" +if git -C "${RUN_DIR}" rev-parse --short HEAD >/dev/null 2>&1; then + RUN_COMMIT="$(git -C "${RUN_DIR}" rev-parse --short HEAD)" +elif [ -f "${RUN_DIR}/commit.txt" ]; then + RUN_COMMIT="$(cut -c1-7 "${RUN_DIR}/commit.txt")" +fi + +# --------------------------------------------------------------------------- +# Required site-specific inputs — fail fast with a clear message. +# --------------------------------------------------------------------------- +missing="" +[ -n "${ACCOUNT:-}" ] || missing+=" ACCOUNT" +[ -n "${CONTAINER:-}" ] || missing+=" CONTAINER" +[ -n "${MODEL_PATH:-}" ] || missing+=" MODEL_PATH" +[ -n "${TRAIN_DATA_PATH:-}" ] || missing+=" TRAIN_DATA_PATH" +if [ -n "${missing}" ]; then + echo "Error: missing required environment variables:${missing}" >&2 + echo "See the header of this script for the full contract." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Cluster / submission (site-specific, all overridable) +# --------------------------------------------------------------------------- +PARTITION="${PARTITION:-batch}" +GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +CPUS_PER_WORKER="${CPUS_PER_WORKER:-114}" +# SLURM --comment payload (e.g. idle-GPU-reaper exemptions on some clusters). +REAPER_COMMENT="${REAPER_COMMENT:-}" + +# --------------------------------------------------------------------------- +# Scale / walltime +# --------------------------------------------------------------------------- +NUM_NODES="${NUM_NODES:-16}" # total allocation +NUM_GEN_NODES="${NUM_GEN_NODES:-8}" # carved out of NUM_NODES for generation +TIME="${TIME:-4:0:0}" +MAX_NUM_STEPS="${MAX_NUM_STEPS:-}" # empty => use the yaml's value + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- +CONFIG_FILE="${CONFIG_FILE:-${SCRIPT_DIR}/grpo_qwen3_30b_async_swe.yaml}" +VAL_DATA_PATH="${VAL_DATA_PATH:-${TRAIN_DATA_PATH}}" + +# ============================ Parallelism ============================ +TP="${TP:-4}"; EP="${EP:-8}"; CP="${CP:-4}"; PP="${PP:-2}"; VLLM_TP="${VLLM_TP:-2}" +MIN_PAD=1 +[ "${CP}" -gt 1 ] && MIN_PAD=$((MIN_PAD * CP * 2)) +[ "${TP}" -gt 1 ] && MIN_PAD=$((MIN_PAD * TP)) +MAKE_SEQ_DIVISIBLE_BY="${MIN_PAD}" +SEQUENCE_PACKING=True + +# ===================== Sequence length ===================== +SEQLEN="${SEQLEN:-131072}" + +# ===== Single-controller async-RL knobs (SC_MODE=1: data_plane + async_rl) ===== +# Maps the classic async_grpo (age=1) 1:1 to async_rl. +MAX_TRAJECTORY_AGE_STEPS="${MAX_TRAJECTORY_AGE_STEPS:-1}" # -> async_rl.max_weight_staleness_versions +BATCH_SELECTION_STRATEGY=staleness_window +# Dispatch semantics ("streaming" modes). Defaults = strict 1:1 repro of the +# classic async_grpo age-1 behavior (streaming-1): no over-generation, rollouts +# consumed strictly by their dispatch-target step. streaming-2 = the code +# defaults (over_sampling=true force_in_order=false): generation keeps +# producing (stale groups get evicted/wasted) and steps consume any groups +# inside the staleness window, out of order. +OVER_SAMPLING="${OVER_SAMPLING:-false}" +FORCE_IN_ORDER="${FORCE_IN_ORDER:-true}" +FORCE_ON_POLICY_RATIO=True +SEQ_LOGPROB_ERROR_THRESHOLD=null +COLOCATED_ENABLED=False +VLLM_GPU_UTIL=0.8 +OVERLAP_GRAD_REDUCE=False +ADVANTAGE_CLIP_LOW=-100 +ADVANTAGE_CLIP_HIGH=100 +TIS_THRESHOLD=5 + +# ========================= GRPO / sampling ========================= +PPS="${PPS:-8}"; GPP="${GPP:-8}"; GBS="${GBS:-64}" +# Intra-step dispatch granularity (SC only): the sampler releases work to the +# trainer once this many complete prompt groups are ready (gradient +# accumulation; the optimizer still steps only after PPS groups). Not passed in +# => same as PPS = fully synchronous within the step; smaller values overlap +# trainer compute with the generation tail. +MIN_PROMPT_GROUPS="${MIN_PROMPT_GROUPS:-${PPS}}" +NORMALIZE_REWARDS=True +OVERLONG_FILTERING=True +VAL_PERIOD="${VAL_PERIOD:-1000}" + +# ========================== Loss function ========================== +KL=0 +CLIP_MIN=0.2 +CLIP_MAX=0.28 +USE_ON_POLICY_KL_APPROXIMATION=True +IMPORTANCE_SAMPLING_CORRECTION=True +SEQ_LEVEL_IS=False +TOKEN_LEVEL_LOSS=True + +# ============================ Optimizer ============================ +LR="${LR:-1e-06}" + +# =============================== MoE =============================== +MOE_FREEZE_ROUTER=True +MOE_PERMUTE_FUSION=True +MOE_ENABLE_DEEPEP=False +MOE_TOKEN_DISPATCHER_TYPE="alltoall" +MOE_AUX_LOSS_COEFF=0 +MOE_ROUTER_LOAD_BALANCING_TYPE="none" +MOE_ROUTER_BIAS_UPDATE_RATE="1e-3" + +# ======================= Generation / vLLM ======================= +TEMPERATURE=1.0 + +# =================== Checkpointing & validation =================== +SAVE_PERIOD="${SAVE_PERIOD:-5}" +KEEP_TOP_K="${KEEP_TOP_K:-2}" +MUST_SAVE_BY="${MUST_SAVE_BY:-00:03:35:00}" # graceful save+exit before TIME + +# ============================ SWE agent ============================ +AGENT_MAX_TURNS="${AGENT_MAX_TURNS:-200}" +AGENT_TIMEOUT="${AGENT_TIMEOUT:-1800}" + +# ============================== Logging ============================== +WANDB_PROJ="${WANDB_PROJ:-nemo-rl-swe-bench}" +LOG_GYM_RESPONSES=true + +# ========================= Experiment naming ========================= +SYNC_MODE="async-age${MAX_TRAJECTORY_AGE_STEPS}" +EXP_SUFFIX="${EXP_SUFFIX:-swe-sc@${RUN_COMMIT}-${SYNC_MODE}-pps${PPS}-gpp${GPP}-gbs${GBS}-lr${LR}-tp${TP}}" +WANDB_NAME="${EXP_SUFFIX}" +EXP_NAME="${EXP_SUFFIX}" +CHECKPOINT_ROOT="${CHECKPOINT_ROOT:-${RUN_DIR}/results}" +CHECKPOINT_DIR="${CHECKPOINT_DIR:-${CHECKPOINT_ROOT}/${EXP_SUFFIX}}" +BASE_LOG_DIR="${BASE_LOG_DIR:-${RUN_DIR}/logs/slurm}" + +# ========================= Runtime env ========================= +# Secrets are passed through from the caller's environment; never set here. +export HUGGINGFACE_TOKEN="${HUGGINGFACE_TOKEN:-${HF_TOKEN:-}}" +export GITHUB_TOKEN="${GITHUB_TOKEN:-}" +export GITLAB_TOKEN="${GITLAB_TOKEN:-}" +export WANDB_API_KEY="${WANDB_API_KEY:-}" +export HF_HOME="${HF_HOME:-${HOME}/.cache/huggingface}" +export HF_DATASETS_CACHE="${HF_DATASETS_CACHE:-${HF_HOME}/datasets}" +# Node-local uv cache (ephemeral). Each node builds its own venv independently — +# a shared-filesystem uv cache corrupts under concurrent multi-node builds. +# Pre-warm LUSTRE_UV_CACHE once with a single process to skip long compiles. +export UV_CACHE_DIR=/tmp/uv_cache +export UV_LOCK_TIMEOUT=3600 +export RAY_DEDUP_LOGS=1 +export SSL_CERT_FILE="${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}" +export REQUESTS_CA_BUNDLE="${REQUESTS_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" +export CURL_CA_BUNDLE="${CURL_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-16}" + +# ===================== Shared-fs compile caches ===================== +# Runtime caches stay node-local (/tmp); the shared-fs copies are only seeded +# from (read) at startup and written back by a periodic sidecar, so repeat runs +# skip the ~20min triton MoE JIT. +PERSISTENT_CACHE="${PERSISTENT_CACHE:-${HOME}/.cache/nemo_rl_swe_bench}" +export LUSTRE_VLLM_CACHE="${PERSISTENT_CACHE}/vllm_compile_cache" +export LUSTRE_INDUCTOR_CACHE="${PERSISTENT_CACHE}/inductor_cache" +export LUSTRE_TRITON_CACHE="${PERSISTENT_CACHE}/triton_cache" +export LUSTRE_UV_CACHE="${PERSISTENT_CACHE}/uv_cache" +export NRL_VLLM_LOCAL_CACHE_DIR="/tmp/nemo_rl_vllm_cache" +export NRL_VLLM_CACHE_SEED_DIR="/tmp/nemo_rl_vllm_cache_warm" +export INDUCTOR_CACHE_DIR="/tmp/nemo_rl_inductor_cache" +export TRITON_CACHE_DIR="/tmp/nemo_rl_triton_cache" +export CACHE_SYNC_FREQUENCY="${CACHE_SYNC_FREQUENCY:-120}" +mkdir -p "${LUSTRE_VLLM_CACHE}" "${LUSTRE_INDUCTOR_CACHE}" "${LUSTRE_TRITON_CACHE}" "${LUSTRE_UV_CACHE}" + +# ===================== NeMo-Gym server venvs ===================== +# Default: node-local build (safe everywhere). If your container has the gym +# server venvs baked in (vllm_model + swe_agents), export GYM_VENV_DIR=/opt/gym_venvs. +# Do NOT point this at a shared filesystem: the editable nemo-gym build hangs on +# uv's flock there, and interrupted builds leave empty venv shells that +# skip-if-present then reuses -> the gym policy_model server crashes on import. +GYM_VENV_DIR="${GYM_VENV_DIR:-/tmp/nemo_gym_venvs}" +case "${GYM_VENV_DIR}" in /opt/*|/tmp/*) ;; *) mkdir -p "${GYM_VENV_DIR}" ;; esac + +# ===== SETUP_COMMAND: install apptainer + seed caches + uv sync ===== +# Runs on all nodes before Ray starts (consumed by ray.sub). +read -r -d '' SETUP_COMMAND </dev/null || true +RET=1 +RETRIES=3 +for attempt in \$(seq 1 \$RETRIES); do + if command -v apptainer >/dev/null 2>&1 || command -v singularity >/dev/null 2>&1; then + echo "[SETUP] singularity/apptainer already available" + RET=0 + break + fi + cd /tmp && \ + wget --no-check-certificate -q https://github.com/apptainer/apptainer/releases/download/v1.3.1/apptainer_1.3.1_amd64.deb && \ + apt install -y ./apptainer_1.3.1_amd64.deb && \ + ln -sf /usr/bin/apptainer /usr/bin/singularity + if command -v apptainer >/dev/null 2>&1; then + echo "[SETUP] apptainer installed successfully" + RET=0 + break + fi + echo "[SETUP] apptainer install attempt \$attempt failed, retrying..." + sleep 10 +done +if [ \$RET -ne 0 ]; then + echo "[SETUP] WARNING: apptainer installation failed after \$RETRIES attempts" +fi + +echo "[CACHE SEED] Clearing stale /tmp caches and seeding from shared fs..." +rm -rf /tmp/nemo_rl_vllm_cache /tmp/nemo_rl_vllm_cache_* +rm -rf "${INDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" +mkdir -p "${INDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" + +find "${LUSTRE_INDUCTOR_CACHE}" -maxdepth 1 -name '.tmp_*' -mmin +30 -exec rm -rf {} + 2>/dev/null || true +find "${LUSTRE_TRITON_CACHE}" -maxdepth 1 -name '.tmp_*' -mmin +30 -exec rm -rf {} + 2>/dev/null || true + +_seed_cache() { + local lustre="\$1" local_dir="\$2" name="\$3" + if [ -d "\$lustre" ] && [ "\$(ls -A "\$lustre" 2>/dev/null)" ]; then + rsync -a --exclude '.tmp_*' "\$lustre/" "\$local_dir/" 2>/dev/null \ + && echo "[CACHE SEED] \$name: seeded from shared fs" \ + || echo "[CACHE SEED] \$name: seed failed (non-fatal)" + else + echo "[CACHE SEED] \$name: no warm cache on shared fs yet" + fi +} + +_seed_cache "${LUSTRE_INDUCTOR_CACHE}" "${INDUCTOR_CACHE_DIR}" "Inductor" +_seed_cache "${LUSTRE_TRITON_CACHE}" "${TRITON_CACHE_DIR}" "Triton" +# uv cache: read-only rsync of single-process-prebuilt wheels into node-local +# /tmp so the uv sync below is a cache hit, not a ~28min build. +# (NOTE: no backticks/'\$(...)' in this heredoc body -- they would be command- +# substituted on the LOGIN node when SETUP_COMMAND is read.) +mkdir -p "${UV_CACHE_DIR}" +_seed_cache "${LUSTRE_UV_CACHE}" "${UV_CACHE_DIR}" "uv (prebuilt wheels)" +echo "[CACHE SEED] Done." + +# ===== Compile-cache WRITE-BACK sidecar ===== +# The seed above only READS shared fs -> /tmp. This sidecar periodically rsyncs +# /tmp -> shared fs (and on TERM/INT) so compiled kernels persist across runs; +# --ignore-existing makes concurrent per-node writes first-writer-wins. +_sync_cache_one() { + local src="\$1" dst="\$2" name="\$3" + mkdir -p "\$dst" + if [ -d "\$src" ] && [ "\$(ls -A "\$src" 2>/dev/null)" ]; then + rsync -a --ignore-existing --exclude '.tmp_*' --exclude 'tmp*' "\$src/" "\$dst/" 2>/dev/null \ + && echo "[CACHE SYNC] \$name: /tmp -> shared fs" \ + || echo "[CACHE SYNC] \$name: sync failed (non-fatal)" + fi +} +_sync_compile_caches_to_lustre() { + _sync_cache_one "${INDUCTOR_CACHE_DIR}" "${LUSTRE_INDUCTOR_CACHE}" "Inductor" + _sync_cache_one "${TRITON_CACHE_DIR}" "${LUSTRE_TRITON_CACHE}" "Triton" +} +_start_cache_sync_sidecar() { + local pidfile="/tmp/nemo_rl_compile_cache_sync.pid" + if [ -f "\$pidfile" ] && kill -0 "\$(cat "\$pidfile" 2>/dev/null)" 2>/dev/null; then + echo "[CACHE SYNC] sidecar already running (pid=\$(cat "\$pidfile" 2>/dev/null))" + return + fi + ( + set +e + trap '_sync_compile_caches_to_lustre; exit 0' TERM INT + echo "[CACHE SYNC] sidecar started, frequency=${CACHE_SYNC_FREQUENCY}s" + while true; do + sleep "${CACHE_SYNC_FREQUENCY}" + _sync_compile_caches_to_lustre + done + ) > /tmp/nemo_rl_compile_cache_sync.log 2>&1 & + echo "\$!" > "\$pidfile" + echo "[CACHE SYNC] sidecar pid=\$!" +} +if [ "${CACHE_SYNC_FREQUENCY}" -gt 0 ] 2>/dev/null; then + _start_cache_sync_sidecar +else + echo "[CACHE SYNC] disabled (CACHE_SYNC_FREQUENCY=${CACHE_SYNC_FREQUENCY})" +fi + +UV_HTTP_TIMEOUT=3600 \ + uv sync --frozen --extra mcore +SETUPEOF +export SETUP_COMMAND + +# Optional extra grpo overrides (only emitted when set, so empty == use yaml). +EXTRA_GRPO="" +[ -n "${MAX_NUM_STEPS}" ] && EXTRA_GRPO="grpo.max_num_steps=${MAX_NUM_STEPS}" + +# ===== Entrypoint switch: SC (single-controller) vs classic async GRPO ===== +# SC_MODE=1 (default): examples/run_grpo_single_controller.py + data_plane/async_rl overrides. +# SC_MODE=0: examples/nemo_gym/run_grpo_nemo_gym.py — the gym-dedicated classic +# entry; async comes from the yaml's native grpo.async_grpo block. +# (The generic examples/run_grpo.py is NOT wired for nemo-gym: it +# discards the gym actor instead of binding task_to_env["nemo_gym"], +# and its configure_generation_config eos injection trips the gym +# rollout stop-criteria assert.) +SC_MODE="${SC_MODE:-1}" +if [ "${SC_MODE}" = "1" ]; then + ENTRYPOINT="./examples/run_grpo_single_controller.py" + SC_OVERRIDES="++data_plane.enabled=true \ + ++data_plane.impl=transfer_queue \ + ++data_plane.backend=simple \ + ++data_plane.storage_capacity=1000000 \ + ++data_plane.num_storage_units=2 \ + ++data_plane.claim_meta_poll_interval_s=0.5 \ + ++data_plane.global_segment_size=549755813888 \ + ++data_plane.local_buffer_size=68719476736 \ + ++async_rl.max_weight_staleness_versions=${MAX_TRAJECTORY_AGE_STEPS} \ + ++async_rl.min_prompt_groups_per_batch=${MIN_PROMPT_GROUPS} \ + ++async_rl.max_inflight_prompts=${PPS} \ + ++async_rl.max_buffered_rollouts=$((PPS * (MAX_TRAJECTORY_AGE_STEPS + 1))) \ + ++async_rl.batch_selection_strategy=${BATCH_SELECTION_STRATEGY} \ + ++async_rl.over_sampling=${OVER_SAMPLING} \ + ++async_rl.force_in_order=${FORCE_IN_ORDER}" +else + ENTRYPOINT="./examples/nemo_gym/run_grpo_nemo_gym.py" + SC_OVERRIDES="" +fi + +# ===== Training command ===== +export COMMAND="NRL_VLLM_USE_V1=1 \ + NRL_WG_USE_RAY_REF=1 \ + WANDB_API_KEY=${WANDB_API_KEY} \ + HUGGINGFACE_TOKEN=${HUGGINGFACE_TOKEN} \ + GITHUB_TOKEN=${GITHUB_TOKEN} \ + GITLAB_TOKEN=${GITLAB_TOKEN} \ + HF_HOME=${HF_HOME} \ + HF_DATASETS_CACHE=${HF_DATASETS_CACHE} \ + UV_CACHE_DIR=${UV_CACHE_DIR} \ + VLLM_ATTENTION_BACKEND=FLASH_ATTN \ + VLLM_CACHE_ROOT=${LUSTRE_VLLM_CACHE} \ + DG_JIT_CACHE_DIR=${LUSTRE_VLLM_CACHE}/deep_gemm \ + VLLM_DEEP_GEMM_WARMUP=skip \ + NRL_FORCE_REBUILD_VENVS=${NRL_FORCE_REBUILD_VENVS:-false} \ + NG_HTTP_BYTES_DIR=${NG_HTTP_BYTES_DIR:-} \ + NRL_HTTP_BYTES_DIR=${NRL_HTTP_BYTES_DIR:-} \ + NRL_SKIP_TQ_RUNTIME_ENV_PATCH=${NRL_SKIP_TQ_RUNTIME_ENV_PATCH:-1} \ + NRL_IGNORE_VERSION_MISMATCH=1 \ + RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 \ + UV_HTTP_TIMEOUT=3600 \ + UV_LOCK_TIMEOUT=900 \ + TORCH_CUDA_ARCH_LIST='9.0 10.0' \ + NEMO_GYM_SKIP_VENV_IF_PRESENT=1 \ + NEMO_GYM_VENV_DIR=${GYM_VENV_DIR} \ + uv run --frozen --extra mcore ${ENTRYPOINT} \ + --config=${CONFIG_FILE} \ + cluster.num_nodes=${NUM_NODES} \ + cluster.gpus_per_node=${GPUS_PER_NODE} \ + ++data.train.data_path=${TRAIN_DATA_PATH} \ + ++data.validation.data_path=${VAL_DATA_PATH} \ + grpo.num_prompts_per_step=${PPS} \ + grpo.num_generations_per_prompt=${GPP} \ + grpo.val_at_start=False \ + grpo.normalize_rewards=${NORMALIZE_REWARDS} \ + grpo.overlong_filtering=${OVERLONG_FILTERING} \ + grpo.val_period=${VAL_PERIOD} \ + grpo.seq_logprob_error_threshold=${SEQ_LOGPROB_ERROR_THRESHOLD} \ + ${EXTRA_GRPO} \ + ${SC_OVERRIDES} \ + ++policy.draft.enabled=false \ + ++policy.draft.model_name=null \ + ++policy.draft.loss_weight=0.1 \ + ++policy.draft.num_layers=null \ + ++policy.draft.aux_layer_indices=null \ + env.should_log_nemo_gym_responses=${LOG_GYM_RESPONSES} \ + policy.generation.colocated.enabled=${COLOCATED_ENABLED} \ + policy.model_name=${MODEL_PATH} \ + policy.max_total_sequence_length=${SEQLEN} \ + policy.dynamic_batching.enabled=False \ + policy.train_global_batch_size=${GBS} \ + policy.make_sequence_length_divisible_by=${MAKE_SEQ_DIVISIBLE_BY} \ + policy.offload_optimizer_for_logprob=true \ + policy.sequence_packing.enabled=${SEQUENCE_PACKING} \ + policy.megatron_cfg.tensor_model_parallel_size=${TP} \ + policy.megatron_cfg.expert_model_parallel_size=${EP} \ + policy.megatron_cfg.context_parallel_size=${CP} \ + policy.megatron_cfg.pipeline_model_parallel_size=${PP} \ + policy.megatron_cfg.sequence_parallel=True \ + policy.megatron_cfg.bias_activation_fusion=False \ + ++policy.megatron_cfg.use_fused_weighted_squared_relu=false \ + policy.megatron_cfg.distributed_data_parallel_config.overlap_grad_reduce=${OVERLAP_GRAD_REDUCE} \ + policy.megatron_cfg.moe_permute_fusion=${MOE_PERMUTE_FUSION} \ + policy.megatron_cfg.moe_enable_deepep=${MOE_ENABLE_DEEPEP} \ + policy.megatron_cfg.moe_token_dispatcher_type=${MOE_TOKEN_DISPATCHER_TYPE} \ + policy.megatron_cfg.moe_aux_loss_coeff=${MOE_AUX_LOSS_COEFF} \ + policy.megatron_cfg.moe_router_load_balancing_type=${MOE_ROUTER_LOAD_BALANCING_TYPE} \ + policy.megatron_cfg.moe_router_bias_update_rate=${MOE_ROUTER_BIAS_UPDATE_RATE} \ + policy.megatron_cfg.freeze_moe_router=${MOE_FREEZE_ROUTER} \ + policy.megatron_cfg.optimizer.lr=${LR} \ + policy.megatron_cfg.optimizer.min_lr=${LR} \ + policy.megatron_cfg.optimizer.weight_decay=0 \ + policy.megatron_cfg.empty_unused_memory_level=2 \ + policy.megatron_cfg.activation_checkpointing=True \ + policy.generation.temperature=${TEMPERATURE} \ + policy.generation.vllm_cfg.tensor_parallel_size=${VLLM_TP} \ + policy.generation.vllm_cfg.gpu_memory_utilization=${VLLM_GPU_UTIL} \ + policy.generation.vllm_cfg.skip_tokenizer_init=False \ + loss_fn.reference_policy_kl_penalty=${KL} \ + loss_fn.ratio_clip_min=${CLIP_MIN} \ + loss_fn.ratio_clip_max=${CLIP_MAX} \ + loss_fn.use_on_policy_kl_approximation=${USE_ON_POLICY_KL_APPROXIMATION} \ + loss_fn.use_importance_sampling_correction=${IMPORTANCE_SAMPLING_CORRECTION} \ + loss_fn.sequence_level_importance_ratios=${SEQ_LEVEL_IS} \ + loss_fn.token_level_loss=${TOKEN_LEVEL_LOSS} \ + loss_fn.force_on_policy_ratio=${FORCE_ON_POLICY_RATIO} \ + checkpointing.checkpoint_dir=${CHECKPOINT_DIR} \ + checkpointing.save_period=${SAVE_PERIOD} \ + checkpointing.keep_top_k=${KEEP_TOP_K} \ + ++checkpointing.metric_name=train:total_reward/mean \ + ++checkpointing.checkpoint_must_save_by=${MUST_SAVE_BY} \ + logger.wandb_enabled=True \ + logger.wandb.name=${WANDB_NAME} \ + logger.wandb.project=${WANDB_PROJ}" + +# Async non-colocated: generation cluster + clipping + agent turn/timeout knobs. +export COMMAND="${COMMAND} \ + policy.generation.colocated.resources.num_nodes=${NUM_GEN_NODES} \ + policy.generation.colocated.resources.gpus_per_node=${GPUS_PER_NODE} \ + grpo.advantage_clip_low=${ADVANTAGE_CLIP_LOW} \ + grpo.advantage_clip_high=${ADVANTAGE_CLIP_HIGH} \ + loss_fn.truncated_importance_sampling_ratio=${TIS_THRESHOLD} \ + env.nemo_gym.swe_agents_train.responses_api_agents.swe_agents.agent_max_turns=${AGENT_MAX_TURNS} \ + env.nemo_gym.swe_agents_train.responses_api_agents.swe_agents.swebench_agent_timeout=${AGENT_TIMEOUT} \ + env.nemo_gym.swe_agents_val.responses_api_agents.swe_agents.agent_max_turns=${AGENT_MAX_TURNS} \ + env.nemo_gym.swe_agents_val.responses_api_agents.swe_agents.swebench_agent_timeout=${AGENT_TIMEOUT}" + +# --------------------------------------------------------------------------- +# Mounts: the run tree at its own path (so ./examples/... resolves) and the +# gym source over the container's bundled copy. Export MOUNTS to replace, or +# EXTRA_MOUNTS to append (e.g. the filesystem holding MODEL_PATH/data). +# --------------------------------------------------------------------------- +GYM_CODE="${RUN_DIR}/3rdparty/Gym-workspace/Gym" +MOUNTS="${MOUNTS:-${RUN_DIR}:${RUN_DIR},${GYM_CODE}:/opt/nemo-rl/3rdparty/Gym-workspace/Gym}" +[ -n "${EXTRA_MOUNTS:-}" ] && MOUNTS="${MOUNTS},${EXTRA_MOUNTS}" + +mkdir -p "${CHECKPOINT_DIR}" "${BASE_LOG_DIR}" + +# ray.sub reads these from the environment. +export CONTAINER MOUNTS COMMAND SETUP_COMMAND GPUS_PER_NODE CPUS_PER_WORKER BASE_LOG_DIR +[ -n "${UV_CACHE_DIR_OVERRIDE:-}" ] && export UV_CACHE_DIR_OVERRIDE + +sbatch_args=( + --nodes="${NUM_NODES}" + --account="${ACCOUNT}" + --job-name="${EXP_NAME}" + --partition="${PARTITION}" + --time="${TIME}" + --gres=gpu:"${GPUS_PER_NODE}" + --exclusive + --dependency=singleton + --output="${BASE_LOG_DIR}/slurm-%j.out" +) +[ -n "${REAPER_COMMENT}" ] && sbatch_args+=(--comment="${REAPER_COMMENT}") +# shellcheck disable=SC2206 +[ -n "${SBATCH_EXTRA_ARGS:-}" ] && sbatch_args+=(${SBATCH_EXTRA_ARGS}) + +echo "==========================================" +echo "SWE async GRPO | Experiment: ${EXP_SUFFIX}" +echo "Entrypoint: ${ENTRYPOINT} (SC_MODE=${SC_MODE})" +echo "Run tree: ${RUN_DIR} @ ${RUN_COMMIT}" +echo "Account: ${ACCOUNT} / ${PARTITION}" +echo "Nodes: ${NUM_NODES} total (generation carves out ${NUM_GEN_NODES}) Time: ${TIME}" +echo "Container: ${CONTAINER}" +echo "Parallelism: TP=${TP}, EP=${EP}, CP=${CP}, PP=${PP}, vLLM_TP=${VLLM_TP}, pad=${MAKE_SEQ_DIVISIBLE_BY}" +echo "Training: PPS=${PPS}, GPP=${GPP}, GBS=${GBS}, LR=${LR}, seqlen=${SEQLEN}, max_steps=${MAX_NUM_STEPS:-}, min_prompt_groups=${MIN_PROMPT_GROUPS}" +echo "Streaming: over_sampling=${OVER_SAMPLING}, force_in_order=${FORCE_IN_ORDER}, age=${MAX_TRAJECTORY_AGE_STEPS}" +echo "Model: ${MODEL_PATH}" +echo "Checkpoint: ${CHECKPOINT_DIR}" +echo "WandB: ${WANDB_PROJ}/${WANDB_NAME}" +echo "==========================================" + +if [ "${DRY_RUN:-0}" = "1" ]; then + echo "[DRY_RUN] sbatch ${sbatch_args[*]} ${RUN_DIR}/ray.sub" + echo "[DRY_RUN] COMMAND (first 400 chars): ${COMMAND:0:400}..." + exit 0 +fi + +cd "${RUN_DIR}" +out=$(sbatch "${sbatch_args[@]}" "${RUN_DIR}/ray.sub") +echo "${out}" +job_id=$(echo "${out}" | grep -oE '[0-9]+' | head -1) +if [ -n "${job_id}" ]; then + echo "Job ID: ${job_id}" + echo "Monitor: squeue -j ${job_id}" + echo "Driver log: ${BASE_LOG_DIR}/${job_id}-logs/ray-driver.log" +fi diff --git a/nemo_rl/algorithms/advantage_estimator.py b/nemo_rl/algorithms/advantage_estimator.py index 59fd0f1ed01..faa40c3617f 100644 --- a/nemo_rl/algorithms/advantage_estimator.py +++ b/nemo_rl/algorithms/advantage_estimator.py @@ -50,7 +50,7 @@ def __init__(self, estimator_config: dict, loss_config: ClippedPGLossConfig): self.use_leave_one_out_baseline = estimator_config["use_leave_one_out_baseline"] self.normalize_rewards = estimator_config["normalize_rewards"] - def compute_advantage(self, prompt_ids, rewards, mask, **kwargs): + def compute_advantage(self, prompt_ids, rewards, mask, valid_mask=None, **kwargs): """Compute GRPO advantages. Args: @@ -58,6 +58,11 @@ def compute_advantage(self, prompt_ids, rewards, mask, **kwargs): rewards: Tensor of shape [batch_size] containing reward for each sample. mask: Response token mask of shape [batch_size, seq_len], 1 for valid response tokens, 0 for padding. Used only for expanding advantages to token-level shape. + valid_mask: Optional tensor of shape [batch_size], 1.0 for samples whose + reward should participate in the per-prompt baseline/std. Token-capture + placeholder rows carry 0.0 (their sample_mask already excludes them + from the loss; excluding them here keeps siblings' baselines unbiased). + None keeps the legacy all-valid behavior. **kwargs: Additional arguments (unused). Returns: @@ -66,7 +71,7 @@ def compute_advantage(self, prompt_ids, rewards, mask, **kwargs): baseline, std = calculate_baseline_and_std_per_prompt( prompt_ids, rewards, - torch.ones_like(rewards), + torch.ones_like(rewards) if valid_mask is None else valid_mask.float(), leave_one_out_baseline=self.use_leave_one_out_baseline, ) advantages = (rewards - baseline).unsqueeze(-1) diff --git a/nemo_rl/algorithms/async_utils/replay_buffer.py b/nemo_rl/algorithms/async_utils/replay_buffer.py index 22939bf72b3..ecc6958e992 100644 --- a/nemo_rl/algorithms/async_utils/replay_buffer.py +++ b/nemo_rl/algorithms/async_utils/replay_buffer.py @@ -12,13 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import threading as _threading +import uuid from collections import Counter +from collections.abc import Mapping from typing import Any, Iterable, Optional import ray from nemo_rl.algorithms.async_utils.interfaces import ReplayBufferProtocol +from nemo_rl.data_plane import KVBatchMeta +from nemo_rl.experience.interfaces import PromptGroupRecord +from nemo_rl.experience.payload import pack_payload, record_to_train_batch +from nemo_rl.experience.row_dump import maybe_dump_train_rows # Classes with @ray.remote can't be inherited from, so we split the implementation out. @@ -551,93 +558,292 @@ class ReplayBuffer(ReplayBufferImpl): pass -# WIP: DO NOT USE - This class is WIP and may be changed without notice, please DO NOT USE it. -# Will be replaced by TQReplayBuffer once TQ is ready. -@ray.remote # pragma: no cover -class ReplayBufferNew(ReplayBufferImpl): - """Staleness-window replay buffer. - - -- WIP: DO NOT USE -- - This class is WIP and may be changed without notice, please DO NOT USE it. - - Differences from ReplayBuffer: - - _evict(): Stale rows (trainer_version - weight_version > max_staleness) are evicted - at the start of every sample() call. - - sample(): selects trajectories in freshest-first order (default) or FIFO order, - controlled by the sample_freshest_first flag, from whatever remains in the buffer - after eviction. - - TODO: remove when cleaning up - - max_age_steps won't be used in ReplayBufferNew; - - self.target_weight_versions won't be used in ReplayBufferNew and will be removed - when cleaning up. target_weight_versions gates generation on specific trainer steps, - which causes generation pauses; ReplayBufferNew intentionally avoids this. - - add this class to nemo_rl/algorithms/async_utils/__init__.py +class TQReplayBuffer: + """Meta cache + TQ writer with reserve-then-commit slot semantics. + + meta_list, weight_list, ready_list, _group_ids are parallel; a slot stays + ready=False until commit fills it. """ def __init__( - self, max_size: int, max_staleness: int, sample_freshest_first: bool = True + self, + dp_client: Any, + partition_id: str, + *, + pad_value_dict: Mapping[str, int], + staging_partition_id: Optional[str] = None, ): - super().__init__(max_size) - if max_staleness < 0: - raise ValueError(f"max_staleness must be non-negative, got {max_staleness}") - self.max_staleness = max_staleness - # will move to StalenessSampler when we implement it - self.sample_freshest_first = sample_freshest_first + self._dp_client = dp_client + self._partition_id = partition_id + self._pad_value_dict = dict(pad_value_dict) + # Token-capture mode only (docs/design-docs/tq-gym-gate-authoritative.md): + # the staging partition whose per-call delta rows `remove` must clear + # alongside the canonical rows. None on the legacy path. + self._staging_partition_id = staging_partition_id + self.meta_list: list[Optional[KVBatchMeta]] = [] + self.start_weight_list: list[int] = [] + self.end_weight_list: list[int] = [] + # Per-slot target training step (set when force_in_order=True, else None). + self.target_step_list: list[Optional[int]] = [] + self.ready_list: list[bool] = [] + self._group_ids: list[str] = [] + # Parallel to the lists above; populated only in token-capture mode. + self._rollout_ids_list: list[Optional[list[str]]] = [] + self._staging_keys_list: list[Optional[list[str]]] = [] + + def reserve( + self, + *, + weight_version: int, + target_step: Optional[int] = None, + group_id: Optional[str] = None, + rollout_ids: Optional[list[str]] = None, + ) -> str: + """Append an unready slot tagged with weight_version. - def _evict(self, current_weight_version: int) -> None: - """Evict rows where trainer_version - weight_version > max_staleness. + Args: + weight_version: Weight version stamped on the slot. + target_step: Training step this slot targets; only consulted by StalenessSampler.force_in_order. + group_id: Per-group sample_id prefix; defaults to a fresh uuid4. + rollout_ids: Token-capture mode: the gate-registered rollout ids + this slot dispatched, recorded so cleanup can name what it + owns even before a receipt exists. - Must be called with self._lock held. + Returns: + group_id used by the matching commit. """ - min_valid = current_weight_version - self.max_staleness - stale = [i for i, v in enumerate(self.trajectory_versions) if v < min_valid] - self._remove_indices(stale) + if group_id is None: + group_id = str(uuid.uuid4()) + self.meta_list.append(None) + self.start_weight_list.append(weight_version) + self.end_weight_list.append(-1) + self.target_step_list.append(target_step) + self.ready_list.append(False) + self._group_ids.append(group_id) + self._rollout_ids_list.append( + list(rollout_ids) if rollout_ids is not None else None + ) + self._staging_keys_list.append(None) + return group_id - def sample( + async def commit( self, - num_prompt_groups: int, - current_weight_version: int, - max_age_steps: int, - ) -> Optional[dict[str, Any]]: - """Sample num_prompt_groups trajectories, freshest-first. + group_id: str, + record: PromptGroupRecord, + start_weight_version: int, + end_weight_version: int, + ) -> KVBatchMeta: + """Tensorize record, write N rows to TQ, and mark the slot ready. - Will evict stale rows before sampling, so we will get [current_weight_version - self.max_staleness, current_weight_version] valid trajectories. + Args: + group_id: group_id returned by the matching reserve call. + record: PromptGroupRecord to tensorize. + start_weight_version: Weight version stamped on the slot before rollout. + The same as the one from reserve, passed again to avoid race condition when lookup. + end_weight_version: Weight version stamped on the slot after rollout. Returns: - Dictionary with 'trajectories' and 'avg_trajectory_age' keys, or None. + KVBatchMeta for the committed group. + + Raises: + ValueError: group_id has no live slot (removed or never reserved). """ - with self._lock: - self._evict(current_weight_version) + # Check the slot is still live BEFORE writing: a slot evicted while + # its rollout was in flight used to fail here only after put_samples, + # leaking N orphaned rows into the partition. + if group_id not in self._group_ids: + raise ValueError( + f"TQReplayBuffer.commit: group {group_id} has no live slot " + "(evicted or never reserved); nothing written" + ) + train_batch = record_to_train_batch(record, pad_value_dict=self._pad_value_dict) + sample_ids, fields, tags = pack_payload( + train_batch, weight_version=start_weight_version, group_id=group_id + ) + maybe_dump_train_rows( + source="legacy_commit", + group_id=group_id, + sample_ids=list(sample_ids), + train_batch=train_batch, + weight_version=start_weight_version, + ) + await self._call_dp( + "put_samples", + sample_ids=sample_ids, + partition_id=self._partition_id, + fields=fields, + tags=tags, + ) - if not self.trajectories: - return None + # mirrors kv_first_write + lengths = train_batch["input_lengths"] + meta = KVBatchMeta( + partition_id=self._partition_id, + task_name="train", + sample_ids=list(sample_ids), + fields=list(fields.keys()), + sequence_lengths=[int(s) for s in lengths.tolist()], + tags=[dict(t) for t in tags], + ) - all_indices = range(len(self.trajectory_versions)) - if self.sample_freshest_first: - all_indices = sorted( - all_indices, - key=lambda i: self.trajectory_versions[i], - reverse=True, - ) + try: + idx = self._group_ids.index(group_id) + except ValueError: + # Evicted during the awaited write: un-write the rows so the + # partition holds nothing the buffer no longer tracks. + await self._call_dp( + "clear_samples", + sample_ids=list(sample_ids), + partition_id=self._partition_id, + ) + raise ValueError( + f"TQReplayBuffer.commit: group {group_id} was evicted during " + "the write; rows cleared" + ) from None + self.meta_list[idx] = meta + self.end_weight_list[idx] = end_weight_version + self.ready_list[idx] = True + return meta + + async def commit_finalized( + self, + group_id: str, + meta: KVBatchMeta, + group_min_wv: int, + group_max_wv: int, + *, + staging_keys: Optional[list[str]] = None, + ) -> KVBatchMeta: + """Mark a slot ready from finalizer output (token-capture mode). + + Unlike :meth:`commit`, the canonical rows are already in TQ — the + finalizer tensorized and put them — so this only fills the slot. + The slot's effective version is the group's OLDEST call version + (``group_min_wv``): staleness accounting stays conservative when a + rollout straddles a refit. - if len(all_indices) < num_prompt_groups: - print( - f"Insufficient trajectories: have {len(all_indices)}, " - f"need {num_prompt_groups}. Waiting." - ) - return None + Args: + group_id: group_id returned by the matching reserve call. + meta: KVBatchMeta the finalizer built over its published rows. + group_min_wv: Oldest weight version any call in the group used. + group_max_wv: Newest weight version any call in the group used. + staging_keys: The group's staged delta keys, recorded so + :meth:`remove` can clear the staging partition too. - selected = all_indices[:num_prompt_groups] - sampled_weights = [self.trajectory_versions[i] for i in selected] - avg_trajectory_age = current_weight_version - sum(sampled_weights) / len( - sampled_weights + Raises: + ValueError: group_id has no live slot (removed or never reserved). + """ + try: + idx = self._group_ids.index(group_id) + except ValueError: + raise ValueError( + f"TQReplayBuffer.commit_finalized: group {group_id} has no " + "live slot (evicted or never reserved)" + ) from None + self.meta_list[idx] = meta + self.start_weight_list[idx] = group_min_wv + self.end_weight_list[idx] = group_max_wv + self.ready_list[idx] = True + self._staging_keys_list[idx] = ( + list(staging_keys) if staging_keys is not None else None + ) + return meta + + def abort(self, group_id: str) -> bool: + """Drop an unready slot whose dispatch failed or was cancelled. + + Token-capture mode; called from the failed dispatch path. + No DataPlane rows are cleared: an unready slot published no canonical + rows, and its staged deltas (keys unknown before a receipt) are swept + by the staging TTL backstop. + + Returns: + True when a slot was dropped; False when the group_id has no + live slot (already committed+consumed or never reserved). + """ + try: + idx = self._group_ids.index(group_id) + except ValueError: + return False + if self.ready_list[idx]: + return False + self._delete_slot(idx) + return True + + def _delete_slot(self, idx: int) -> None: + del self.meta_list[idx] + del self.start_weight_list[idx] + del self.end_weight_list[idx] + del self.target_step_list[idx] + del self.ready_list[idx] + del self._group_ids[idx] + del self._rollout_ids_list[idx] + del self._staging_keys_list[idx] + + async def remove(self, idxs: list[int], remove_in_dp: bool) -> int: + """Drop entries at the given indices and optionally clear them from DataPlane. + + In token-capture mode (``staging_partition_id`` set), clearing a + group also clears its recorded staged delta rows, so eviction leaves + neither canonical nor staging bytes behind. + + Args: + idxs: Entry indices to drop. Must be within [0, size). + remove_in_dp: If True, also clear the dropped rows from DataPlane. + + Returns: + Number of group entries removed from the buffer. + """ + if len(idxs) == 0: + return 0 + + drop_idxs = sorted(idxs, reverse=True) + if drop_idxs[0] >= len(self.meta_list): + raise IndexError( + f"TQReplayBuffer.remove: indices out of range: {drop_idxs[0]}; " + f"size={len(self.meta_list)}" ) - sampled_items = [self.trajectories[i] for i in selected] - self._remove_indices(selected) + dropped_sample_ids: list[str] = [] + dropped_staging_keys: list[str] = [] + for i in drop_idxs: + meta = self.meta_list[i] + if meta is not None: + dropped_sample_ids.extend(meta.sample_ids) + staging_keys = self._staging_keys_list[i] + if staging_keys: + dropped_staging_keys.extend(staging_keys) + self._delete_slot(i) + + if remove_in_dp: + await self._call_dp( + "clear_samples", + sample_ids=dropped_sample_ids, + partition_id=self._partition_id, + ) + if dropped_staging_keys and self._staging_partition_id is not None: + await self._call_dp( + "clear_samples", + sample_ids=dropped_staging_keys, + partition_id=self._staging_partition_id, + ) - return { - "trajectories": sampled_items, - "avg_trajectory_age": avg_trajectory_age, - } + return len(drop_idxs) + + def size(self) -> int: + """Return the number of prompt-group entries currently held.""" + return len(self.meta_list) + + def __len__(self) -> int: + return len(self.meta_list) + + async def _call_dp(self, method_name: str, **kwargs: Any) -> Any: + """Call a DataPlaneClient method, awaiting Ray remotes if needed.""" + method = getattr(self._dp_client, method_name) + remote = getattr(method, "remote", None) + if remote is not None: + return await remote(**kwargs) + result = method(**kwargs) + if asyncio.iscoroutine(result): + return await result + return result diff --git a/nemo_rl/algorithms/async_utils/staleness_sampler.py b/nemo_rl/algorithms/async_utils/staleness_sampler.py new file mode 100644 index 00000000000..144dabdd3d2 --- /dev/null +++ b/nemo_rl/algorithms/async_utils/staleness_sampler.py @@ -0,0 +1,160 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prompt-group selection over a TQReplayBuffer.""" + +from nemo_rl.algorithms.async_utils.replay_buffer import TQReplayBuffer +from nemo_rl.data_plane import KVBatchMeta + + +class StalenessSampler: + """Pick complete prompt groups from a TQReplayBuffer. + + Args: + buffer: Shared TQReplayBuffer holding the candidate slots. + max_staleness_versions: Max weight-version gap a sample may have from the trainer. + sample_freshest_first: Prefer smallest lag when picking from the in-window set. + require_order: Take only from the oldest in-window weight_version and wait for its batch to fill. + force_in_order: Match each slot's target_step against current_train_weight, ignoring the window; mirrors legacy async_grpo target_weight semantics. + """ + + def __init__( + self, + buffer: TQReplayBuffer, + max_staleness_versions: int, + sample_freshest_first: bool = False, + require_order: bool = False, + force_in_order: bool = False, + ) -> None: + if max_staleness_versions < 0: + raise ValueError( + f"max_staleness_versions must be non-negative, got " + f"{max_staleness_versions}" + ) + if require_order and sample_freshest_first: + raise ValueError( + "require_order and sample_freshest_first are mutually exclusive" + ) + self._buffer = buffer + self.max_staleness_versions = max_staleness_versions + self.sample_freshest_first = sample_freshest_first + self.require_order = require_order + self.force_in_order = force_in_order + + async def select( + self, + *, + current_train_weight: int, + min_prompt_groups: int, + max_prompt_groups: int, + ) -> tuple[KVBatchMeta | None, int]: + """Concat up to max_prompt_groups eligible groups and drop them from the buffer. + + Eligibility = ready and weight in + [current_train_weight - max_staleness_versions, current_train_weight]. + DataPlane rows survive the local drop; caller clears them at step boundary. + + Args: + current_train_weight: Current trainer weight version. + min_prompt_groups: Minimum groups required; returns (None, 0) below this. + max_prompt_groups: Cap on groups returned when the threshold is met. + + Returns: + meta: Concatenated KVBatchMeta, or None if not enough groups. + num_groups: Number of prompt groups in meta; 0 when meta is None. + """ + if min_prompt_groups < 1: + raise ValueError(f"min_prompt_groups must be >= 1, got {min_prompt_groups}") + if max_prompt_groups < min_prompt_groups: + raise ValueError( + f"max_prompt_groups ({max_prompt_groups}) must be >= " + f"min_prompt_groups ({min_prompt_groups})" + ) + + if self.force_in_order: + # target_step exact match; staleness window ignored. + valid_idxs = [ + i + for i, target in enumerate(self._buffer.target_step_list) + if target == current_train_weight and self._buffer.ready_list[i] + ] + else: + min_valid_version = max( + 0, current_train_weight - self.max_staleness_versions + ) + if self.require_order: + in_window = [ + weight + for weight in self._buffer.start_weight_list + if min_valid_version <= weight <= current_train_weight + ] + if not in_window: + return None, 0 + target_version = min(in_window) + valid_idxs = [ + i + for i, weight in enumerate(self._buffer.start_weight_list) + if weight == target_version and self._buffer.ready_list[i] + ] + else: + valid_idxs = [ + i + for i, weight in enumerate(self._buffer.start_weight_list) + if min_valid_version <= weight <= current_train_weight + and self._buffer.ready_list[i] + ] + + if len(valid_idxs) < min_prompt_groups: + return None, 0 + + if self.sample_freshest_first: + valid_idxs.sort( + key=lambda i: ( + current_train_weight - self._buffer.start_weight_list[i], + i, + ) + ) + + requested_groups = min(len(valid_idxs), max_prompt_groups) + selected_idxs = valid_idxs[:requested_groups] + selected_metas = [self._buffer.meta_list[i] for i in selected_idxs] + + await self._buffer.remove(selected_idxs, remove_in_dp=False) + + return ( + selected_metas[0].concat(*selected_metas[1:]), # type: ignore + len(selected_idxs), + ) + + async def evict(self, *, current_train_weight: int) -> int: + """Drop groups whose weight falls below the staleness window. + + Future entries (weight > current_train_weight) are left alone. + + Args: + current_train_weight: Current trainer weight version; groups with + weight < current_train_weight - max_staleness_versions are dropped. + + Returns: + Number of group entries removed from the buffer. + """ + min_valid_version = max(0, current_train_weight - self.max_staleness_versions) + stale_idxs = [ + i + for i, weight in enumerate(self._buffer.start_weight_list) + if weight < min_valid_version + ] + if not stale_idxs: + return 0 + return await self._buffer.remove(stale_idxs, remove_in_dp=True) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 5435df757a9..0d6453a6e7b 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -23,7 +23,6 @@ import ray import torch from pydantic import BaseModel -from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy from torchdata.stateful_dataloader import StatefulDataLoader from transformers import AutoProcessor from transformers.tokenization_utils_base import PreTrainedTokenizerBase @@ -74,12 +73,7 @@ prepare_segment_topology, ) from nemo_rl.environments.interfaces import EnvironmentInterface -from nemo_rl.environments.nemo_gym import ( - NemoGym, - NemoGymConfig, - get_nemo_gym_uv_cache_dir, - get_nemo_gym_venv_dir, -) +from nemo_rl.environments.nemo_gym import spinup_nemo_gym_actor from nemo_rl.experience.rollouts import ( EffortLevelsConfig, run_async_multi_turn_rollout, @@ -460,60 +454,11 @@ def init_train_dataloader(dataset, suffix: str = ""): # spinup can overlap with vLLM model loading via deferred model load. enable_nemo_gym = _should_use_nemo_gym(master_config) nemo_gym_actor = None - if enable_nemo_gym: - nemo_gym_num_nodes = env_configs.get("nemo_gym", {}).get("num_gpu_nodes", 0) - ray_runtime_ctx = ray.get_runtime_context() - ray_cur_node_id = ray_runtime_ctx.get_node_id() - else: - nemo_gym_num_nodes = 0 - ray_cur_node_id = None def _spinup_nemo_gym(base_urls, model_name): """Spin up the NeMo Gym actor against the given generation server URLs.""" t0 = time.perf_counter() - nemo_gym_py_exec = get_actor_python_env("nemo_rl.environments.nemo_gym.NemoGym") - if nemo_gym_py_exec.startswith("uv"): - nemo_gym_py_exec = create_local_venv_on_each_node( - nemo_gym_py_exec, "nemo_rl.environments.nemo_gym.NemoGym" - ) - nemo_gym_dict = env_configs["nemo_gym"] - # NeMo-RL-side detection knobs are top-level NemoGymConfig fields - # (where the detector reads them), not part of Gym's global config. - invalid_tool_call_patterns = nemo_gym_dict.pop( - "invalid_tool_call_patterns", None - ) - thinking_tags = nemo_gym_dict.pop("thinking_tags", None) - # Pass prebuilt cache + venv dirs through the global config so the gym reuses - # image-baked venvs instead of rebuilding them. - uv_cache_dir = get_nemo_gym_uv_cache_dir() - if uv_cache_dir is not None: - nemo_gym_dict.setdefault("uv_cache_dir", uv_cache_dir) - uv_venv_dir = get_nemo_gym_venv_dir() - if uv_venv_dir is not None: - nemo_gym_dict.setdefault("uv_venv_dir", uv_venv_dir) - nemo_gym_cfg = NemoGymConfig( - model_name=model_name, - base_urls=base_urls, - invalid_tool_call_patterns=invalid_tool_call_patterns, - thinking_tags=thinking_tags, - initial_global_config_dict=nemo_gym_dict, - ) - nemo_gym_opts = {} - if nemo_gym_num_nodes: - nemo_gym_opts["scheduling_strategy"] = NodeAffinitySchedulingStrategy( - node_id=ray_cur_node_id, - soft=True, - ) - nemo_gym_opts["runtime_env"] = { - "py_executable": nemo_gym_py_exec, - "env_vars": { - **os.environ, - "VIRTUAL_ENV": nemo_gym_py_exec, - "UV_PROJECT_ENVIRONMENT": nemo_gym_py_exec, - }, - } - actor = NemoGym.options(**nemo_gym_opts).remote(nemo_gym_cfg) - ray.get(actor._spinup.remote()) + actor = spinup_nemo_gym_actor(env_configs, base_urls, model_name) return actor, time.perf_counter() - t0 total_nodes = cluster_config["num_nodes"] diff --git a/nemo_rl/algorithms/single_controller.py b/nemo_rl/algorithms/single_controller.py new file mode 100644 index 00000000000..6f0db9142b5 --- /dev/null +++ b/nemo_rl/algorithms/single_controller.py @@ -0,0 +1,727 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SingleController: asyncio orchestrator for the RL training loop. + +CPU-only Ray actor that runs three concurrent pumps and coordinates the +other actors via lightweight RPCs. SC sends control signals and reads +metadata only — model tensors still move through DataPlane or NCCL. + +Data flow: + _rollout_pump → rollout_manager.generate_and_push(prompt) + → TQReplayBuffer.reserve claims a slot at dispatch time; + run_rollout runs; TQReplayBuffer.commit tensorizes the + record, writes N training rows to TQ, and marks ready. + _train_pump → sampler.evict → buffer.remove (stale groups, with DP clear). + → sampler.select → drops chosen groups from buffer, returns + KVBatchMeta of K groups (or None); meta is already trainable. + → _advantage_pump (get → compute → put). + → trainer.train_on_meta. + → dp_client.clear_samples (trained groups; buffer already dropped). + _sync_weights → drain _inflight_rollouts → WeightSynchronizer.sync_weights. +""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any, Optional, Union + +import ray +import torch + +from nemo_rl.algorithms.async_utils.staleness_sampler import StalenessSampler +from nemo_rl.algorithms.single_controller_utils.config import ( + AdvantageConfig, + MasterConfig, + WeightSyncConfig, +) +from nemo_rl.algorithms.single_controller_utils.setup import SingleControllerBundle +from nemo_rl.algorithms.single_controller_utils.utils import ( + aggregate_step_metrics, + fields_for_put, + reduce_advantage_pump_metrics, + squeeze_trailing_unit_dim, + tensor_field, +) +from nemo_rl.data.interfaces import DatumSpec +from nemo_rl.data_plane import KVBatchMeta +from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration +from nemo_rl.models.generation.vllm import VllmGeneration +from nemo_rl.models.policy.tq_policy import TQPolicy +from nemo_rl.utils.logger import Logger +from nemo_rl.utils.timer import Timer + +Generation = Union[VllmGeneration, SGLangGeneration] + + +@ray.remote(num_cpus=1, num_gpus=0) # pragma: no cover +class SingleControllerActor: + """CPU-only Ray actor that orchestrates the RL training loop. + + Owns three concurrent asyncio tasks: + - _rollout_pump: dispatches prompts via RolloutManager; reserve+commit in + TQReplayBuffer preserves dispatch order + - _train_pump: evicts stale groups, samples a batch, trains, drops it + - _sync_weights: drain gate + weight synchronization + + All other actors are passive — they expose methods and wait to be called. + """ + + def __init__( + self, + master_config: MasterConfig, + bundle: SingleControllerBundle, + ) -> None: + """Initialize the SingleController actor. + + Args: + master_config: SC MasterConfig. + bundle: Pre-built bundle from setup_single_controller. Tests can + construct a bundle by hand (or with fakes) to bypass the real factories. + """ + self._advantage_cfg = AdvantageConfig() + # Mirror setup.py's init_reference_model condition: with no reference + # KL penalty the worker never builds reference_state_dict, so asking + # it for reference logprobs dies with AttributeError mid-step. + if master_config.loss_fn.reference_policy_kl_penalty <= 0: + self._advantage_cfg.reference_logprobs_field = None + self._weight_sync_cfg = WeightSyncConfig() + self._partition_id: str = bundle.partition_id + self._diagnostics: bool = False + + self._master_config = master_config + self._async_cfg = master_config.async_rl + self._dp_client = bundle.dp_client + self._gen: Generation = bundle.gen_handle + self._trainer: TQPolicy = bundle.trainer_handle + self._dataloader = bundle.dataloader + self._weight_synchronizer = bundle.weight_synchronizer + self._advantage_estimator = bundle.advantage_estimator + self._loss_fn = bundle.loss_fn + self._buffer = bundle.tq_buffer + self._rollout_manager = bundle.rollout_manager + # Rebind so writer and sampler share one buffer instance even + # when Ray deserializes rollout_manager and tq_buffer separately. + self._rollout_manager._tq_buffer = self._buffer + + # Built here, not on the driver: Logger backends (wandb/tb/...) hold + # _thread.lock that Ray can't cloudpickle into the actor. + self._logger = Logger(master_config.logger) # type: ignore + self._timer = Timer() + + # Pin clusters so RayVirtualCluster.__del__ doesn't remove the PGs. + self._train_cluster = bundle.train_cluster + self._inference_cluster = bundle.inference_cluster + + num_prompts_per_step = self._master_config.grpo["num_prompts_per_step"] + if num_prompts_per_step < self._async_cfg.min_prompt_groups_per_batch: + raise ValueError( + f"grpo.num_prompts_per_step ({num_prompts_per_step}) " + f"must be >= async_rl.min_prompt_groups_per_batch " + f"({self._async_cfg.min_prompt_groups_per_batch})" + ) + + if self._async_cfg.batch_selection_strategy == "strict_on_policy": + self._async_cfg.max_weight_staleness_versions = 0 + self._async_cfg.over_sampling = False + print( + "Using strict_on_policy, auto setting max_weight_staleness_versions to 0 and over_sampling to False.", + flush=True, + ) + + if not self._async_cfg.over_sampling: + expected_buffer = num_prompts_per_step * ( + self._async_cfg.max_weight_staleness_versions + 1 + ) + if self._async_cfg.max_buffered_rollouts != expected_buffer: + raise ValueError( + f"over_sampling=False requires max_buffered_rollouts " + f"({self._async_cfg.max_buffered_rollouts}) == " + f"num_prompts_per_step * (max_weight_staleness_versions + 1) " + f"({expected_buffer})" + ) + + if self._async_cfg.force_in_order and self._async_cfg.over_sampling: + raise ValueError( + "force_in_order=True requires over_sampling=False so that each " + "dispatched batch corresponds to exactly one target training step." + ) + + # SC split path does one optimizer.step per RL step. + # TODO: support multi-mini-step (legacy train() does gbs-sized + # mini-steps with shared prev_logprobs). + rl_step_samples = ( + num_prompts_per_step + * self._master_config.grpo["num_generations_per_prompt"] + ) + train_gbs = self._master_config.policy["train_global_batch_size"] + if rl_step_samples != train_gbs: + raise ValueError( + f"num_prompts_per_step * num_generations_per_prompt " + f"({rl_step_samples}) must equal policy.train_global_batch_size " + f"({train_gbs}) so that one RL step maps to exactly one " + f"optimizer.step. Multi-mini-step inside a single RL step is " + f"not supported on the SC split path." + ) + + self._sampler = StalenessSampler( + self._buffer, + max_staleness_versions=self._async_cfg.max_weight_staleness_versions, + require_order=not self._async_cfg.over_sampling, + force_in_order=self._async_cfg.force_in_order, + ) + + # ── asyncio state ────────────────────────────────────────────────── + # Gate: cleared during _sync_weights, set when generation may proceed + self._rollout_permitted: asyncio.Event = asyncio.Event() + self._rollout_permitted.set() + + # Count of in-flight generate_and_push calls + self._inflight_rollouts: int = 0 + + # Cancellation handles for in-flight rollout dispatches. + self._dispatched_rollouts: set[asyncio.Task[None]] = set() + + # over_sampling=False batch gate: farthest trainer_version covered by + # already-dispatched batches. + self._max_rollout_version: int = -1 + + # Backpressure valve: max unconsumed rollout groups allowed in DataPlane. + # Acquired before each rollout dispatch; released when the buffer + # drops a group (sampler.evict or post-train buffer.remove). + self._buffer_capacity: asyncio.Semaphore = asyncio.Semaphore( + self._async_cfg.max_buffered_rollouts + ) + + self._trainer_version: int = 0 + self._train_steps: int = 0 + self._step_log_dict: dict[str, list] = { + "rewards": [], + "masked_advantages": [], + "sequence_lengths": [], + } + + print( + f"SingleControllerActor: " + f"staleness_cap={self._async_cfg.max_weight_staleness_versions} " + f"buffer={self._async_cfg.max_buffered_rollouts} " + f"inflight={self._async_cfg.max_inflight_prompts} " + f"over_sampling={self._async_cfg.over_sampling} " + f"transport={self._weight_sync_cfg.transport}", + flush=True, + ) + + # ── public API ───────────────────────────────────────────────────────── + + async def run(self) -> dict[str, Any]: + """Main entry point. Runs until max_train_steps is reached.""" + # Synchronize weights before starting the pumps + await self._sync_weights() + + # Start the rollout and train pumps + rollout_task = asyncio.create_task(self._rollout_pump()) + train_task = asyncio.create_task(self._train_pump()) + + # Wait until the train pump is done + await train_task + self._logger.finish() + + # Cancel the rollout pump and any in-flight dispatches so we exit immediately. + rollout_task.cancel() + try: + await rollout_task + except asyncio.CancelledError: + pass + inflight = list(self._dispatched_rollouts) + for task in inflight: + task.cancel() + if inflight: + await asyncio.gather(*inflight, return_exceptions=True) + + return { + "train_steps": self._train_steps, + "trainer_version": self._trainer_version, + } + + async def ping(self) -> dict[str, Any]: + """Liveness check — returns immediately if event loop is running.""" + return { + "alive": True, + "trainer_version": self._trainer_version, + "train_steps": self._train_steps, + "inflight_rollouts": self._inflight_rollouts, + "rollout_permitted": self._rollout_permitted.is_set(), + } + + # ── internal helpers ─────────────────────────────────────────────────── + + async def _ray_get(self, obj_ref: Any) -> Any: + """Await a Ray ObjectRef without blocking the asyncio event loop.""" + return await obj_ref + + async def _call_dp(self, method_name: str, **kwargs) -> Any: + """Call a DataPlaneClient method or a Ray actor exposing that method.""" + method = getattr(self._dp_client, method_name) + remote = getattr(method, "remote", None) + if remote is not None: + return await self._ray_get(remote(**kwargs)) + result = method(**kwargs) + if asyncio.iscoroutine(result): + return await result + return result + + # ── the three pumps + advantage helper ──────────────────────────────── + + async def _rollout_pump(self) -> None: + """Continuously dispatch rollout tasks until cancellation. + + Per batch (over_sampling=False): + 0. Wait while _max_rollout_version >= trainer_version + max_staleness, + then claim the next step by incrementing _max_rollout_version. + + Per prompt: + 1. Acquire _buffer_capacity slot (backpressure) + 2. Acquire sem (cap concurrent in-flight rollouts) + 3. Wait for _rollout_permitted (paused during weight sync) + 4. Call rollout_manager.generate_and_push(prompt) — local async + RolloutManager reserves a slot, runs the rollout, then commits the + group via TQReplayBuffer (→ dp_client.put_samples + mark ready) + 5. Decrement _inflight_rollouts + """ + sem = asyncio.Semaphore(self._async_cfg.max_inflight_prompts) + over_sampling = self._async_cfg.over_sampling + max_staleness = self._async_cfg.max_weight_staleness_versions + force_in_order = self._async_cfg.force_in_order + print("rollout_pump: starting", flush=True) + + async def _dispatch_one_prompt( + prompt: DatumSpec, target_step: Optional[int] + ) -> None: + self._inflight_rollouts += 1 + try: + await self._rollout_manager.generate_and_push( + prompt, target_step=target_step + ) + if self._diagnostics: + content = "" + for i in range(len(prompt["message_log"])): + if prompt["message_log"][i]["role"] == "user": + content = prompt["message_log"][i]["content"] + break + print(f" rollout done for prompt='{content[:20]}...'", flush=True) + except BaseException: + # A failed dispatch never reaches the train pump, so its + # _buffer_capacity slot (released there per consumed group) + # would leak and eventually starve the rollout pump. + self._buffer_capacity.release() + raise + finally: + self._inflight_rollouts -= 1 + sem.release() + + max_epochs = self._master_config.grpo["max_num_epochs"] + epoch = 0 + while max_epochs is None or epoch < max_epochs: + for prompt_batch in self._dataloader: + # over_sampling=False: batch-level gate on max_rollout_version. + if not over_sampling: + while ( + self._max_rollout_version + >= self._trainer_version + max_staleness + ): + await asyncio.sleep(0.005) + self._max_rollout_version += 1 + + # target_step = batch dispatch index when force_in_order is on. + target_step = self._max_rollout_version if force_in_order else None + + for prompt_idx in range(prompt_batch.size): + prompt: DatumSpec = { # type: ignore + k: v[prompt_idx] for k, v in prompt_batch.items() + } + + # check if buffer is full + await self._buffer_capacity.acquire() + # check if inflight rollouts is full + await sem.acquire() + # wait for rollout to be permitted + await self._rollout_permitted.wait() + + # dispatch rollout + task = asyncio.create_task( + _dispatch_one_prompt(prompt, target_step) + ) + self._dispatched_rollouts.add(task) + task.add_done_callback(self._dispatched_rollouts.discard) + epoch += 1 + + print(f"rollout_pump: completed {epoch} epoch(s)", flush=True) + + async def _train_pump(self) -> None: + """Drain stale groups, sample, train, drop. + + Per step: + 1. sampler.evict drops stale groups from the buffer and clears their TQ rows. + 2. sampler.select returns K prompt groups (or None) and drops them from the + buffer; DP rows survive so the trainer can read them. Already trainable — + buffer wrote training-shaped rows at rollout time. + 3. _advantage_pump(train_meta). + 4. trainer.train_microbatch_from_meta + finish_train_step. + 5. dp_client.clear_samples on consumed sample_ids; release _buffer_capacity + per dropped group, then sync. + """ + adv_cfg = self._advantage_cfg + grpo_cfg = self._master_config.grpo + + # TODO: fix the compute_prev_logprobs and compute_reference_logprobs logic + compute_prev_logprobs = adv_cfg.policy_logprobs_field is not None + compute_reference_logprobs = adv_cfg.reference_logprobs_field is not None + + while self._train_steps < grpo_cfg["max_num_steps"]: + step_id = f"sc-step-{self._train_steps:06d}" + groups_dispatched = 0 + min_sample_version = None + step_open = False + + with self._timer.time("total_step_time"): + while groups_dispatched < grpo_cfg["num_prompts_per_step"]: + # Wait for a selectable batch + with self._timer.time("exposed_generation"): + await asyncio.sleep(0) + + # Evict stale groups + evicted = await self._sampler.evict( + current_train_weight=self._trainer_version, + ) + if evicted: + print( + f" evicted {evicted} stale prompt group(s)", + flush=True, + ) + for _ in range(evicted): + self._buffer_capacity.release() + + # Select a batch + max_prompt_groups = ( + grpo_cfg["num_prompts_per_step"] - groups_dispatched + ) + min_prompt_groups = min( + self._async_cfg.min_prompt_groups_per_batch, + max_prompt_groups, + ) + train_meta, num_groups = await self._sampler.select( + current_train_weight=self._trainer_version, + min_prompt_groups=min_prompt_groups, + max_prompt_groups=max_prompt_groups, + ) + + # If no batch is selectable, sleep and retry + if train_meta is None: + await asyncio.sleep(0.05) + continue + + # Release buffer capacity + for _ in range(num_groups): + self._buffer_capacity.release() + + # Compute prev_logprobs / ref_logprobs + with self._timer.time("logprob_inference_prep"): + await asyncio.to_thread(self._trainer.prepare_for_lp_inference) + with self._timer.time("policy_and_reference_logprobs"): + if compute_prev_logprobs: + await asyncio.to_thread( + self._trainer.get_logprobs_from_meta, train_meta + ) + if compute_reference_logprobs: + await asyncio.to_thread( + self._trainer.get_reference_policy_logprobs_from_meta, + train_meta, + ) + + # Compute advantages + with self._timer.time("advantage_calculation"): + train_meta = await self._advantage_pump(train_meta) + + # Train + with self._timer.time("training_prep"): + await asyncio.to_thread(self._trainer.prepare_for_training) + with self._timer.time("policy_training"): + if not step_open: + await asyncio.to_thread( + self._trainer.begin_train_step, + step_id, + loss_fn=self._loss_fn, + ) + step_open = True + await asyncio.to_thread( + self._trainer.train_microbatch_from_meta, + step_id, + train_meta, + ) + + if train_meta.sequence_lengths: + self._step_log_dict["sequence_lengths"].extend( + int(s) for s in train_meta.sequence_lengths + ) + + # Refresh min_sample_version + curr_min_sample_version = min( + t["weight_version"] + for t in train_meta.tags # type: ignore + ) + if min_sample_version is not None: + min_sample_version = min( + min_sample_version, curr_min_sample_version + ) + else: + min_sample_version = curr_min_sample_version + + # Remove consumed sample_ids from the buffer + await self._call_dp( + "clear_samples", + sample_ids=list(train_meta.sample_ids), + partition_id=self._partition_id, + ) + + groups_dispatched += num_groups + + if not step_open: + print( + "train_pump: rollout exhausted before any group ready", + flush=True, + ) + break + + with self._timer.time("policy_training"): + result = await asyncio.to_thread( + self._trainer.finish_train_step, step_id + ) + + step_metrics = aggregate_step_metrics(result) + step_metrics.update( + reduce_advantage_pump_metrics(**self._step_log_dict) + ) + self._step_log_dict = {k: [] for k in self._step_log_dict} + + self._trainer_version += 1 + self._train_steps += 1 + with self._timer.time("weight_sync"): + await self._sync_weights() + + timing_metrics: dict[str, float] = self._timer.get_timing_metrics( + reduction_op="sum" + ) # type: ignore + + total_time = timing_metrics.get("total_step_time", 0.0) + cluster_cfg = self._master_config.cluster + total_num_gpus = cluster_cfg["num_nodes"] * cluster_cfg["gpus_per_node"] + if total_time > 0 and "global_valid_toks" in step_metrics: + timing_metrics["valid_tokens_per_sec_per_gpu"] = ( + step_metrics["global_valid_toks"] / total_time / total_num_gpus + ) + + print("\n⏱️ Timing:") + print(f" • Total step time: {total_time:.2f}s") + for k, v in sorted( + timing_metrics.items(), key=lambda item: item[1], reverse=True + ): + if k == "total_step_time": + continue + percent = (v / total_time * 100) if total_time > 0 else 0.0 + print(f" • {k}: {v:.2f}s ({percent:.1f}%)") + + # TODO: checkpointing (save_period/top-k metric_name, + # policy.save_checkpoint, dataloader state, TQReplayBuffer state). + # TODO: per-step train_data jsonl dump, vllm metrics logger, + # histogram log, rollout_metrics, seq_logprob_error_metrics, + # pretty-print "Training Results" block, print_performance_metrics. + print(f"step_metrics={step_metrics}", flush=True) + self._logger.log_metrics( + step_metrics, step=self._train_steps, prefix="train" + ) + self._logger.log_metrics( + timing_metrics, step=self._train_steps, prefix="timing/train" + ) + if self._master_config.token_capture.enabled: + await self._log_gate_metrics() + self._timer.reset() + + # min sample version refers to the version each consumed sample was + # generated with; lag = current trainer version - oldest sample version. + lag = self._trainer_version - min_sample_version # type: ignore + print( + f"train step {self._train_steps}/{grpo_cfg['max_num_steps']} " + f"trainer_v={self._trainer_version} " + f"lag={lag} ", + flush=True, + ) + + async def _sync_weights(self) -> None: + """Drain in-flight rollouts then synchronize weights. + + SC owns the drain gate (when to sync); WeightSynchronizer owns how. + + Flow: + 1. _rollout_permitted.clear() — no new dispatches + 2. drain _inflight_rollouts → 0 (5ms poll) + 3. weight_synchronizer.sync_weights(trainer_version) + 4. _rollout_permitted.set() — resume + """ + self._rollout_permitted.clear() + + # TODO: currently sync_weights is not implemented, comment out for now + # # Drain: wait for all in-flight rollouts to complete before NCCL + # # Critical: if GenWorker has queued calls when NCCL init is dispatched, + # # the init sits behind them — trainer blocks in rendezvous → deadlock + # drain_start = time.monotonic() + # while self._inflight_rollouts > 0: + # await asyncio.sleep(0.005) + + # drain_elapsed = time.monotonic() - drain_start + # print( + # f" _sync_weights: drained in {drain_elapsed:.3f}s, " + # f"syncing weights v{self._trainer_version}", + # flush=True, + # ) + + t0 = time.monotonic() + await asyncio.to_thread(self._weight_synchronizer.sync_weights) + elapsed = time.monotonic() - t0 + + print(f" _sync_weights: sync done in {elapsed:.3f}s", flush=True) + self._rollout_manager.set_weight_version(self._trainer_version) + if self._master_config.token_capture.enabled: + # Rotate the version vLLM workers stamp on captured model calls + # (per-call tagging; group staleness = min over the group's calls). + await asyncio.to_thread( + self._gen.set_rollout_weight_version, self._trainer_version + ) + self._rollout_permitted.set() + + async def _log_gate_metrics(self) -> None: + """Log the capture gate's cumulative § 8 counters + token_in_rate. + + Counters are cumulative over the run; ``gate/token_in_rate`` is the + cumulative marker-hit rate over all admitted model calls. Fetch + failures are logged and swallowed — metrics must never kill a step. + """ + try: + counters = await self._rollout_manager.gate_metrics() + except (RuntimeError, OSError) as error: + print(f"gate metrics fetch failed: {error}", flush=True) + return + if not counters: + return + calls = counters["token_in"] + sum( + v for k, v in counters.items() if k.startswith("fallback_") + ) + gate_metrics: dict[str, float] = {k: float(v) for k, v in counters.items()} + if calls: + gate_metrics["token_in_rate"] = counters["token_in"] / calls + self._logger.log_metrics(gate_metrics, step=self._train_steps, prefix="gate") + print(f"gate_metrics={gate_metrics}", flush=True) + + async def _advantage_pump(self, meta: KVBatchMeta) -> KVBatchMeta: + """Fetch advantage inputs, compute advantages, and write them back. + + SC owns the prompt-group-scoped advantage stage because the selected + ``KVBatchMeta`` still contains complete prompt groups before trainer + DP sharding. Tensor payloads still move through DataPlane: SC fetches + only the configured advantage input columns and writes the computed + ``advantages`` column back under the same ``sample_ids``. + """ + if self._advantage_estimator is None: + return meta + adv_cfg = self._advantage_cfg + + data = await self._call_dp( + "get_samples", + sample_ids=meta.sample_ids, + partition_id=meta.partition_id, + select_fields=self._advantage_input_fields(), + ) + + prompt_ids = tensor_field(data, adv_cfg.prompt_ids_field) + rewards = squeeze_trailing_unit_dim( + tensor_field(data, adv_cfg.reward_field) + ).float() + self._step_log_dict["rewards"].append(rewards.detach()) + token_mask = tensor_field(data, adv_cfg.token_mask_field).float() + sample_mask = squeeze_trailing_unit_dim( + tensor_field(data, adv_cfg.sample_mask_field) + ).float() + mask = token_mask * sample_mask.unsqueeze(-1) + + repeated_batch: dict[str, torch.Tensor] = { + "total_reward": rewards, + } + for field_name in adv_cfg.repeated_batch_fields: + repeated_batch[field_name] = squeeze_trailing_unit_dim( + tensor_field(data, field_name) + ) + + kwargs: dict[str, torch.Tensor] = {} + if adv_cfg.policy_logprobs_field is not None: + kwargs["logprobs_policy"] = tensor_field( + data, + adv_cfg.policy_logprobs_field, + ) + if adv_cfg.reference_logprobs_field is not None: + kwargs["logprobs_reference"] = tensor_field( + data, + adv_cfg.reference_logprobs_field, + ) + + advantages = self._advantage_estimator.compute_advantage( + prompt_ids=prompt_ids, + rewards=rewards, + mask=mask, + repeated_batch=repeated_batch, + # Real validity (token-capture placeholders carry sample_mask 0) + # instead of the hardwired all-ones — § 9.1, advantage_estimator. + valid_mask=sample_mask, + **kwargs, + ) + self._step_log_dict["masked_advantages"].append( + torch.masked_select(advantages.detach(), mask.bool()) + ) + + await self._call_dp( + "put_samples", + sample_ids=meta.sample_ids, + partition_id=meta.partition_id, + fields=fields_for_put( + meta, + {adv_cfg.output_field: advantages}, + ), + ) + return meta.with_fields([adv_cfg.output_field]) + + # ── utility helpers ──────────────────────────────────────────────────── + + def _advantage_input_fields(self) -> list[str]: + adv_cfg = self._advantage_cfg + fields = [ + adv_cfg.prompt_ids_field, + adv_cfg.reward_field, + adv_cfg.token_mask_field, + adv_cfg.sample_mask_field, + *adv_cfg.repeated_batch_fields, + ] + if adv_cfg.policy_logprobs_field is not None: + fields.append(adv_cfg.policy_logprobs_field) + if adv_cfg.reference_logprobs_field is not None: + fields.append(adv_cfg.reference_logprobs_field) + return list(dict.fromkeys(fields)) diff --git a/nemo_rl/algorithms/single_controller_utils/__init__.py b/nemo_rl/algorithms/single_controller_utils/__init__.py new file mode 100644 index 00000000000..a2c7001f926 --- /dev/null +++ b/nemo_rl/algorithms/single_controller_utils/__init__.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SingleController utilities: config schema + setup factories.""" + +from nemo_rl.algorithms.single_controller_utils.config import ( + AdvantageConfig, + AsyncRLConfig, + MasterConfig, + WeightSyncConfig, +) +from nemo_rl.algorithms.single_controller_utils.setup import ( + SingleControllerBundle, + setup_single_controller, +) + +__all__ = [ + "AdvantageConfig", + "AsyncRLConfig", + "MasterConfig", + "SingleControllerBundle", + "WeightSyncConfig", + "setup_single_controller", +] diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py new file mode 100644 index 00000000000..991d5c80206 --- /dev/null +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -0,0 +1,135 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal, Optional + +from pydantic import BaseModel, Field + +from nemo_rl.algorithms.grpo import GRPOConfig, GRPOLoggerConfig +from nemo_rl.algorithms.loss import ClippedPGLossConfig +from nemo_rl.data import DataConfig +from nemo_rl.data_plane.interfaces import DataPlaneConfig +from nemo_rl.distributed.virtual_cluster import ClusterConfig +from nemo_rl.models.policy import PolicyConfig +from nemo_rl.utils.checkpoint import CheckpointingConfig + +# ── User-facing SingleController configs ──────────────────────────────────── + + +class AsyncRLConfig(BaseModel, extra="allow"): + batch_selection_strategy: Literal[ + "strict_on_policy", + "staleness_window", + ] = "strict_on_policy" + # Sampler / on-policy enforcement. + max_weight_staleness_versions: int = 1 + min_prompt_groups_per_batch: int = 2 + # Pump concurrency caps. + max_inflight_prompts: int = 8 + max_buffered_rollouts: int = 8 + # True : over-generates and wastes rollouts that age past the staleness window; + # False: enforces per-weight-version dispatch quota. + over_sampling: bool = True + # Tag rollouts with their dispatch-time target step and require an exact + # match at sample time (legacy target_weight semantics). Requires + # over_sampling=False. + force_in_order: bool = False + + +class TokenCaptureConfig(BaseModel, extra="allow"): + """Gate-authoritative token capture (token-in/token-out via NeMo-Gym). + + Dormant by default: with ``enabled=False`` every legacy codepath behaves + exactly as before — no staging partition is registered, no gate is + installed, and rollouts ride the token-echo path. See + docs/design-docs/tq-gym-gate-authoritative.md. + """ + + enabled: bool = False + # TQ partition holding per-call staged token deltas (cleared by the + # finalizer; distinct from the canonical rollout partition). + staging_partition: str = "rollout_staging" + # A failed worker-side stage poisons the rollout; "continue" serves the + # completion and lets the finalizer emit a placeholder row, "abort" fails + # the whole rollout at the gate. + on_capture_failure: Literal["continue", "abort"] = "continue" + # "allow" trains groups whose calls span a refit (staleness accounted via + # group_min_wv); "reject" placeholders them. Strict modes beyond the MVP + # matrix raise NotImplementedError at setup. + mixed_weight_version_policy: Literal["allow", "reject"] = "allow" + # Drop the whole group when fewer than this fraction of its rollouts + # produced valid rows (None keeps every group). + min_valid_fraction_per_group: Optional[float] = None + # Gate-side cleanup backstops. + registration_ttl_s: float = 3600.0 + staging_ttl_s: float = 3600.0 + # Gym LineageIndex capacity (finding M: it holds each in-flight rollout's + # full cumulative token sequence, and eviction of a live rollout silently + # degrades token-in to fallbacks). None = derived at setup from the + # training config: rollouts ≈ 2 × max in-flight; tokens ≈ rollouts × max + # sequence length. Set explicitly for agentic workloads whose per-rollout + # call trees hold more than one context of tokens. + lineage_max_rollouts: Optional[int] = None + lineage_max_tokens: Optional[int] = None + # Bearer token for the gate's /ng-control/* routes (finding S). None = + # minted per run at setup; set explicitly only for multi-controller + # setups that must share one gate. + control_auth_token: Optional[str] = None + # Hard deadline per control-plane call (S5 finding: gate death must + # surface as a failed dispatch, not a silent retry stall). + control_timeout_s: float = 60.0 + # Directory for the Gym base capture layer the gate rides on (#2124-c1: + # the capture middleware only engages with a capture dir configured; the + # dir stays essentially empty on the gate path). None = derived at setup + # under the run's log dir. + capture_dir: Optional[str] = None + + +class MasterConfig(BaseModel, extra="allow"): + policy: PolicyConfig + loss_fn: ClippedPGLossConfig + env: dict[str, Any] = Field(default_factory=dict) + data: DataConfig + grpo: GRPOConfig + logger: GRPOLoggerConfig + cluster: ClusterConfig + checkpointing: CheckpointingConfig + data_plane: DataPlaneConfig + async_rl: AsyncRLConfig = Field(default_factory=AsyncRLConfig) + token_capture: TokenCaptureConfig = Field(default_factory=TokenCaptureConfig) + + +# ── Internal SingleController configs ──────────────────────────────────── + + +@dataclass +class AdvantageConfig: + output_field: str = "advantages" + prompt_ids_field: str = "prompt_ids_for_adv" + reward_field: str = "total_reward" + token_mask_field: str = "token_mask" + sample_mask_field: str = "sample_mask" + repeated_batch_fields: list[str] = field(default_factory=list) + policy_logprobs_field: Optional[str] = "prev_logprobs" + reference_logprobs_field: Optional[str] = "reference_policy_logprobs" + + +@dataclass +class WeightSyncConfig: + transport: str = "stub" + nccl_addr: str = "127.0.0.1" + nccl_port: Optional[int] = None diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py new file mode 100644 index 00000000000..224573fbba7 --- /dev/null +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -0,0 +1,532 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Driver-side factory for the SingleController (async-RL) training path. + +setup builds the full SingleControllerBundle on the driver and the caller passes it to +SingleControllerActor.remote. Everything lives on the driver because driver-side +TQPolicy owns the worker group directly — running this inside another Ray actor nests +runtime_envs and breaks Ray's resource resolution (see the PR #2692 follow-up). +""" + +from __future__ import annotations + +import os +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Any, Optional + +from torchdata.stateful_dataloader import StatefulDataLoader +from transformers import AutoProcessor +from transformers.tokenization_utils_base import PreTrainedTokenizerBase + +from nemo_rl.algorithms.async_utils.replay_buffer import TQReplayBuffer +from nemo_rl.algorithms.grpo import _create_advantage_estimator, _should_use_nemo_gym +from nemo_rl.algorithms.loss import ClippedPGLossFn +from nemo_rl.algorithms.loss.interfaces import LossFunction +from nemo_rl.algorithms.single_controller_utils.config import MasterConfig +from nemo_rl.algorithms.utils import set_seed +from nemo_rl.data.collate_fn import rl_collate_fn +from nemo_rl.data.utils import setup_response_data +from nemo_rl.data_plane import build_data_plane_client +from nemo_rl.distributed.virtual_cluster import RayVirtualCluster +from nemo_rl.environments.interfaces import EnvironmentInterface +from nemo_rl.environments.nemo_gym import spinup_nemo_gym_actor +from nemo_rl.experience.rollout_manager import RolloutManager +from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration +from nemo_rl.models.generation.vllm import VllmGeneration +from nemo_rl.models.policy.tq_policy import TQPolicy +from nemo_rl.weight_sync import WeightSynchronizer, create_weight_synchronizer + + +@dataclass +class SingleControllerBundle: + """All inputs SingleControllerActor needs, built driver-side by setup_single_controller(). + + Passed as a single arg to SingleControllerActor.remote so the actor's __init__ does + no construction work — every heavy object is cloudpickled in. + """ + + gen_handle: Any + trainer_handle: Any # driver-side TQPolicy + env_handles: dict[str, EnvironmentInterface] + train_cluster: RayVirtualCluster + inference_cluster: RayVirtualCluster + dp_client: Any + dataloader: StatefulDataLoader + weight_synchronizer: WeightSynchronizer + advantage_estimator: Any + loss_fn: LossFunction + rollout_manager: RolloutManager + tq_buffer: TQReplayBuffer + partition_id: str + + +def _build_clusters( + master_config: MasterConfig, +) -> tuple[RayVirtualCluster, RayVirtualCluster]: + """Allocate train + inference clusters; one shared cluster when colocated.""" + cluster_config = master_config.cluster + generation_config = master_config.policy["generation"] + colocated = generation_config["colocated"]["enabled"] + backend = generation_config["backend"] + num_nodes = cluster_config["num_nodes"] + gpus_per_node = cluster_config["gpus_per_node"] + port_range_low = cluster_config.get("master_port_range_low") + port_range_high = cluster_config.get("master_port_range_high") + + if colocated: + # Policy + generation share GPUs — one cluster. + cluster = RayVirtualCluster( + name="sc_policy_cluster", + bundle_ct_per_node_list=[gpus_per_node] * num_nodes, + use_gpus=True, + num_gpus_per_node=gpus_per_node, + max_colocated_worker_groups=1 if backend == "megatron" else 2, + port_range_low=port_range_low, + port_range_high=port_range_high, + ) + return cluster, cluster + + # Non-colocated: split node into train + inference clusters. + assert backend != "megatron", ( + "Non-colocated inference is not supported for Megatron generation backends." + ) + inference_resources = generation_config["colocated"]["resources"] + inference_gpus_per_node = inference_resources["gpus_per_node"] + inference_nodes = inference_resources["num_nodes"] or 1 + if num_nodes == 1: + train_gpus_per_node = gpus_per_node - inference_gpus_per_node + train_nodes = 1 + assert train_gpus_per_node > 0, ( + f"Not enough GPUs for training: {gpus_per_node} - {inference_gpus_per_node} = {train_gpus_per_node}" + ) + else: + train_gpus_per_node = gpus_per_node + train_nodes = num_nodes - inference_nodes + assert train_nodes > 0, ( + f"train_nodes must be > 0: {num_nodes} - {inference_nodes} = {train_nodes}" + ) + + train_cluster = RayVirtualCluster( + name="sc_train_cluster", + bundle_ct_per_node_list=[train_gpus_per_node] * train_nodes, + use_gpus=True, + num_gpus_per_node=train_gpus_per_node, + max_colocated_worker_groups=1, + port_range_low=port_range_low, + port_range_high=port_range_high, + ) + inference_cluster = RayVirtualCluster( + name="sc_inference_cluster", + bundle_ct_per_node_list=[inference_gpus_per_node] * inference_nodes, + use_gpus=True, + num_gpus_per_node=inference_gpus_per_node, + max_colocated_worker_groups=1, + port_range_low=port_range_low, + port_range_high=port_range_high, + ) + return train_cluster, inference_cluster + + +def _build_generation( + inference_cluster: RayVirtualCluster, + master_config: MasterConfig, +): + """Spin up the generation backend (vLLM or SGLang).""" + generation_config = master_config.policy["generation"] + generation_config["model_name"] = master_config.policy["model_name"] + backend = generation_config["backend"] + if backend == "vllm": + generation_config["vllm_kwargs"]["hf_overrides"] = master_config.policy.get( + "hf_config_overrides", {} + ) + gen = VllmGeneration(cluster=inference_cluster, config=generation_config) + elif backend == "sglang": + generation_config["sglang_cfg"].setdefault( + "model_path", master_config.policy["model_name"] + ) + gen = SGLangGeneration(cluster=inference_cluster, config=generation_config) + else: + raise ValueError( + f"single_controller_utils.setup only supports vllm or sglang generation; got {backend!r}" + ) + gen.finish_generation() + return gen + + +def _build_trainer( + train_cluster: RayVirtualCluster, + master_config: MasterConfig, + tokenizer, + processor, +): + """Build the TQ-mediated trainer (driver-side TQPolicy). + + Driver-side on purpose: instantiating TQPolicy inside another Ray + actor nests runtime_envs and triggers Ray's + get_accelerator_ids_for_accelerator_resource IndexError. Keep this + here until PolicyTrainerActor (PR #2692) lands. + """ + loss_config = master_config.loss_fn + init_reference_model = loss_config.reference_policy_kl_penalty > 0 + return TQPolicy( + cluster=train_cluster, + config=master_config.policy, + tokenizer=tokenizer, + processor=processor, + weights_path=None, + optimizer_path=None, + init_optimizer=True, + init_reference_model=init_reference_model, + dp_cfg=master_config.data_plane, + ) + + +def _generation_max_seq_len(generation_config) -> int: + """Return the per-backend max sequence length. + + vllm uses vllm_cfg.max_model_len; sglang uses sglang_cfg.context_length; + megatron generation has no dedicated field and routes max_new_tokens + through as max_sequence_length on the inference worker. + """ + backend = generation_config["backend"] + if backend == "vllm": + return generation_config["vllm_cfg"]["max_model_len"] + if backend == "sglang": + return generation_config["sglang_cfg"]["context_length"] + if backend == "megatron": + return generation_config["max_new_tokens"] + raise ValueError(f"Unknown generation backend: {backend!r}") + + +def _clamp_max_num_steps( + master_config: MasterConfig, dataloader: StatefulDataLoader +) -> None: + """Clamp grpo.max_num_steps to max_num_epochs * len(dataloader).""" + grpo_config = master_config.grpo + max_num_epochs = grpo_config.get("max_num_epochs") + if max_num_epochs is None: + return + grpo_config["max_num_steps"] = min( + grpo_config["max_num_steps"], + max_num_epochs * len(dataloader), + ) + + +def _maybe_inject_megatron_train_iters( + master_config: MasterConfig, dataloader: StatefulDataLoader +) -> None: + """Set megatron_cfg.train_iters; must run before _build_trainer.""" + policy_config = master_config.policy + if not policy_config.get("megatron_cfg", {}).get("enabled", False): + return + grpo_config = master_config.grpo + policy_config["megatron_cfg"]["train_iters"] = min( + grpo_config["max_num_steps"], + grpo_config["max_num_epochs"] * len(dataloader), + ) + + +def setup_single_controller( + master_config: MasterConfig, + tokenizer: PreTrainedTokenizerBase, + *, + processor: Optional[AutoProcessor] = None, + partition_id: str = "rollout_data", +) -> SingleControllerBundle: + """Build the full SC bundle driver-side. + + Args: + master_config: SC MasterConfig. + tokenizer: Tokenizer used by the policy. + processor: Optional AutoProcessor for VLM paths. + partition_id: TQ partition the rollout writer + sampler share. + + Returns: + SingleControllerBundle ready to be passed to SingleControllerActor. + """ + dp_cfg = master_config.data_plane + if dp_cfg is None or not dp_cfg.get("enabled", False): + raise ValueError( + "single_controller_utils.setup requires " + "master_config.data_plane.enabled=True. The async-RL " + "SingleController path is built on the TransferQueue data plane." + ) + + data_config = master_config.data + grpo_config = master_config.grpo + generation_config = master_config.policy["generation"] + assert generation_config is not None, ( + "single_controller_utils.setup requires policy.generation in master_config" + ) + + if data_config["use_multiple_dataloader"]: + raise NotImplementedError( + "single_controller_utils does not support " + "data.use_multiple_dataloader=True yet." + ) + + # Token capture: validate the MVP matrix loudly at setup (§ 6, § 10) and + # give capture-enabled vLLM workers a venv that carries nemo_gym (the + # worker hosts Gym's capture core + adapter in-process). + token_capture_cfg = master_config.token_capture + if token_capture_cfg.enabled: + if not _should_use_nemo_gym(master_config): + raise ValueError( + "token_capture.enabled requires the NeMo-Gym rollout path " + "(env.should_use_nemo_gym=true) — the gate lives in Gym's " + "policy model server" + ) + if generation_config["backend"] != "vllm": + raise NotImplementedError( + "token_capture.enabled supports the vllm backend only; got " + f"{generation_config['backend']!r}" + ) + if not generation_config["vllm_cfg"]["async_engine"]: + raise ValueError( + "token_capture.enabled requires " + "policy.generation.vllm_cfg.async_engine=true (the capture " + "host is the worker's in-process HTTP server)" + ) + from nemo_rl.distributed.ray_actor_environment_registry import ( + ACTOR_ENVIRONMENT_REGISTRY, + ) + from nemo_rl.distributed.virtual_cluster import PY_EXECUTABLES + + ACTOR_ENVIRONMENT_REGISTRY[ + "nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker" + ] = PY_EXECUTABLES.VLLM_GYM + + # Fill the derived gate-hosting fields (see TokenCaptureConfig): + # a per-run control-plane bearer token, the base capture dir the + # gate rides on, and LineageIndex capacity sized from the training + # config (finding M: eviction of a live rollout must not happen + # under normal operation). + if token_capture_cfg.control_auth_token is None: + # Deferred import: only needed on the capture path. + import secrets + + token_capture_cfg.control_auth_token = secrets.token_hex(32) + if token_capture_cfg.capture_dir is None: + token_capture_cfg.capture_dir = os.path.abspath( + os.path.join( + master_config.logger.get("log_dir") or "logs", + "gym_token_capture", + ) + ) + if token_capture_cfg.lineage_max_rollouts is None: + group_size = grpo_config["num_generations_per_prompt"] + in_flight = ( + master_config.async_rl.max_buffered_rollouts + + master_config.async_rl.max_inflight_prompts + ) * group_size + token_capture_cfg.lineage_max_rollouts = 2 * in_flight + if token_capture_cfg.lineage_max_tokens is None: + token_capture_cfg.lineage_max_tokens = ( + token_capture_cfg.lineage_max_rollouts + * int(master_config.policy["max_total_sequence_length"]) + ) + + set_seed(grpo_config["seed"]) + + # ========================== + # Setup Dataset & Environments + # ========================== + # TODO: add validate dataset wiring. + use_nemo_gym = _should_use_nemo_gym(master_config) + if use_nemo_gym: + # NeMo-Gym creates the env actor outside setup_response_data; we wire + # it in after generation is up (it needs the OpenAI server URLs). + dataset, _val_dataset = setup_response_data( + tokenizer, data_config, env_configs=None + ) + env_handles: dict[str, EnvironmentInterface] = {} + else: + dataset, _val_dataset, env_handles, _val_env_handles = setup_response_data( + tokenizer, data_config, env_configs=master_config.env + ) + dataloader = StatefulDataLoader( + dataset, + batch_size=grpo_config["num_prompts_per_step"], + shuffle=data_config["shuffle"], + collate_fn=rl_collate_fn, + drop_last=True, + num_workers=data_config["num_workers"], + ) + + _clamp_max_num_steps(master_config, dataloader) + _maybe_inject_megatron_train_iters(master_config, dataloader) + + # ========================== + # Setup Clusters & Workers + # ========================== + train_cluster, inference_cluster = _build_clusters(master_config) + colocated = generation_config["colocated"]["enabled"] + if colocated: + # Colocated: vLLM prefers a clean GPU at load time, so generation + # comes up before the policy. + generation = _build_generation(inference_cluster, master_config) + policy = _build_trainer(train_cluster, master_config, tokenizer, processor) + else: + # Non-colocated: generation + policy run on disjoint GPUs, so + # bring them up in parallel. + with ThreadPoolExecutor(max_workers=2) as executor: + gen_future = executor.submit( + _build_generation, inference_cluster, master_config + ) + policy_future = executor.submit( + _build_trainer, train_cluster, master_config, tokenizer, processor + ) + generation = gen_future.result() + policy = policy_future.result() + + # ========================== + # NeMo-Gym actor (after generation is up so OpenAI URLs are available) + # ========================== + if use_nemo_gym: + if generation_config["backend"] != "vllm": + raise NotImplementedError( + "SC NeMo-Gym integration currently supports the vllm backend " + f"only; got {generation_config['backend']!r}" + ) + env_handles["nemo_gym"] = spinup_nemo_gym_actor( + env_configs=master_config.env, + base_urls=generation.dp_openai_server_base_urls, + model_name=generation_config["model_name"], + # Gate config rides into Gym's policy model server (§ 9.1). + token_capture=( + master_config.token_capture.model_dump() + if master_config.token_capture.enabled + else None + ), + ) + + # ========================== + # Setup Data Plane Client & Weight Sync + # ========================== + # Connect-only DP client; TQPolicy already bootstrapped the controller. + dp_client = build_data_plane_client(dp_cfg, bootstrap=False) + + # Token-capture mode: pre-register both rollout partitions from this + # single driver thread before any producer is live. TQ's controller + # registers unseen field names lazily inside update_production_status + # without a lock, so the first concurrent puts into an unregistered + # partition can race kv_retrieve_meta and kill the controller thread + # (see TQDataPlaneClient.register_partition). + token_capture_cfg = master_config.token_capture + if token_capture_cfg.enabled: + from nemo_rl.data_plane.schema import DP_TRAIN_FIELDS + from nemo_rl.data_plane.tq_token_sink import STAGING_FIELDS + + group_size = grpo_config["num_generations_per_prompt"] + num_rollout_samples = master_config.async_rl.max_buffered_rollouts * group_size + dp_client.register_partition( + partition_id=partition_id, + fields=list(DP_TRAIN_FIELDS), + num_samples=num_rollout_samples, + consumer_tasks=["prev_lp", "ref_lp", "train"], + grpo_group_size=group_size, + ) + dp_client.register_partition( + partition_id=token_capture_cfg.staging_partition, + fields=list(STAGING_FIELDS), + num_samples=num_rollout_samples, + consumer_tasks=["finalize"], + ) + # Host Gym's capture core in every vLLM DP leader (in-worker DP + # client + TQTokenSink + the single install_capture call), and give + # workers the initial weight version to stamp on captured calls. + try: + generation.setup_token_capture(dp_cfg, token_capture_cfg.staging_partition) + except Exception as error: + if "No module named 'nemo_gym'" in str(error): + # Worker venvs are cached by actor class name + # (nemo_rl/utils/venvs.py), so a venv prebuilt before token + # capture predates the nemo_gym extra and is reused as-is. + raise RuntimeError( + "token_capture.enabled requires nemo_gym inside the vLLM " + "worker venv, but the cached worker venv predates it. " + "Rebuild worker venvs (NRL_FORCE_REBUILD_VENVS=true) or " + "delete $NEMO_RL_VENV_DIR/nemo_rl.models.generation.vllm." + "vllm_worker_async.VllmAsyncGenerationWorker and rerun." + ) from error + raise + generation.set_rollout_weight_version(0) + + backend = generation_config["backend"] + weight_synchronizer = create_weight_synchronizer( + policy=policy, + generation=generation, + generation_backend=backend, + colocated=colocated, + train_cluster=train_cluster, + inference_cluster=inference_cluster, + ) + weight_synchronizer.init_communicator() + + # ========================== + # Setup Algorithm + Rollout Wiring + # ========================== + advantage_estimator = _create_advantage_estimator(master_config) + loss_fn: LossFunction = ClippedPGLossFn(master_config.loss_fn) + + pad_id = int(getattr(tokenizer, "pad_token_id", 0) or 0) + tq_buffer = TQReplayBuffer( + dp_client, + partition_id=partition_id, + pad_value_dict={"token_ids": pad_id, "input_ids": pad_id}, + staging_partition_id=( + token_capture_cfg.staging_partition if token_capture_cfg.enabled else None + ), + ) + finalizer = None + if token_capture_cfg.enabled: + from nemo_rl.experience.blackbox_finalizer import BlackboxFinalizer + + finalizer = BlackboxFinalizer( + dp_client, + partition_id=partition_id, + staging_partition=token_capture_cfg.staging_partition, + pad_token_id=pad_id, + mixed_weight_version_policy=token_capture_cfg.mixed_weight_version_policy, + min_valid_fraction_per_group=token_capture_cfg.min_valid_fraction_per_group, + ) + rollout_manager = RolloutManager( + tokenizer=tokenizer, + env_handles=env_handles, + num_generations_per_prompt=grpo_config["num_generations_per_prompt"], + max_seq_len=_generation_max_seq_len(generation_config), + max_rollout_turns=grpo_config.get("max_rollout_turns"), + policy_generation=generation, + generation_config=generation_config, + use_nemo_gym=use_nemo_gym, + tq_buffer=tq_buffer, + finalizer=finalizer, + ) + + return SingleControllerBundle( + gen_handle=generation, + trainer_handle=policy, + env_handles=env_handles, + train_cluster=train_cluster, + inference_cluster=inference_cluster, + dp_client=dp_client, + dataloader=dataloader, + weight_synchronizer=weight_synchronizer, + advantage_estimator=advantage_estimator, + loss_fn=loss_fn, + rollout_manager=rollout_manager, + tq_buffer=tq_buffer, + partition_id=partition_id, + ) diff --git a/nemo_rl/algorithms/single_controller_utils/utils.py b/nemo_rl/algorithms/single_controller_utils/utils.py new file mode 100644 index 00000000000..e202d99c4d1 --- /dev/null +++ b/nemo_rl/algorithms/single_controller_utils/utils.py @@ -0,0 +1,189 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers used by SingleControllerActor.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch +from tensordict import TensorDict + +from nemo_rl.data_plane import KVBatchMeta + +# Reduction rules for all_mb_metrics. Mirror grpo.py / grpo_sync.py. +_MB_METRIC_MIN: frozenset[str] = frozenset( + {"probs_ratio_min", "probs_ratio_clamped_min"} +) +_MB_METRIC_MAX: frozenset[str] = frozenset( + {"probs_ratio_max", "probs_ratio_clamped_max"} +) +_MB_METRIC_MEAN: frozenset[str] = frozenset( + { + "lr", + "wd", + "reward", + "global_valid_seqs", + "global_valid_toks", + "mean_prompt_length", + } +) + + +def aggregate_step_metrics(train_result: dict[str, Any]) -> dict[str, Any]: + """Reduce per-microbatch metric lists into step-level scalars. + + Args: + train_result: Output of TQPolicy.finish_train_step. + + Returns: + Flat dict of step-level scalars ready for logging. + """ + metrics: dict[str, Any] = {} + loss = train_result.get("loss") + if isinstance(loss, torch.Tensor): + metrics["loss"] = loss.detach().mean().item() + elif loss is not None: + metrics["loss"] = float(loss) + grad_norm = train_result.get("grad_norm") + if isinstance(grad_norm, torch.Tensor): + metrics["grad_norm"] = grad_norm.detach().mean().item() + elif grad_norm is not None: + metrics["grad_norm"] = float(grad_norm) + if "total_flops" in train_result: + metrics["total_flops"] = float(train_result["total_flops"]) + if "num_ranks" in train_result: + metrics["num_ranks"] = int(train_result["num_ranks"]) + + # moe/mtp share the same reduction rules as all_mb_metrics in grpo.py. + mb: dict[str, list[Any]] = {} + if "moe_metrics" in train_result: + mb.update({f"moe/{k}": v for k, v in train_result["moe_metrics"].items()}) + if "mtp_metrics" in train_result: + mb.update({f"mtp/{k}": v for k, v in train_result["mtp_metrics"].items()}) + mb.update(train_result.get("all_mb_metrics", {})) + + for k, v in mb.items(): + if k in _MB_METRIC_MIN: + valid = [x for x in v if not np.isinf(x)] + metrics[k] = float(np.min(valid)) if valid else -1.0 + elif k in _MB_METRIC_MAX: + valid = [x for x in v if not np.isinf(x)] + metrics[k] = float(np.max(valid)) if valid else -1.0 + elif k in _MB_METRIC_MEAN: + metrics[k] = float(np.mean(v)) + else: + metrics[k] = float(np.sum(v)) + return metrics + + +def reduce_advantage_pump_metrics( + rewards: list[torch.Tensor], + masked_advantages: list[torch.Tensor], + sequence_lengths: list[int], +) -> dict[str, float]: + """Reduce per-step accumulators from _advantage_pump into step scalars. + + Args: + rewards: One tensor per advantage_pump call; each row a sample reward. + masked_advantages: Token-masked advantages, one tensor per call. + sequence_lengths: All input_lengths trained on this step. + + Returns: + Dict with reward, advantages/{mean,max,min}, total_num_tokens. + """ + out: dict[str, float] = {} + if rewards: + out["reward"] = float(torch.cat([r.flatten() for r in rewards]).mean()) + if masked_advantages: + cat = torch.cat([a.flatten() for a in masked_advantages]) + if cat.numel() > 0: + out["advantages/mean"] = float(cat.mean()) + out["advantages/max"] = float(cat.max()) + out["advantages/min"] = float(cat.min()) + else: + out["advantages/mean"] = 0.0 + out["advantages/max"] = 0.0 + out["advantages/min"] = 0.0 + if sequence_lengths: + out["total_num_tokens"] = float(sum(sequence_lengths)) + return out + + +def tensor_field(data: TensorDict, field_name: str) -> torch.Tensor: + """Read a tensor column from a TensorDict, depadding if nested. + + Args: + data: TensorDict returned by the data plane. + field_name: Column name to fetch. + + Returns: + Dense tensor (nested columns are padded with zeros). + """ + value = data[field_name] + if not isinstance(value, torch.Tensor): + raise TypeError(f"expected tensor field {field_name!r}; got {type(value)}") + if value.is_nested: + return torch.nested.to_padded_tensor(value, padding=0) + return value + + +def squeeze_trailing_unit_dim(value: torch.Tensor) -> torch.Tensor: + """Drop a trailing dim of size 1 if present. + + Args: + value: Input tensor. + + Returns: + Tensor without the trailing unit dim. + """ + if value.dim() >= 2 and value.shape[-1] == 1: + return value.squeeze(-1) + return value + + +def fields_for_put(meta: KVBatchMeta, fields: dict[str, torch.Tensor]) -> TensorDict: + """Pack tensors for DataPlane put, re-nesting jagged rows when needed. + + Args: + meta: Batch meta whose sequence_lengths drive the nesting. + fields: Field name to dense tensor. + + Returns: + TensorDict shaped for dp_client.put_samples. + """ + packed: dict[str, torch.Tensor] = {} + if meta.sequence_lengths is None: + for field_name, value in fields.items(): + packed[field_name] = value.detach().contiguous() + # pyrefly: ignore[bad-argument-type] + return TensorDict(packed, batch_size=[meta.size]) + + lengths = torch.tensor(meta.sequence_lengths, dtype=torch.long) + for field_name, value in fields.items(): + if value.dim() >= 2 and value.shape[1] == int(lengths.max().item()): + rows = [ + value[i, : int(lengths[i].item())].detach().contiguous() + for i in range(meta.size) + ] + packed[field_name] = torch.nested.as_nested_tensor( + rows, + layout=torch.jagged, + ) + else: + packed[field_name] = value.detach().contiguous() + # pyrefly: ignore[bad-argument-type] + return TensorDict(packed, batch_size=[meta.size]) diff --git a/nemo_rl/data_plane/interfaces.py b/nemo_rl/data_plane/interfaces.py index 6bdc5e940cf..41a98f0c0ed 100644 --- a/nemo_rl/data_plane/interfaces.py +++ b/nemo_rl/data_plane/interfaces.py @@ -219,6 +219,31 @@ def concat(self, *others: "KVBatchMeta") -> "KVBatchMeta": sample_ids=sample_ids, sequence_lengths=seq_lens, tags=tags ) + def drop(self, indices: "Sequence[int]") -> "KVBatchMeta | None": + """Complement of :meth:`subset`. Returns ``None`` when all rows are dropped.""" + dropped = set(indices) + keep = [i for i in range(self.size) if i not in dropped] + if not keep: + return None + return self.subset(keep) + + def with_fields(self, field_names: "Sequence[str]") -> "KVBatchMeta": + """Return a copy with ``field_names`` merged into ``fields`` (deduped, order-preserving).""" + merged = list(dict.fromkeys([*(self.fields or []), *field_names])) + return KVBatchMeta( + partition_id=self.partition_id, + task_name=self.task_name, + sample_ids=list(self.sample_ids), + fields=merged, + sequence_lengths=( + list(self.sequence_lengths) + if self.sequence_lengths is not None + else None + ), + extra_info=dict(self.extra_info or {}), + tags=[dict(tag) for tag in self.tags] if self.tags is not None else None, + ) + class DataPlaneClient(ABC): """Stable, swappable data-plane boundary. diff --git a/nemo_rl/data_plane/tq_token_sink.py b/nemo_rl/data_plane/tq_token_sink.py new file mode 100644 index 00000000000..0f83c1e9fc9 --- /dev/null +++ b/nemo_rl/data_plane/tq_token_sink.py @@ -0,0 +1,188 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TransferQueue implementations of NeMo-Gym's token staging protocols. + +``TQTokenSink``/``TQTokenSource`` are NeMo-RL's providers for the +gate-authoritative capture design (docs/design-docs/tq-gym-gate-authoritative.md): +the sink is the worker-side write of one model call's token delta to the +``rollout_staging`` partition — the design's only heavy token hop — and the +source is the finalizer's read-back of those rows by staging key. This module +is the only hot-path file that knows tokens live in TQ; Gym sees opaque +staging keys. + +Each staged row carries three jagged columns (``token_ids_delta``, +``token_mask_delta``, ``generation_logprobs_delta``) plus scalar ``prev_len`` +and ``weight_version`` columns so a fetched row round-trips to a complete +``StagedCallSnapshot`` (parent pointers are lineage state and are rejoined +from the receipt manifest by the finalizer). Masks/logprobs are float32 on +the wire, matching ``compute_staging_digest``'s float32-bit-pattern scheme, +so the finalizer's digest recomputation over fetched values is byte-exact. +""" + +from __future__ import annotations + +from typing import Any, Optional + +import ray +import torch +from tensordict import TensorDict + +from nemo_gym.token_id_capture.staging.records import ( + StagedCallRecord, + StagedCallSnapshot, + StageResult, +) + +STAGING_FIELDS = [ + "token_ids_delta", + "token_mask_delta", + "generation_logprobs_delta", + "prev_len", + "weight_version", +] + + +def _call_dp(dp_client: Any, method_name: str, **kwargs: Any) -> Any: + """Call a DataPlaneClient method on a local client or a Ray actor handle.""" + method = getattr(dp_client, method_name) + remote = getattr(method, "remote", None) + if remote is not None: + return ray.get(remote(**kwargs)) + return method(**kwargs) + + +class TQTokenSink: + """Gym ``StagingSink`` over ``DataPlaneClient.put_samples``. + + ``stage`` is synchronous and returns only after TQ acknowledged the + write, so the capture layer's fail-closed ordering (bytes durable before + the model call is acked) holds by construction. Failures are reported in + the ``StageResult`` — the caller decides whether the rollout poisons or + aborts (``token_capture.on_capture_failure``). + """ + + def __init__(self, dp_client: Any, *, staging_partition: str) -> None: + self._dp_client = dp_client + self._staging_partition = staging_partition + + def stage(self, record: StagedCallRecord) -> StageResult: + key = record.staging_key + try: + fields = TensorDict( + { + "token_ids_delta": torch.tensor( + [record.token_ids_delta], dtype=torch.int64 + ), + "token_mask_delta": torch.tensor( + [record.token_mask_delta], dtype=torch.float32 + ), + "generation_logprobs_delta": torch.tensor( + [record.generation_logprobs_delta], dtype=torch.float32 + ), + "prev_len": torch.tensor([record.prev_len], dtype=torch.int64), + "weight_version": torch.tensor( + [record.weight_version], dtype=torch.int64 + ), + }, + batch_size=[1], + ) + tags = [ + { + "rollout_id": record.rollout_id, + "call_id": record.call_id, + "parent_call_id": record.parent_call_id, + "prev_len": record.prev_len, + "new_len": record.new_len, + "weight_version": record.weight_version, + "digest": record.digest, + "schema_version": record.schema_version, + } + ] + _call_dp( + self._dp_client, + "put_samples", + sample_ids=[key], + partition_id=self._staging_partition, + fields=fields, + tags=tags, + ) + except Exception as error: # noqa: BLE001 — any failure must poison, not crash serving + return StageResult( + ok=False, staging_key=key, error=f"{type(error).__name__}: {error}" + ) + return StageResult(ok=True, staging_key=key) + + def clear(self, staging_keys: list[str]) -> None: + """Drop staged rows (finalizer / eviction cleanup).""" + if not staging_keys: + return + _call_dp( + self._dp_client, + "clear_samples", + sample_ids=list(staging_keys), + partition_id=self._staging_partition, + ) + + +class TQTokenSource: + """Gym ``StagingSource`` over ``DataPlaneClient.get_samples``. + + Rows are fetched one key at a time (deltas are jagged across calls) in + the order requested. A missing or unreadable row raises ``KeyError`` per + the protocol — the finalizer maps that to a placeholder, never a silent + skip. + """ + + def __init__(self, dp_client: Any, *, staging_partition: str) -> None: + self._dp_client = dp_client + self._staging_partition = staging_partition + + def fetch(self, staging_keys: list[str]) -> list[StagedCallSnapshot]: + snapshots: list[StagedCallSnapshot] = [] + for key in staging_keys: + _, _, call_id = key.rpartition("/") + try: + row = _call_dp( + self._dp_client, + "get_samples", + sample_ids=[key], + partition_id=self._staging_partition, + select_fields=STAGING_FIELDS, + ) + snapshots.append(_row_to_snapshot(call_id, row)) + except KeyError: + raise + except Exception as error: # noqa: BLE001 — protocol maps any miss to KeyError + raise KeyError( + f"staged row {key!r} could not be fetched: {error}" + ) from error + return snapshots + + +def _row_to_snapshot(call_id: str, row: Any) -> StagedCallSnapshot: + def _leaf(name: str) -> torch.Tensor: + value = row[name] + tensor = value[0] if value.dim() > 1 or value.numel() > 1 else value + return tensor.reshape(-1) + + prev_len = int(_leaf("prev_len")[0].item()) + weight_version: Optional[int] = int(_leaf("weight_version")[0].item()) + return StagedCallSnapshot( + call_id=call_id, + prev_len=prev_len, + token_ids_delta=[int(t) for t in _leaf("token_ids_delta").tolist()], + token_mask_delta=[float(m) for m in _leaf("token_mask_delta").tolist()], + logprobs_delta=[float(p) for p in _leaf("generation_logprobs_delta").tolist()], + weight_version=weight_version, + ) diff --git a/nemo_rl/data_plane/worker_mixin.py b/nemo_rl/data_plane/worker_mixin.py index 89c0cb622d1..8546336590c 100644 --- a/nemo_rl/data_plane/worker_mixin.py +++ b/nemo_rl/data_plane/worker_mixin.py @@ -516,3 +516,82 @@ def get_reference_policy_logprobs_presharded( tq_field="reference_policy_logprobs", ) del result + + # ── split-API entrypoints (SC async path) ────────────────────────────── + # + # The split path lets SingleController drive forward/backward per + # microbatch (or per pipeline-batch on Megatron) without stepping the + # optimizer until a full logical batch has accumulated. Backend + # methods (``begin_train_step``, ``train_microbatch``, + # ``finish_train_step``, ``abort_train_step``) own the train-step + # state machine; this mixin just gates them on TQ-presharded data. + + @wrap_with_nvtx_name("policy_worker/begin_train_step_presharded") + def begin_train_step_presharded( + self, + step_id: str, + loss_fn: Any, + gbs: Optional[int] = None, + mbs: Optional[int] = None, + ) -> None: + """Open a logical train step. No fetch — pure lifecycle. + + The backend stores ``step_id`` / ``loss_fn`` / ``gbs`` / ``mbs``, + clears gradients, and initialises accumulators for + ``local_valid_seqs`` / ``local_valid_toks`` and any per-microbatch + metrics. Optimizer state is untouched here. + """ + self.begin_train_step( # type: ignore[attr-defined] + step_id=step_id, + loss_fn=loss_fn, + gbs=gbs, + mbs=mbs, + ) + + @wrap_with_nvtx_name("policy_worker/train_microbatch_presharded") + def train_microbatch_presharded( + self, + step_id: str, + meta: "KVBatchMeta", + ) -> dict[str, Any]: + """Per-rank microbatch entrypoint. Fetch → packing prep → forward+backward. + + Gradients accumulate into ``.grad`` across calls; no + ``optimizer.step`` here. Returns per-microbatch metrics (loss, + local_valid_*); the backend folds them into the step accumulator + and the caller may surface them for diagnostics. + """ + data = self._fetch(meta) + data = self._attach_or_repack_pack_metadata(data, meta) + return self.train_microbatch( # type: ignore[attr-defined] + step_id=step_id, + data=data, + ) + + @wrap_with_nvtx_name("policy_worker/finish_train_step_presharded") + def finish_train_step_presharded( + self, + step_id: str, + ) -> dict[str, Any]: + """Close a logical train step. No fetch — pure lifecycle. + + Backend all-reduces accumulated ``local_valid_seqs/toks``, + rescales gradients to the final global normalization, runs grad + clip, steps the optimizer + scheduler, then zeros gradients. + Returns the aggregated step result (``loss``, ``grad_norm``, + ``all_mb_metrics``, …). + """ + return self.finish_train_step(step_id=step_id) # type: ignore[attr-defined] + + @wrap_with_nvtx_name("policy_worker/abort_train_step_presharded") + def abort_train_step_presharded( + self, + step_id: str, + ) -> None: + """Discard partial train-step state without stepping the optimizer. + + Used when SC decides the logical batch will not complete (e.g. + weight-sync triggered mid-step). Backend drops accumulators and + zeros gradients. + """ + self.abort_train_step(step_id=step_id) # type: ignore[attr-defined] diff --git a/nemo_rl/distributed/virtual_cluster.py b/nemo_rl/distributed/virtual_cluster.py index 30dbc049943..2b0feccb7fe 100644 --- a/nemo_rl/distributed/virtual_cluster.py +++ b/nemo_rl/distributed/virtual_cluster.py @@ -73,6 +73,10 @@ class PY_EXECUTABLES: # Use NeMo-Gym dependencies NEMO_GYM = f"uv run --locked --extra nemo_gym --directory {git_root}" + # vLLM worker hosting Gym's token capture (token_capture.enabled): the + # worker imports nemo_gym's dependency-free capture core + vLLM adapter. + VLLM_GYM = f"uv run --locked --extra vllm --extra nemo_gym --directory {git_root}" + # Use NeMo-RL direct dependencies and SGLang. SGLANG = f"uv run --locked --extra sglang --directory {git_root}" diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 611751af362..9aebbec9963 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -11,15 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import os import subprocess from pathlib import Path -from typing import Any, Dict, List, NotRequired, TypedDict +from typing import Any, Dict, List, NotRequired, Optional, TypedDict +import aiohttp import ray import torch +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy from transformers import PreTrainedTokenizerBase +from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env from nemo_rl.distributed.virtual_cluster import ( DEFAULT_GYM_PORT_RANGE_HIGH, DEFAULT_GYM_PORT_RANGE_LOW, @@ -28,6 +32,7 @@ ) from nemo_rl.environments.interfaces import EnvironmentInterface from nemo_rl.utils.timer import Timer +from nemo_rl.utils.venvs import create_local_venv_on_each_node DEFAULT_INVALID_TOOL_CALL_PATTERNS = [ "", @@ -77,6 +82,19 @@ class NemoGymConfig(TypedDict): thinking_tags: NotRequired[ List[str] | None ] # Thinking tags to check for malformed usage + # Gate-authoritative token capture (token_capture.enabled): the dumped + # TokenCaptureConfig. Turns on the gate in Gym's policy model server, + # switches run_rollouts to receipt mode, and adds the register/seal/fail + # control-plane helpers. None/absent = legacy token-echo path. + token_capture: NotRequired[Dict[str, Any] | None] + + +# Gym control-plane server name (the model server hosting the gate) and the +# opaque run-body key rollout ids ride on (Gym's ROLLOUT_ID_KEY_NAME): the +# agent derives the id from the run body and stamps /ng-rollout/ on every +# model call, so the TQ sample id IS the capture key end to end. +_POLICY_SERVER_NAME = "policy_model" +_NG_ROLLOUT_ID_BODY_KEY = "_ng_rollout_id" def _detect_invalid_tool_call_and_malformed_thinking( @@ -232,6 +250,71 @@ def _spinup(self) -> None: "`rollout_max_attempts_to_avoid_lp_nan` must be at least 1" ) + # Gate-authoritative token capture: turn on the gate in the policy + # model server (via the policy_model global-config override block the + # env yamls already use) and disable the legacy token echo. Receipt + # mode is incompatible with re-dispatching a batch under the same + # rollout ids, so the NaN retry must be exactly 1 (create-only + # registration would fail the retry loudly anyway; fail at setup). + token_capture = self.cfg.get("token_capture") or None + self._token_capture_enabled = bool( + token_capture and token_capture.get("enabled") + ) + self._server_client = None + self._control_headers: Dict[str, str] = {} + self._control_timeout_s = 60.0 + if self._token_capture_enabled: + if self.rollout_max_attempts_to_avoid_lp_nan != 1: + raise ValueError( + "token_capture.enabled requires " + "rollout_max_attempts_to_avoid_lp_nan == 1: a NaN retry " + "would re-register create-only rollout ids at the gate" + ) + policy_overrides = ( + initial_global_config_dict.setdefault("policy_model", {}) + .setdefault("responses_api_models", {}) + .setdefault("vllm_model", {}) + ) + policy_overrides["return_token_id_information"] = False + policy_overrides["token_capture_gate"] = { + "enabled": True, + "registration_ttl_s": token_capture["registration_ttl_s"], + # Finding M: the LineageIndex holds each in-flight rollout's + # cumulative tokens; capacity is sized from the training + # config at setup, never left at the eval-sized defaults. + "lineage_max_rollouts": token_capture["lineage_max_rollouts"], + "lineage_max_tokens": token_capture["lineage_max_tokens"], + # Finding S: control routes are bearer-authed, + # default-required; the token is minted per run at setup. + "control_auth_token": token_capture["control_auth_token"], + } + # #2124-c1 workaround: the capture middleware mints model_call_id + # (and sets the capture context the gate keys on) only when the + # base token capture is active, which requires a capture dir. + # The base's own capture_tokens no-ops on the gate path (the + # worker strips token fields before responding), so the dir + # stays essentially empty. + initial_global_config_dict["token_id_capture_enabled"] = True + initial_global_config_dict["token_id_capture_dir"] = token_capture[ + "capture_dir" + ] + # Agents apply the /ng-rollout/ correlation prefix for token + # capture only when they opt in (the global enabled switch alone + # gates the infrastructure, not the prefix). In an RL training + # run every agent must correlate, and the SC cannot enumerate + # agent servers configured via config_paths — so flip the + # run-wide opt-in the Gym branch adds for exactly this case. + initial_global_config_dict["token_id_capture_all_agents"] = True + self._control_headers = { + "Authorization": f"Bearer {token_capture['control_auth_token']}" + } + # S5 chaos finding: Gym's shared request() retries connection + # errors indefinitely; a dead gate must surface as a failed + # dispatch (placeholders + TTL), not a silent stall. + self._control_timeout_s = float( + token_capture.get("control_timeout_s") or 60.0 + ) + self.rh = RunHelper() self.rh.start( global_config_dict_parser_config=GlobalConfigDictParserConfig( @@ -249,6 +332,65 @@ def _spinup(self) -> None: ) self.rch = RolloutCollectionHelper() + # ── gate control plane (token-capture mode) ───────────────────────────── + + def _control_client(self): + """Gym ServerClient resolving servers by name from the head server.""" + if self._server_client is None: + from nemo_gym.server_utils import ServerClient + + self._server_client = ServerClient.load_from_global_config( + self.head_server_config + ) + return self._server_client + + async def _control(self, method: str, path: str, **kwargs: Any) -> dict: + headers = {**kwargs.pop("headers", {}), **self._control_headers} + try: + response = await asyncio.wait_for( + self._control_client().request( + server_name=_POLICY_SERVER_NAME, + url_path=path, + method=method, + headers=headers, + **kwargs, + ), + timeout=self._control_timeout_s, + ) + except asyncio.TimeoutError: + raise RuntimeError( + f"gate control call {method} {path} exceeded " + f"{self._control_timeout_s}s (gate unreachable or stalled)" + ) from None + if response.status != 200: + raise RuntimeError( + f"gate control call {method} {path} failed: " + f"HTTP {response.status} {await response.text()}" + ) + return await response.json() + + async def register_rollouts(self, rollout_ids: list[str]) -> None: + """Create-only registration before dispatch (§ 3.1).""" + for rollout_id in rollout_ids: + await self._control("PUT", f"/ng-control/rollouts/{rollout_id}") + + async def fail_rollouts(self, rollout_ids: list[str], *, reason: str) -> None: + """Best-effort gate cleanup for a cancelled/failed dispatch (§ 7).""" + for rollout_id in rollout_ids: + try: + await self._control( + "POST", + f"/ng-control/rollouts/{rollout_id}/fail", + json={"reason": reason}, + ) + except (RuntimeError, OSError) as error: + # The registration TTL is the backstop when the gate is + # unreachable during teardown. + print(f"fail_rollout({rollout_id}) failed: {error}", flush=True) + + async def gate_metrics(self) -> dict: + return await self._control("GET", "/ng-control/metrics") + async def run_rollouts( self, nemo_gym_examples: list[dict], @@ -257,6 +399,14 @@ async def run_rollouts( ) -> list[dict]: timer = Timer() + if self._token_capture_enabled: + # Receipt mode: register the ids riding each row's run body + # before dispatch; seal at completion. + rollout_ids = [ + example[_NG_ROLLOUT_ID_BODY_KEY] for example in nemo_gym_examples + ] + await self.register_rollouts(rollout_ids) + timer.start("_run_rollouts_total") max_attempts, trial = self.rollout_max_attempts_to_avoid_lp_nan, 0 while trial < max_attempts: @@ -269,12 +419,27 @@ async def run_rollouts( nemo_rl_results = [] for task in nemo_gym_result_iterator: with timer.time(label=f"{timer_prefix}/await_results"): - nemo_gym_row, nemo_gym_result = await task + try: + nemo_gym_row, nemo_gym_result = await task + except aiohttp.ClientResponseError as e: + # aiohttp exceptions carry CIMultiDictProxy headers + # that Ray cannot pickle across the actor boundary, + # masking the real error with a TypeError; re-raise + # as a plain, picklable RuntimeError. + raise RuntimeError( + f"NemoGym rollout HTTP error: {e.status} " + f"{e.message} url={e.request_info.real_url}" + ) from None with timer.time(label=f"{timer_prefix}/postprocess_results"): - nemo_rl_result = self._postprocess_nemo_gym_to_nemo_rl_result( - nemo_gym_result, tokenizer - ) + if self._token_capture_enabled: + nemo_rl_result = await self._postprocess_receipt_mode( + nemo_gym_row, nemo_gym_result + ) + else: + nemo_rl_result = self._postprocess_nemo_gym_to_nemo_rl_result( + nemo_gym_result, tokenizer + ) nemo_rl_rowidxs.append(nemo_gym_row["_rowidx"]) nemo_rl_results.append(nemo_rl_result) @@ -313,6 +478,40 @@ async def run_rollouts( return nemo_rl_results, timing_metrics + async def _postprocess_receipt_mode( + self, nemo_gym_row: dict, nemo_gym_result: dict + ) -> dict: + """Seal the rollout and return a token-free result (§ 9.1). + + The legacy token walk (and its contiguity assert) does not run: the + gate owns lineage now, output items carry no token arrays, and the + canonical row is rebuilt by the finalizer from staged deltas. The + Ray return carries only the receipt (~100 B/call) beside the + agent-level result. + """ + assert isinstance(nemo_gym_result, dict), ( + f"Hit a non-successful response when querying NeMo Gym for rollouts: {nemo_gym_result}" + ) + rollout_id = nemo_gym_row[_NG_ROLLOUT_ID_BODY_KEY] + try: + receipt = await self._control( + "POST", + f"/ng-control/rollouts/{rollout_id}/seal", + json={"reward": float(nemo_gym_result.get("reward") or 0.0)}, + ) + except (RuntimeError, OSError) as error: + # An unsealable rollout finalizes as a placeholder; the gate's + # registration TTL sweeps its state. + print(f"seal({rollout_id}) failed: {error}", flush=True) + receipt = None + return { + "message_log": [], + "input_message_log": [], + "full_result": nemo_gym_result, + "rollout_id": rollout_id, + "receipt": receipt, + } + def _postprocess_nemo_gym_to_nemo_rl_result( self, nemo_gym_result: dict, tokenizer: PreTrainedTokenizerBase ) -> dict: @@ -445,3 +644,62 @@ def setup_nemo_gym_config(config, tokenizer) -> None: # Stop strings or token ids are not supported generation_config["stop_strings"] = None generation_config["stop_token_ids"] = None + + +def spinup_nemo_gym_actor( + env_configs: dict[str, Any], + base_urls: list[Optional[str]], + model_name: str, + token_capture: Optional[dict[str, Any]] = None, +) -> Any: + """Spin up the NeMo-Gym actor against the given generation server URLs. + + When ``env_configs["nemo_gym"]["num_gpu_nodes"] > 0``, the actor is + scheduled with soft NodeAffinity to the current Ray node so its colocated + GPU resources land where the caller expects. + """ + nemo_gym_py_exec = get_actor_python_env("nemo_rl.environments.nemo_gym.NemoGym") + if nemo_gym_py_exec.startswith("uv"): + nemo_gym_py_exec = create_local_venv_on_each_node( + nemo_gym_py_exec, "nemo_rl.environments.nemo_gym.NemoGym" + ) + + nemo_gym_dict = env_configs["nemo_gym"] + # NeMo-RL-side detection knobs are top-level NemoGymConfig fields + # (where the detector reads them), not part of Gym's global config. + invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) + thinking_tags = nemo_gym_dict.pop("thinking_tags", None) + uv_cache_dir = get_nemo_gym_uv_cache_dir() + if uv_cache_dir is not None: + nemo_gym_dict.setdefault("uv_cache_dir", uv_cache_dir) + uv_venv_dir = get_nemo_gym_venv_dir() + if uv_venv_dir is not None: + nemo_gym_dict.setdefault("uv_venv_dir", uv_venv_dir) + + nemo_gym_cfg = NemoGymConfig( + model_name=model_name, + base_urls=base_urls, + invalid_tool_call_patterns=invalid_tool_call_patterns, + thinking_tags=thinking_tags, + initial_global_config_dict=nemo_gym_dict, + token_capture=token_capture, + ) + + nemo_gym_opts: dict[str, Any] = {} + if nemo_gym_dict.get("num_gpu_nodes", 0): + nemo_gym_opts["scheduling_strategy"] = NodeAffinitySchedulingStrategy( + node_id=ray.get_runtime_context().get_node_id(), + soft=True, + ) + nemo_gym_opts["runtime_env"] = { + "py_executable": nemo_gym_py_exec, + "env_vars": { + **os.environ, + "VIRTUAL_ENV": nemo_gym_py_exec, + "UV_PROJECT_ENVIRONMENT": nemo_gym_py_exec, + }, + } + + actor = NemoGym.options(**nemo_gym_opts).remote(nemo_gym_cfg) + ray.get(actor._spinup.remote()) + return actor diff --git a/nemo_rl/experience/blackbox_finalizer.py b/nemo_rl/experience/blackbox_finalizer.py new file mode 100644 index 00000000000..b59d96c73a2 --- /dev/null +++ b/nemo_rl/experience/blackbox_finalizer.py @@ -0,0 +1,374 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Blackbox finalization: token-free receipts + staged deltas -> canonical rows. + +Orchestration only (docs/design-docs/tq-gym-gate-authoritative.md § 5, § 9.1): +per rollout, fetch the staged rows the receipt manifest names through the +``TokenSource``, re-verify them (digest recomputation over fetched values, +shape/mask/finite-logprob checks, length chaining, weight-version tag +equality), then delegate semantics to Gym's pure ``linearize`` +(``main_chain_only`` + ``terminal_hint``). Any rejection becomes a masked +placeholder row — the group always publishes exactly N rows so GRPO group +shape survives; validity folds into ``sample_mask`` (no new train field) and +placeholders copy ``prompt_ids_for_adv`` from a valid sibling so per-prompt +baselines stay well-formed. + +The finalizer is the only reader of the staging partition and clears a +group's staged rows after its canonical rows are durably published. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Optional + +import torch + +from nemo_rl.data_plane import KVBatchMeta +from nemo_rl.data_plane.tq_token_sink import TQTokenSink, TQTokenSource +from nemo_rl.experience.payload import pack_payload +from nemo_rl.experience.row_dump import maybe_dump_train_rows + + +@dataclass(frozen=True) +class FinalizedRollout: + """One rollout's canonical row, or its rejection.""" + + rollout_id: str + valid: bool + rejection_reason: Optional[str] + token_ids: list[int] + token_mask: list[float] + logprobs: list[float] + prompt_len: int + reward: float + staging_keys: list[str] + min_wv: Optional[int] = None + max_wv: Optional[int] = None + + +@dataclass +class FinalizedGroup: + """What ``finalize_group`` hands back for ``commit_finalized``.""" + + meta: Optional[KVBatchMeta] + group_min_wv: int + group_max_wv: int + staging_keys: list[str] + metrics: dict[str, float] = field(default_factory=dict) + # True when min_valid_fraction_per_group rejected the whole group; the + # caller aborts the slot instead of committing it. + dropped: bool = False + + +class BlackboxFinalizer: + """Receipts -> verified rows -> N-row publish, off the generation hot path.""" + + def __init__( + self, + dp_client: Any, + *, + partition_id: str, + staging_partition: str, + pad_token_id: int, + mixed_weight_version_policy: str, + min_valid_fraction_per_group: Optional[float], + ) -> None: + self._dp_client = dp_client + self._partition_id = partition_id + self._pad_token_id = int(pad_token_id) + self._mixed_weight_version_policy = mixed_weight_version_policy + self._min_valid_fraction = min_valid_fraction_per_group + self._source = TQTokenSource(dp_client, staging_partition=staging_partition) + # The sink's clear() is the staging-partition delete; no staging + # writes happen here. + self._staging = TQTokenSink(dp_client, staging_partition=staging_partition) + + # ── per rollout ───────────────────────────────────────────────────────── + + def finalize_rollout( + self, rollout_id: str, receipt: Optional[dict[str, Any]], *, reward: float + ) -> FinalizedRollout: + """Verify one receipt against its staged rows and linearize the main chain. + + Never raises for rollout-level problems: every rejection returns an + invalid row whose reason feeds the metrics; the group publisher + substitutes a placeholder. + """ + # Deferred: nemo_gym is an optional extra absent in non-gym runs. + from nemo_gym.token_id_capture.staging.digest import compute_staging_digest + from nemo_gym.token_id_capture.staging.rebuild import RebuildError, linearize + from nemo_gym.token_id_capture.staging.records import RolloutReceipt + + def rejected(reason: str, staging_keys: list[str]) -> FinalizedRollout: + return FinalizedRollout( + rollout_id=rollout_id, + valid=False, + rejection_reason=reason, + token_ids=[], + token_mask=[], + logprobs=[], + prompt_len=0, + reward=reward, + staging_keys=staging_keys, + ) + + if receipt is None: + return rejected("missing_receipt", []) + try: + parsed = RolloutReceipt.model_validate(receipt) + except ValueError as error: + return rejected(f"invalid_receipt:{error}", []) + staging_keys = [record.staging_key for record in parsed.manifest] + if parsed.rollout_id != rollout_id: + return rejected(f"identity_mismatch:{parsed.rollout_id}", staging_keys) + if parsed.failure_reason is not None: + return rejected(f"rollout_failed:{parsed.failure_reason}", staging_keys) + if parsed.capture_poisoned: + return rejected("capture_poisoned", staging_keys) + if not parsed.manifest: + return rejected("empty_manifest", staging_keys) + + try: + snapshots = self._source.fetch(staging_keys) + except KeyError as error: + return rejected(f"missing_staging_row:{error}", staging_keys) + + for record, snapshot in zip(parsed.manifest, snapshots): + if not ( + len(snapshot.token_ids_delta) + == len(snapshot.token_mask_delta) + == len(snapshot.logprobs_delta) + ): + return rejected(f"misaligned_delta:{record.call_id}", staging_keys) + if any(m not in (0.0, 1.0) for m in snapshot.token_mask_delta): + return rejected(f"invalid_token_mask:{record.call_id}", staging_keys) + if any(not math.isfinite(p) for p in snapshot.logprobs_delta): + return rejected(f"non_finite_logprob:{record.call_id}", staging_keys) + if record.delta_len != len(snapshot.token_ids_delta) or ( + snapshot.prev_len + record.delta_len != record.cum_len + ): + return rejected(f"length_mismatch:{record.call_id}", staging_keys) + if snapshot.weight_version != record.weight_version: + return rejected( + f"weight_version_mismatch:{record.call_id}", staging_keys + ) + digest = compute_staging_digest( + rollout_id=rollout_id, + call_id=record.call_id, + prev_len=snapshot.prev_len, + token_ids_delta=snapshot.token_ids_delta, + token_mask_delta=snapshot.token_mask_delta, + logprobs_delta=snapshot.logprobs_delta, + ) + if digest != record.digest: + return rejected(f"digest_mismatch:{record.call_id}", staging_keys) + + # Parent pointers are lineage state, not storage state: rejoin from + # the manifest before rebuilding (the storage rows carry them too, + # but the receipt is authoritative). + rejoined = [ + snapshot.model_copy(update={"parent_call_id": record.parent_call_id}) + for snapshot, record in zip(snapshots, parsed.manifest) + ] + try: + row = linearize( + rollout_id, + rejoined, + parsed.manifest, + terminal_hint=parsed.terminal_call_id, + ) + except (RebuildError, NotImplementedError) as error: + return rejected(f"rebuild_failed:{error}", staging_keys) + + weight_versions = [record.weight_version for record in parsed.manifest] + min_wv, max_wv = min(weight_versions), max(weight_versions) + if self._mixed_weight_version_policy == "reject" and min_wv != max_wv: + return rejected(f"mixed_weight_versions:{min_wv}..{max_wv}", staging_keys) + + return FinalizedRollout( + rollout_id=rollout_id, + valid=True, + rejection_reason=None, + token_ids=row.token_ids, + token_mask=row.token_mask, + logprobs=row.logprobs, + prompt_len=row.prompt_len, + reward=reward, + staging_keys=staging_keys, + min_wv=min_wv, + max_wv=max_wv, + ) + + # ── per group ─────────────────────────────────────────────────────────── + + def finalize_group( + self, + group_id: str, + rollout_ids: list[str], + receipts: list[Optional[dict[str, Any]]], + rewards: list[float], + *, + fallback_weight_version: int, + ) -> FinalizedGroup: + """Publish exactly N canonical rows for one prompt group. + + Blocking (TQ round trips); run via ``asyncio.to_thread`` from the + dispatch task. ``fallback_weight_version`` stamps a group none of + whose rollouts produced a valid row (placeholder-only groups still + need a staleness tag). + """ + assert len(rollout_ids) == len(receipts) == len(rewards), ( + "rollout_ids, receipts, and rewards must be parallel" + ) + rows = [ + self.finalize_rollout(rollout_id, receipt, reward=reward) + for rollout_id, receipt, reward in zip(rollout_ids, receipts, rewards) + ] + valid_rows = [row for row in rows if row.valid] + staging_keys = [key for row in rows for key in row.staging_keys] + metrics = { + "finalize/invalid_row_rate": 1.0 - len(valid_rows) / len(rows), + "finalize/calls_per_rollout": ( + sum(len(row.staging_keys) for row in rows) / len(rows) + ), + } + for row in rows: + if not row.valid: + print( + f" finalize: rollout {row.rollout_id} rejected " + f"({row.rejection_reason}) — placeholder", + flush=True, + ) + + group_min_wv = min( + (r.min_wv for r in valid_rows), default=fallback_weight_version + ) + group_max_wv = max( + (r.max_wv for r in valid_rows), default=fallback_weight_version + ) + + valid_fraction = len(valid_rows) / len(rows) + if ( + self._min_valid_fraction is not None + and valid_fraction < self._min_valid_fraction + ): + self._clear_staging(staging_keys) + metrics["finalize/group_dropped"] = 1.0 + return FinalizedGroup( + meta=None, + group_min_wv=group_min_wv, + group_max_wv=group_max_wv, + staging_keys=[], + metrics=metrics, + dropped=True, + ) + + # Placeholders borrow a valid sibling's prompt ids so per-prompt + # baselines group correctly; an all-placeholder group uses a single + # pad token (its rows all carry sample_mask 0 and never train). + sibling_prompt = ( + valid_rows[0].token_ids[: valid_rows[0].prompt_len] if valid_rows else [] + ) or [self._pad_token_id] + + n = len(rows) + seq_lens = [max(1, len(row.token_ids)) for row in rows] + max_len = max(seq_lens) + input_ids = torch.full((n, max_len), self._pad_token_id, dtype=torch.int64) + token_mask = torch.zeros((n, max_len), dtype=torch.float32) + logprobs = torch.zeros((n, max_len), dtype=torch.float32) + prompt_ids_for_adv = torch.tensor([sibling_prompt] * n, dtype=torch.int64) + sample_mask = torch.zeros(n, dtype=torch.float32) + lengths = torch.tensor(seq_lens, dtype=torch.long) + rewards_t = torch.tensor([row.reward for row in rows], dtype=torch.float32) + for i, row in enumerate(rows): + if not row.valid: + continue + length = len(row.token_ids) + input_ids[i, :length] = torch.tensor(row.token_ids, dtype=torch.int64) + token_mask[i, :length] = torch.tensor(row.token_mask, dtype=torch.float32) + logprobs[i, :length] = torch.tensor(row.logprobs, dtype=torch.float32) + sample_mask[i] = 1.0 + + train_batch = { + "input_ids": input_ids, + "input_lengths": lengths, + "generation_logprobs": logprobs, + "token_mask": token_mask, + "sample_mask": sample_mask, + "prompt_ids_for_adv": prompt_ids_for_adv, + "total_reward": rewards_t, + } + sample_ids, fields, tags = pack_payload( + train_batch, weight_version=group_min_wv, group_id=group_id + ) + maybe_dump_train_rows( + source="finalizer", + group_id=group_id, + sample_ids=list(sample_ids), + train_batch=train_batch, + weight_version=group_min_wv, + ) + assert sample_ids == rollout_ids, ( + "canonical sample ids must equal the gate-registered rollout ids: " + f"{sample_ids} != {rollout_ids}" + ) + self._call_dp( + "put_samples", + sample_ids=sample_ids, + partition_id=self._partition_id, + fields=fields, + tags=tags, + ) + self._clear_staging(staging_keys) + + meta = KVBatchMeta( + partition_id=self._partition_id, + task_name="train", + sample_ids=list(sample_ids), + fields=list(fields.keys()), + sequence_lengths=[int(s) for s in lengths.tolist()], + tags=[dict(t) for t in tags], + ) + return FinalizedGroup( + meta=meta, + group_min_wv=group_min_wv, + group_max_wv=group_max_wv, + # Already cleared; nothing left for eviction to clear. + staging_keys=[], + metrics=metrics, + ) + + # ── internals ─────────────────────────────────────────────────────────── + + def _clear_staging(self, staging_keys: list[str]) -> None: + if not staging_keys: + return + try: + self._staging.clear(staging_keys) + except Exception as error: # noqa: BLE001 — cleanup must not fail the group; TTL sweeps leftovers + print( + f" finalize: staging clear failed ({error}); TTL will sweep", + flush=True, + ) + + def _call_dp(self, method_name: str, **kwargs: Any) -> Any: + import ray + + method = getattr(self._dp_client, method_name) + remote = getattr(method, "remote", None) + if remote is not None: + return ray.get(remote(**kwargs)) + return method(**kwargs) diff --git a/nemo_rl/experience/payload.py b/nemo_rl/experience/payload.py new file mode 100644 index 00000000000..bdf95ab290f --- /dev/null +++ b/nemo_rl/experience/payload.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Producer-side payload helpers for the async-RL TQ path.""" + +from collections.abc import Mapping +from typing import Any + +import numpy as np +import torch +from tensordict import TensorDict + +from nemo_rl.data_plane.codec import pack_jagged_fields +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.experience.interfaces import PromptGroupRecord + + +def record_to_train_batch( + record: PromptGroupRecord, + *, + pad_value_dict: Mapping[str, int], +) -> BatchedDataDict[Any]: + """Convert one prompt group's record into a packed BatchedDataDict of N rows. + + Args: + record: Rollout's PromptGroupRecord with N completions to flatten into rows. + pad_value_dict: Field-name → pad value used by batched_message_log_to_flat_message. + + Returns: + BatchedDataDict with input_ids, input_lengths, generation_logprobs, token_mask, + sample_mask, prompt_ids_for_adv, and total_reward. + """ + # Lazy imports: grpo and llm_message_utils transitively pull + # experience.rollouts, so importing at module top risks a cycle. + from nemo_rl.algorithms.grpo import ( + add_grpo_token_loss_masks_and_generation_logprobs, + extract_initial_prompt_messages, + ) + from nemo_rl.data.llm_message_utils import batched_message_log_to_flat_message + + completions = record.completions + n = len(completions) + assert n > 0, "PromptGroupRecord has no completions" + + message_logs = [c.message_log for c in completions] + prompt_token_count = sum(len(m["token_ids"]) for m in record.prompt) + prompt_lengths = torch.full((n,), prompt_token_count, dtype=torch.long) + + prompt_message_logs = extract_initial_prompt_messages(message_logs, prompt_lengths) + prompt_flat, _ = batched_message_log_to_flat_message( + prompt_message_logs, + pad_value_dict=dict(pad_value_dict), # type: ignore + ) + + add_grpo_token_loss_masks_and_generation_logprobs(message_logs) + flat, input_lengths = batched_message_log_to_flat_message( + message_logs, # type: ignore + pad_value_dict=dict(pad_value_dict), # type: ignore + ) + + total_reward = torch.tensor( + [float(c.reward) for c in completions], dtype=torch.float32 + ) + sample_mask = torch.ones(n, dtype=torch.float32) + + return BatchedDataDict[Any]( + { + "input_ids": flat["token_ids"], + "input_lengths": input_lengths, + "generation_logprobs": flat["generation_logprobs"], + "token_mask": flat["token_loss_mask"], + "sample_mask": sample_mask, + "prompt_ids_for_adv": prompt_flat["token_ids"], + "total_reward": total_reward, + } + ) + + +def pack_payload( + train_batch: Mapping[str, Any], + *, + weight_version: int, + group_id: str, +) -> tuple[list[str], TensorDict, list[dict[str, Any]]]: + """Pack a producer batch into (sample_ids, fields, tags) for put_samples. + + Args: + train_batch: Mapping with at least input_lengths plus the tensor/object fields to send. + weight_version: Trainer weight version stamped on every row's tag. + group_id: Per-group identifier used as the sample_id prefix; the caller owns uniqueness. + + Returns: + sample_ids of the form {group_id}_g{i}, a jagged-packed TensorDict, and per-row tags. + """ + lengths = train_batch["input_lengths"] + n = int(lengths.shape[0]) + tensor_fields: dict[str, torch.Tensor | np.ndarray] = { + k: v + for k, v in train_batch.items() + if isinstance(v, torch.Tensor) + or (isinstance(v, np.ndarray) and v.dtype == object) + } + fields_td = pack_jagged_fields(tensor_fields, lengths=lengths) + sample_ids = [f"{group_id}_g{i}" for i in range(n)] + tags = [{"weight_version": weight_version} for _ in range(n)] + return sample_ids, fields_td, tags diff --git a/nemo_rl/experience/rollout_manager.py b/nemo_rl/experience/rollout_manager.py index 4b6b5b9a856..b141b60a905 100644 --- a/nemo_rl/experience/rollout_manager.py +++ b/nemo_rl/experience/rollout_manager.py @@ -15,13 +15,15 @@ import asyncio import copy import json +import uuid from typing import Any, Optional import torch from transformers import PreTrainedTokenizerBase from wandb import Table -from nemo_rl.data.interfaces import DatumSpec +from nemo_rl.algorithms.async_utils.replay_buffer import TQReplayBuffer +from nemo_rl.data.interfaces import DatumSpec, LLMMessageLogType from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.environments.interfaces import EnvironmentInterface from nemo_rl.experience.interfaces import Completion, PromptGroupRecord @@ -50,7 +52,7 @@ class AsyncRolloutImpl: def __init__( self, tokenizer: TokenizerType, - task_to_env: dict[str, EnvironmentInterface], + env_handles: dict[str, EnvironmentInterface], num_generations_per_prompt: int, max_seq_len: int, policy_generation: GenerationInterface, @@ -58,21 +60,27 @@ def __init__( **kwargs: Any, ) -> None: self._tokenizer = tokenizer - self._task_to_env = task_to_env + self._env_handles = env_handles self._num_generations_per_prompt = num_generations_per_prompt self._max_seq_len = max_seq_len self._max_rollout_turns = max_rollout_turns self._policy_generation = policy_generation - async def run_rollout(self, input_sample: DatumSpec) -> PromptGroupRecord: + async def run_rollout( + self, input_sample: DatumSpec, *, rollout_ids: Optional[list[str]] = None + ) -> PromptGroupRecord: """Run num_generations_per_prompt rollouts for one prompt. Args: input_sample: A single prompt (one DatumSpec entry). + rollout_ids: Unsupported here — token capture is NeMo-Gym only. Returns: PromptGroupRecord with num_generations_per_prompt completions. """ + assert rollout_ids is None, ( + "token capture (rollout_ids) is only supported on the NeMo-Gym path" + ) timer = Timer() timer_prefix = "timing/rollout" timer.start(f"{timer_prefix}/total") @@ -191,7 +199,7 @@ async def _run_single_rollout( # step. In this case, need to wrap with asyncio.to_thread to make # this function yieldable. env_output = await asyncio.to_thread( - calculate_rewards, sample_batch, self._task_to_env + calculate_rewards, sample_batch, self._env_handles ) # Update reward and termination statistics @@ -387,15 +395,15 @@ class AsyncNemoGymRolloutImpl: def __init__( self, tokenizer: TokenizerType, - task_to_env: dict[str, EnvironmentInterface], + env_handles: dict[str, EnvironmentInterface], num_generations_per_prompt: int, max_seq_len: int, generation_config: GenerationConfig, - max_rollout_turns: Optional[int] = None, + max_rollout_turns: int, **kwargs: Any, ) -> None: self._tokenizer = tokenizer - self._task_to_env = task_to_env + self._env_handles = env_handles self._num_generations_per_prompt = num_generations_per_prompt self._max_seq_len = max_seq_len self._max_rollout_turns = max_rollout_turns @@ -403,11 +411,17 @@ def __init__( self._validate_init_params() - async def run_rollout(self, input_sample: DatumSpec) -> PromptGroupRecord: + async def run_rollout( + self, input_sample: DatumSpec, *, rollout_ids: Optional[list[str]] = None + ) -> PromptGroupRecord: """Run num_generations_per_prompt rollouts for one prompt. Args: input_sample: A single prompt (one DatumSpec entry). + rollout_ids: Token-capture mode: gate-registered rollout ids, one + per generation, injected into each row's + ``responses_create_params.metadata`` (the zero-agent-change + side-channel carrier). Returns: PromptGroupRecord with num_generations_per_prompt completions. @@ -416,8 +430,8 @@ async def run_rollout(self, input_sample: DatumSpec) -> PromptGroupRecord: timer_prefix = "timing/rollout" timer.start(f"{timer_prefix}/total") - rollout_inputs = self._build_inputs(input_sample) - completions, rollout_metrics = await self._run_rollouts( + rollout_inputs = self._build_inputs(input_sample, rollout_ids=rollout_ids) + completions, prompt_message_log, rollout_metrics = await self._run_rollouts( rollout_inputs, timer, timer_prefix ) @@ -426,7 +440,7 @@ async def run_rollout(self, input_sample: DatumSpec) -> PromptGroupRecord: return PromptGroupRecord( prompt_idx=input_sample["idx"], - prompt=input_sample["message_log"], + prompt=prompt_message_log, extra_env_info=input_sample["extra_env_info"], metadata={"task_name": "nemo_gym"}, completions=completions, @@ -442,11 +456,14 @@ def _validate_init_params(self) -> None: ) # Validate max_rollout_turns. - assert self._max_rollout_turns is None, ( - "`max_rollout_turns` is not supported in NeMo-Gym path!" + assert self._max_rollout_turns == 1, ( + "`max_rollout_turns` is not supported in NeMo-Gym path! " + "Please set `max_rollout_turns` to 1." ) - def _build_inputs(self, input_sample: DatumSpec) -> list[dict]: + def _build_inputs( + self, input_sample: DatumSpec, *, rollout_ids: Optional[list[str]] = None + ) -> list[dict]: """Build N row dicts from input_sample, applying generation config params.""" # Build a template row from the input_sample's extra_env_info, applying generation params. template_row: dict = copy.deepcopy(input_sample["extra_env_info"]) # type: ignore @@ -467,24 +484,36 @@ def _build_inputs(self, input_sample: DatumSpec) -> list[dict]: ) # Build N rows with distinct rowidxs so run_rollouts can sort them correctly. + if rollout_ids is not None: + assert len(rollout_ids) == self._num_generations_per_prompt, ( + "token-capture rollout ids must be one per generation" + ) rows = [] for i in range(self._num_generations_per_prompt): row = copy.deepcopy(template_row) row["_rowidx"] = i + if rollout_ids is not None: + # Opaque run-body carrier (Gym's _ng_rollout_id key): the agent + # derives the id from the run body and stamps /ng-rollout/ + # on every model call, so the TQ sample id IS the capture key. + row["_ng_rollout_id"] = rollout_ids[i] rows.append(row) return rows async def _run_rollouts( self, inputs: list[dict], timer: Timer, timer_prefix: str - ) -> tuple[list[Completion], dict[str, Any]]: - """Dispatch rows to NeMo-Gym and return completions + metrics.""" - nemo_gym_env = self._task_to_env["nemo_gym"] + ) -> tuple[list[Completion], LLMMessageLogType, dict[str, Any]]: + """Dispatch rows to NeMo-Gym; return completions, prompt, and metrics.""" + nemo_gym_env = self._env_handles["nemo_gym"] # Run generation. with timer.time(f"{timer_prefix}/run_rollouts"): results, env_timing_metrics = await nemo_gym_env.run_rollouts.remote( inputs, self._tokenizer, timer_prefix ) + # All N rollouts share the same input prompt; tensorize one copy. + prompt_message_log = results[0]["input_message_log"] + _tensorize_by_key(prompt_message_log, "token_ids") # Convert results to completions. completions = [self._result_to_completion(r) for r in results] @@ -496,12 +525,26 @@ async def _run_rollouts( rollout_metrics.update(env_timing_metrics) - return completions, rollout_metrics + return completions, prompt_message_log, rollout_metrics def _result_to_completion(self, result: dict) -> Completion: """Convert one run_rollouts result dict into a Completion.""" + if "receipt" in result: + # Receipt mode (token capture): the result is token-free — the + # message_log is empty and the canonical row is rebuilt by the + # finalizer from staged deltas. The receipt and rollout id ride + # env_extras for the finalize step. + env_extras = dict(result["full_result"]) + env_extras["ng_receipt"] = result["receipt"] + env_extras["ng_rollout_id"] = result["rollout_id"] + return Completion( + message_log=result["message_log"], + env_extras=env_extras, + truncated=False, + reward=float(result["full_result"]["reward"]), + ) + # Tensorize token fields. - _tensorize_by_key(result["input_message_log"], "token_ids") _tensorize_by_key(result["message_log"], "token_ids") _tensorize_by_key( [m for m in result["message_log"] if m["role"] == "assistant"], @@ -528,17 +571,39 @@ def _compute_rollout_metrics( """Aggregate per-sample and per-agent metrics.""" # Prepare lists of values for each metric. total_reward = [c.reward for c in completions] - turn_count = [ - sum(1 for m in c.message_log if m["role"] == "user") for c in completions - ] - # token metrics - total_tokens = [ - sum(len(m["token_ids"]) for m in c.message_log) for c in completions - ] - assistant_tokens = [ - sum(len(m["token_ids"]) for m in c.message_log if m["role"] == "assistant") - for c in completions - ] + receipt_mode = bool(completions) and "ng_receipt" in completions[0].env_extras + if receipt_mode: + # Token-free receipts: token accounting comes from the manifest + # (cum_len of the deepest chain; delta sums as the generation + # proxy) instead of a message_log walk. + manifests = [ + ((c.env_extras.get("ng_receipt") or {}).get("manifest") or []) + for c in completions + ] + turn_count = [len(m) for m in manifests] + total_tokens = [ + max((entry["cum_len"] for entry in m), default=0) for m in manifests + ] + assistant_tokens = [ + sum(entry["delta_len"] for entry in m) for m in manifests + ] + else: + turn_count = [ + sum(1 for m in c.message_log if m["role"] == "user") + for c in completions + ] + # token metrics + total_tokens = [ + sum(len(m["token_ids"]) for m in c.message_log) for c in completions + ] + assistant_tokens = [ + sum( + len(m["token_ids"]) + for m in c.message_log + if m["role"] == "assistant" + ) + for c in completions + ] # truncated metrics truncated = [c.truncated for c in completions] @@ -556,8 +621,12 @@ def _compute_rollout_metrics( "truncation_rate": sum(truncated) / n, } - # Agent-level metrics. - agent_extras = [c.env_extras for c in completions] + # Agent-level metrics. Receipts are lineage records, not agent + # results — keep them (and their manifests) out of the logged table. + agent_extras = [ + {k: v for k, v in c.env_extras.items() if k not in ("ng_receipt",)} + for c in completions + ] for key in agent_extras[0].keys(): values = [ float(r[key]) # type: ignore @@ -581,22 +650,28 @@ def _compute_rollout_metrics( class RolloutManager: - """Factory that routes to AsyncRolloutImpl (native async) or AsyncNemoGymRolloutImpl (NeMo-Gym).""" + """Routes to AsyncRolloutImpl (native async) or AsyncNemoGymRolloutImpl (NeMo-Gym), and pushes results to a TQReplayBuffer.""" def __init__( self, tokenizer: TokenizerType, - task_to_env: dict[str, EnvironmentInterface], + env_handles: dict[str, EnvironmentInterface], num_generations_per_prompt: int, max_seq_len: int, max_rollout_turns: Optional[int] = None, policy_generation: Optional[GenerationInterface] = None, generation_config: Optional[GenerationConfig] = None, use_nemo_gym: bool = False, + tq_buffer: Optional[TQReplayBuffer] = None, + finalizer: Optional[Any] = None, ) -> None: assert num_generations_per_prompt >= 1, ( "num_generations_per_prompt must be >= 1" ) + if finalizer is not None: + assert use_nemo_gym, ( + "token capture (finalizer) is only supported on the NeMo-Gym path" + ) if not use_nemo_gym: rollout_cls = AsyncRolloutImpl @@ -613,13 +688,146 @@ def __init__( self._impl: AsyncRolloutImpl | AsyncNemoGymRolloutImpl = rollout_cls( tokenizer=tokenizer, - task_to_env=task_to_env, + env_handles=env_handles, num_generations_per_prompt=num_generations_per_prompt, max_seq_len=max_seq_len, max_rollout_turns=max_rollout_turns, # type: ignore policy_generation=policy_generation, # type: ignore generation_config=generation_config, ) + self._tokenizer = tokenizer + self._num_generations_per_prompt = num_generations_per_prompt + self._tq_buffer = tq_buffer + self._finalizer = finalizer + self._env_handles = env_handles + self._weight_version: int = 0 + + def set_weight_version(self, version: int) -> None: + """Set the weight_version used for rollout tags. - async def run_rollout(self, input_sample: DatumSpec) -> PromptGroupRecord: - return await self._impl.run_rollout(input_sample) + Args: + version: Trainer weight version to stamp on future rollout tags. + """ + self._weight_version = int(version) + + async def gate_metrics(self) -> Optional[dict[str, int]]: + """Fetch the capture gate's § 8 counters, or None off the capture path. + + Returns: + Cumulative gate counters (token_in, fallback_*, capture_failed, + registered/sealed/failed/expired) from ``/ng-control/metrics``, + or None when no NemoGym env handle is wired. + """ + env = self._env_handles.get("nemo_gym") if self._env_handles else None + if env is None: + return None + return await env.gate_metrics.remote() + + async def run_rollout( + self, input_sample: DatumSpec, *, rollout_ids: Optional[list[str]] = None + ) -> PromptGroupRecord: + if rollout_ids is None: + # Legacy path: keep the impl call signature byte-identical. + return await self._impl.run_rollout(input_sample) + return await self._impl.run_rollout(input_sample, rollout_ids=rollout_ids) + + async def generate_and_push( + self, input_sample: DatumSpec, *, target_step: Optional[int] = None + ) -> None: + """Reserve a buffer slot, run one prompt's rollout, then commit the slot. + + Args: + input_sample: A single prompt (one DatumSpec entry). + target_step: Training step this rollout targets; stamped on the buffer slot for StalenessSampler.force_in_order. + """ + assert self._tq_buffer is not None, ( + "generate_and_push requires tq_buffer to be set at __init__" + ) + if self._finalizer is not None: + await self._generate_and_finalize(input_sample, target_step=target_step) + return + start_version = self._weight_version + group_id = self._tq_buffer.reserve( + weight_version=start_version, target_step=target_step + ) + + try: + record = await self.run_rollout(input_sample) + end_version = self._weight_version + + await self._tq_buffer.commit( + group_id, + record, + start_weight_version=start_version, + end_weight_version=end_version, + ) + except BaseException: + # A slot that will never commit must not linger as a phantom + # unready entry until staleness eviction — drop it now so buffer + # occupancy and _buffer_capacity stay in step (the dispatch task + # releases the capacity permit on the same exception). + self._tq_buffer.abort(group_id) + raise + + async def _generate_and_finalize( + self, input_sample: DatumSpec, *, target_step: Optional[int] = None + ) -> None: + """Token-capture dispatch: receipts in, canonical rows via the finalizer. + + Mints the group's rollout ids up front — sample ids and gate-registered + rollout ids are the same strings (``{group_id}_g{i}``) — reserves the + slot with them so cleanup can name what it owns before a receipt + exists, and commits via ``commit_finalized`` with the group's + min/max call weight versions. + """ + start_version = self._weight_version + group_id = str(uuid.uuid4()) + rollout_ids = [ + f"{group_id}_g{i}" for i in range(self._num_generations_per_prompt) + ] + self._tq_buffer.reserve( + weight_version=start_version, + target_step=target_step, + group_id=group_id, + rollout_ids=rollout_ids, + ) + try: + record = await self.run_rollout(input_sample, rollout_ids=rollout_ids) + receipts = [c.env_extras.get("ng_receipt") for c in record.completions] + rewards = [float(c.reward) for c in record.completions] + finalized = await asyncio.to_thread( + self._finalizer.finalize_group, + group_id, + rollout_ids, + receipts, + rewards, + fallback_weight_version=start_version, + ) + record.rollout_metrics.update(finalized.metrics) + if finalized.dropped: + # min_valid_fraction_per_group rejected the group: nothing + # was published, so drop the slot like a failed dispatch. + self._tq_buffer.abort(group_id) + raise RuntimeError( + f"token capture: group {group_id} dropped " + "(min_valid_fraction_per_group)" + ) + await self._tq_buffer.commit_finalized( + group_id, + finalized.meta, + finalized.group_min_wv, + finalized.group_max_wv, + staging_keys=finalized.staging_keys, + ) + except BaseException: + self._tq_buffer.abort(group_id) + # Best-effort gate cleanup: no receipt will ever seal these ids. + nemo_gym_env = self._env_handles.get("nemo_gym") + if nemo_gym_env is not None: + try: + await nemo_gym_env.fail_rollouts.remote( + rollout_ids, reason="dispatch_failed" + ) + except Exception as error: # noqa: BLE001 — TTL is the backstop + print(f"fail_rollouts({group_id}) failed: {error}", flush=True) + raise diff --git a/nemo_rl/experience/row_dump.py b/nemo_rl/experience/row_dump.py new file mode 100644 index 00000000000..12b831a6e11 --- /dev/null +++ b/nemo_rl/experience/row_dump.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Env-gated dump of canonical training rows at their TQ publish sites. + +Set ``NRL_SC_DUMP_TRAIN_ROWS=`` to append one JSON line per training row +whenever a group is published to the ``rollout_data`` partition — from the +legacy ``TQReplayBuffer.commit`` path and from the token-capture +``BlackboxFinalizer`` publish. Off (no I/O, no imports of the payload) unless +the env var is set. Used by the S5 legacy-vs-capture offline row diff. +""" + +import json +import os +import threading +from collections.abc import Mapping +from typing import Any, Optional + +import torch + +_DUMP_ENV_VAR = "NRL_SC_DUMP_TRAIN_ROWS" +# The finalizer publishes from a worker thread (asyncio.to_thread) while the +# legacy path publishes from the SC event loop; serialize appends. +_G_WRITE_LOCK = threading.Lock() + + +def _row_value(tensor: torch.Tensor, row: int) -> Any: + value = tensor[row] + if value.dim() == 0: + return value.item() + return value.tolist() + + +def maybe_dump_train_rows( + *, + source: str, + group_id: str, + sample_ids: list[str], + train_batch: Mapping[str, torch.Tensor], + weight_version: Optional[int], +) -> None: + """Append each row of a published group to the dump file, if enabled. + + Args: + source: Publish site tag (``"legacy_commit"`` or ``"finalizer"``). + group_id: Prompt-group id the rows belong to. + sample_ids: Canonical per-row sample ids (``{group_id}_g{i}``). + train_batch: Column tensors as passed to ``pack_payload``. + weight_version: Weight version stamped on the rows' tags. + """ + dump_dir = os.environ.get(_DUMP_ENV_VAR) + if not dump_dir: + return + os.makedirs(dump_dir, exist_ok=True) + path = os.path.join(dump_dir, f"train_rows_{source}.jsonl") + lines = [] + for i, sample_id in enumerate(sample_ids): + record = { + "source": source, + "group_id": group_id, + "sample_id": sample_id, + "weight_version": weight_version, + **{name: _row_value(tensor, i) for name, tensor in train_batch.items()}, + } + lines.append(json.dumps(record)) + with _G_WRITE_LOCK, open(path, "a") as f: + f.write("\n".join(lines) + "\n") diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index d68bf512bcc..5efd34c7341 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -526,6 +526,37 @@ def _post_init(self): results = ray.get(futures) return results + def setup_token_capture( + self, dp_cfg: dict[str, Any], staging_partition: str + ) -> None: + """Install gate-authoritative token capture in every DP-leader worker. + + Called once at setup when ``token_capture.enabled``; each async worker + builds its in-worker data-plane client + TQTokenSink and makes the + single Gym ``install_capture`` call (see + docs/design-docs/tq-gym-gate-authoritative.md § 9.1). + """ + assert self.cfg["vllm_cfg"]["async_engine"], ( + "token capture requires the async vLLM engine (the capture host " + "is the worker's in-process HTTP server)" + ) + futures = self.worker_group.run_all_workers_single_data( + "setup_token_capture", + dp_cfg=dp_cfg, + staging_partition=staging_partition, + run_rank_0_only_axes=["tensor_parallel", "pipeline_parallel"], + ) + ray.get(futures) + + def set_rollout_weight_version(self, version: int) -> None: + """Rotate the weight version workers stamp on captured model calls.""" + futures = self.worker_group.run_all_workers_single_data( + "set_rollout_weight_version", + version=version, + run_rank_0_only_axes=["tensor_parallel", "pipeline_parallel"], + ) + ray.get(futures) + def _get_raw_spec_counters(self) -> dict[str | tuple[str, int], float]: """Collect raw spec decode counters from workers.""" futures = self.worker_group.run_all_workers_single_data( @@ -905,6 +936,9 @@ def shutdown(self) -> bool: try: # Use the worker group's shutdown method with the worker's cleanup method return self.worker_group.shutdown(cleanup_method="shutdown") + except ray.exceptions.RayActorError: + # Workers already dead (e.g., shut down via another handle to the same actors). + return True except Exception as e: print(f"Error during policy shutdown: {e}") return False diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 591b929fbf4..ed3cab46473 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -16,6 +16,7 @@ import copy import gc import logging +import os import threading import time import uuid @@ -195,6 +196,17 @@ def __init__( self.base_url = None self.http_server = None + # Gate-authoritative token capture (dormant until the + # setup_token_capture fan-out runs; see + # docs/design-docs/tq-gym-gate-authoritative.md § 9.1). The weight + # version is stamped per model call at begin_call time and rotated by + # the set_rollout_weight_version fan-out from the SC's _sync_weights. + self.token_capture = None + self._rollout_weight_version = 0 + # In-flight captured calls keyed by id(request): (ActiveCall, the + # exact engine prompt ids recorded at preprocess time). + self._capture_calls: dict[int, tuple[Any, list[int]]] = {} + super().__init__( config, bundle_indices, @@ -427,8 +439,100 @@ async def get_reserved_url(self) -> Optional[str]: async def report_dp_openai_server_base_url(self) -> Optional[str]: return self.base_url + def install_token_capture(self, capture: Any) -> None: + """Gym's ``install_capture`` seam (the ``CaptureHost`` contract).""" + self.token_capture = capture + + async def setup_token_capture( + self, dp_cfg: dict[str, Any], staging_partition: str + ) -> bool: + """Host gate-authoritative token capture in this worker. + + Fan-out target (token_capture.enabled only): builds the in-worker + data-plane client and TQTokenSink, then makes the single + ``install_capture`` call wiring Gym's engine-blind capture core + + vLLM adapter into this worker. Returns whether capture was installed + (False on non-model-owner ranks, which serve no HTTP). + """ + if not self.is_model_owner: + return False + # Deferred: nemo_gym is an optional extra absent in non-gym runs. + from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter + from nemo_gym.token_id_capture.staging import install_capture + + from nemo_rl.data_plane import build_data_plane_client + from nemo_rl.data_plane.tq_token_sink import TQTokenSink + + dp_client = build_data_plane_client(dp_cfg, bootstrap=False) + sink = TQTokenSink(dp_client, staging_partition=staging_partition) + install_capture( + self, + sink=sink, + weight_version_fn=lambda: self._rollout_weight_version, + adapter=VLLMCaptureAdapter(), + ) + return True + + async def set_rollout_weight_version(self, version: int) -> None: + """Rotate the weight version stamped on subsequent captured calls.""" + self._rollout_weight_version = int(version) + + def _begin_request_capture(self, request: Any, prompt_token_ids: list[int]) -> None: + """Admit one gate-forwarded call into the capture layer. + + Called from preprocess_chat once the exact engine prompt is known + (post-splice in token-in mode, full render in text mode). No-op + unless capture is installed and the request carries the gate's + ``ng_capture`` context. + """ + capture = self.token_capture + context = getattr(request, "ng_capture", None) + if capture is None or not context: + return + call = capture.begin_call( + rollout_id=context["rollout_id"], + call_id=context["call_id"], + parent_call_id=context.get("parent_call_id"), + prev_len=int(context.get("prev_len") or 0), + mode=context.get("mode") or "text", + stream=bool(getattr(request, "stream", False)), + ) + self._capture_calls[id(request)] = (call, list(prompt_token_ids)) + + def _finish_request_capture(self, request: Any, content: dict) -> dict: + """Stage the finished call and ride its coords on the response. + + Fail-closed (§ 3.5): the sink write happens inside complete_call — + the coords exist only after the bytes are durable, and any capture + failure degrades to capture_failed coords without breaking the + completion. Token ids and logprobs are stripped: the staged delta is + the only token store on this path, so the worker->gate hop carries + text + delta ids + coords only (§ 3.2). + """ + state = self._capture_calls.pop(id(request), None) + if state is None: + return content + call, prompt_token_ids = state + payload = dict(content) + # vLLM's OpenAI response carries no prompt ids; the adapter reads the + # preprocess-time engine prompt off the payload (see + # nemo_gym.token_id_capture.adapters.vllm.extract_prompt_ids). + payload["prompt_token_ids"] = prompt_token_ids + coords = self.token_capture.complete_call_from_response(call, payload) + for choice in content.get("choices") or []: + choice.pop("logprobs", None) + content["ng_commit_coords"] = coords.model_dump() + return content + + def _abort_request_capture(self, request: Any, *, reason: str) -> None: + """Drop the in-flight capture state for a request that errored.""" + state = self._capture_calls.pop(id(request), None) + if state is not None and self.token_capture is not None: + self.token_capture.fail_call(state[0], reason=reason) + # ruff: noqa def _setup_vllm_openai_api_server(self, app: FastAPI) -> FastAPI: + worker_self = self from copy import deepcopy from logging import Filter as LoggingFilter from logging import LogRecord @@ -599,6 +703,11 @@ async def preprocess_chat( actual_request_max_tokens, res[1][0]["prompt_token_ids"], ) + # Token capture, text mode: the full render is the exact + # engine prompt. + worker_self._begin_request_capture( + request, res[1][0]["prompt_token_ids"] + ) return res last_assistant_message_idx = None @@ -656,6 +765,10 @@ async def preprocess_chat( final_prompt_token_ids, ) + # Token capture, token-in mode: the spliced prompt is the + # exact engine prompt. + worker_self._begin_request_capture(request, final_prompt_token_ids) + return res ######################################## @@ -667,6 +780,9 @@ class NeMoRLChatCompletionRequest( NeMoRLOpenAIChatRequestMixin, ChatCompletionRequest ): required_prefix_token_ids: Optional[List[int]] = None + # Gate-authoritative token capture: the call identity the gate + # attaches (rollout_id, call_id, parent_call_id, prev_len, mode). + ng_capture: Optional[dict[str, Any]] = None # vLLM 0.20 routes both /v1/chat/completions and /tokenize through # OpenAIServingRender.preprocess_chat, so the prefix-token override @@ -736,6 +852,7 @@ async def create_chat_completion( # max_model_len during tokenization, instead of returning an # ErrorResponse. Convert to HTTP 400 so the Gym proxy can # detect context-length overflow and handle it gracefully. + worker_self._abort_request_capture(request, reason="context_length") return JSONResponse( content={ "error": { @@ -746,15 +863,24 @@ async def create_chat_completion( }, status_code=400, ) + except BaseException: + worker_self._abort_request_capture(request, reason="engine_error") + raise if isinstance(generator, ErrorResponse): + worker_self._abort_request_capture(request, reason="error_response") return JSONResponse( content=generator.model_dump(), status_code=generator.error.code ) elif isinstance(generator, ChatCompletionResponse): - return JSONResponse(content=generator.model_dump()) + content = generator.model_dump() + # Token capture: stage the delta and ride the coords on the + # response; strips logprobs/ids (no-op when capture is off). + content = worker_self._finish_request_capture(request, content) + return JSONResponse(content=content) + worker_self._abort_request_capture(request, reason="streaming_response") return StreamingResponse(content=generator, media_type="text/event-stream") ######################################## @@ -885,6 +1011,13 @@ def _setup_vllm_server(self) -> "tuple[threading.Thread, str, uvicorn.Server]": base_url = f"http://{node_ip}:{free_port}/v1" print(f"Starting server on {base_url}") + byte_dir = os.environ.get("NRL_HTTP_BYTES_DIR") + if byte_dir: + # Perf-measurement tooling only (see nemo_rl/utils/http_byte_counter.py). + from nemo_rl.utils.http_byte_counter import HttpByteCounterMiddleware + + app = HttpByteCounterMiddleware(app, "vllm_worker", byte_dir) # type: ignore[assignment] + config = uvicorn.Config( app, host="0.0.0.0", diff --git a/nemo_rl/models/policy/lm_policy.py b/nemo_rl/models/policy/lm_policy.py index f1a5d705c06..6eea7497a38 100644 --- a/nemo_rl/models/policy/lm_policy.py +++ b/nemo_rl/models/policy/lm_policy.py @@ -1096,6 +1096,9 @@ def shutdown(self) -> bool: try: # Use the worker group's shutdown method with the worker's cleanup method return self.worker_group.shutdown(cleanup_method="shutdown") + except ray.exceptions.RayActorError: + # Workers already dead (e.g., shut down via another handle to the same actors). + return True except Exception as e: print(f"Error during policy shutdown: {e}") return False @@ -1103,12 +1106,11 @@ def shutdown(self) -> bool: def __del__(self) -> None: """Shuts down the worker groups when the object is deleted or is garbage collected. - This is an extra safety net in case the user forgets to call worker_group.shutdown() and the pointer to + This is an extra safety net in case the user forgets to call shutdown() and the pointer to the object is lost due to leaving a function scope. It's always recommended that the - user calls worker_group.shutdown(). + user calls shutdown(). """ - if hasattr(self, "worker_group"): - self.worker_group.shutdown(cleanup_method="shutdown") + self.shutdown() def start_gpu_profiling(self) -> None: """Start GPU profiling.""" diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index 1e4bc93ba09..058647ba0ad 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -457,3 +457,128 @@ def train_from_meta( warnings.warn(f"Error getting theoretical flops: {e}") return aggregated_results + + # ── split-API fanout (SC async path) ─────────────────────────────────── + # + # Counterpart to :meth:`train_from_meta`, exposed to ``PolicyTrainerActor`` + # so :class:`SingleControllerActor` can stream microbatches without + # forcing a full-step optimizer.step on every dispatch. + # + # Lifecycle: + # begin_train_step — open step; broadcast loss_fn/gbs/mbs + # train_microbatch_from_meta (N×) — DP-sharded fwd/bwd, grads accumulate + # finish_train_step — all_reduce + opt.step + sched.step + # abort_train_step — drop accumulators, no opt.step + # + # ``train_from_meta`` is unchanged and remains the sync entrypoint. + + def begin_train_step( + self, + step_id: str, + loss_fn: LossFunction, + gbs: Optional[int] = None, + mbs: Optional[int] = None, + ) -> None: + """Open a logical train step on every worker.""" + batch_size = gbs or self.cfg["train_global_batch_size"] + micro_batch_size = mbs or self.cfg["train_micro_batch_size"] + if self.flops_tracker is not None: + self.flops_tracker.reset() + futures = self.worker_group.run_all_workers_single_data( + "begin_train_step_presharded", + step_id=step_id, + loss_fn=loss_fn, + gbs=batch_size, + mbs=micro_batch_size, + ) + ray.get(futures) + + def train_microbatch_from_meta( + self, + step_id: str, + meta: KVBatchMeta, + timer: Optional[Timer] = None, + ) -> dict[str, Any]: + """Dispatch one microbatch (DP-sharded) into an open train step. + + Mirrors the sharding logic of :meth:`train_from_meta` but without + a logical-batch sizing constraint: this routes ``meta`` to DP + ranks and runs forward+backward; gradients accumulate in + ``.grad``. The optimizer step happens at :meth:`finish_train_step`. + """ + self._stamp_pad_seqlen(meta) + spa, dba = self._packing_args("train_mb_tokens") + train_meta = replace( + meta, + fields=list(DP_TRAIN_FIELDS), + task_name="train", + ) + with timer.time("policy_training/shard_meta") if timer else nullcontext(): + dp_metas, _ = shard_meta_for_dp( + train_meta, + dp_world=self.sharding_annotations.get_axis_size("data_parallel"), + batch_size=None, + sequence_packing_args=spa, + dynamic_batching_args=dba, + ) + + if self.flops_tracker is not None: + for m in dp_metas: + self.flops_tracker.track_batch(list(m.sequence_lengths or [])) + + with ( + timer.time("policy_training/submit_microbatch_futures") + if timer + else nullcontext() + ): + futures = self.worker_group.run_all_workers_sharded_data( + "train_microbatch_presharded", + meta=dp_metas, + in_sharded_axes=["data_parallel"], + replicate_on_axes=[ + "context_parallel", + "tensor_parallel", + "pipeline_parallel", + ], + output_is_replicated=[ + "context_parallel", + "tensor_parallel", + "pipeline_parallel", + ], + common_kwargs={"step_id": step_id}, + ) + results = self.worker_group.get_all_worker_results(futures) + # Per-microbatch metrics: pass through DP-rank-0 by convention, + # backend may aggregate later if needed. Surface as-is for now. + return results[0] if results else {} + + def finish_train_step(self, step_id: str) -> dict[str, Any]: + """Close an open train step: all_reduce, rescale, optimizer.step. + + Aggregates per-rank step results into the same shape as + :meth:`train_from_meta` so callers don't have to special-case + the split path. + """ + futures = self.worker_group.run_all_workers_single_data( + "finish_train_step_presharded", + step_id=step_id, + ) + results = ray.get(futures) + aggregated_results = _aggregate_train_results(results) + + if self.flops_tracker is not None: + aggregated_results["total_flops"] = self.flops_tracker.total_flops + aggregated_results["num_ranks"] = self.worker_group.cluster.world_size() + + return aggregated_results + + def abort_train_step(self, step_id: str) -> None: + """Drop partial step state on every worker. No optimizer.step.""" + futures = self.worker_group.run_all_workers_single_data( + "abort_train_step_presharded", + step_id=step_id, + ) + ray.get(futures) + + if self.flops_tracker is not None: + self.flops_tracker.reset() diff --git a/nemo_rl/models/policy/workers/megatron_policy_worker.py b/nemo_rl/models/policy/workers/megatron_policy_worker.py index 0eec29f04a2..a13e4943732 100644 --- a/nemo_rl/models/policy/workers/megatron_policy_worker.py +++ b/nemo_rl/models/policy/workers/megatron_policy_worker.py @@ -143,6 +143,11 @@ class MegatronPolicyWorkerImpl( AbstractPolicyWorker, ColocatablePolicyInterface, ): + # Holds the split-API train-step state between begin/finish or + # begin/abort; None when no step is open. Declared at class level so + # ``self._train_step_state = None`` after finish/abort type-checks. + _train_step_state: Optional[dict[str, Any]] = None + def __repr__(self): """Customizes the actor's prefix in the Ray logs. @@ -835,6 +840,389 @@ def _set_moe_grad_scale_func(self, func): if config is not None: config.moe_grad_scale_func = func + # ── split-API train-step state machine (SingleController async path) ── + # + # Mirrors the v1/v2 implementations, adapted for mcore. Key differences: + # + # 1. mcore DDP accumulates ``param.main_grad`` per backward and dispatches + # a cross-DP reduce when ``is_last_microbatch=True`` (one per + # ``forward_backward_func`` call). Naively chaining multiple + # ``forward_backward_func`` calls between ``optimizer.step()`` would + # over-count: each call's terminal reduce sums an already-reduced + # bucket again. We wrap every call in ``self.model.no_sync()`` so + # hooks accumulate locally only; one explicit ``start_grad_sync`` + + # ``finish_grad_sync`` at finish does the single true reduce. + # 2. PP>1: the pipeline scheduler invokes ``config.grad_sync_func`` + # directly on last-microbatch boundaries — this bypasses the + # ``no_sync`` gate. We null it for the duration of the step and + # restore at finish/abort. + # 3. Grad clip is bundled inside ``MegatronOptimizer.step()``; the 1/N + # rescale via ``self.model.scale_gradients(1/N)`` must run before + # ``optimizer.step()`` so the clip operates on the rescaled grad. + # 4. With ``calculate_per_token_loss=True`` + ``average_in_collective= + # False``, mcore's DDP sums (does not average) grads across DP, so + # no FSDP-style ``loss *= dp_size*cp_size`` cancellation is needed + # per microbatch. + + def _split_step_state_init( + self, + step_id: str, + loss_fn: LossFunction, + gbs: Optional[int], + mbs: Optional[int], + ) -> dict[str, Any]: + from nemo_rl.algorithms.loss.interfaces import LossType + + return { + "step_id": step_id, + "loss_fn": loss_fn, + "loss_type": getattr(loss_fn, "loss_type", LossType.TOKEN_LEVEL), + "gbs": gbs or self.cfg["train_global_batch_size"], + "mbs": mbs or self.cfg["train_micro_batch_size"], + "local_valid_seqs": torch.zeros((), dtype=torch.float64, device="cuda"), + "local_valid_toks": torch.zeros((), dtype=torch.float64, device="cuda"), + "all_mb_metrics": [], + "mb_losses": [], + "total_num_microbatches": 0, + # Saved across the step so we can restore at finish/abort. + "saved_grad_sync_func": None, + "saved_no_sync_func": None, + "no_sync_active": False, + } + + def _assert_step_open(self, step_id: str) -> dict[str, Any]: + state = getattr(self, "_train_step_state", None) + if state is None: + raise RuntimeError( + f"no train step open; begin_train_step({step_id!r}) must be called first" + ) + if state["step_id"] != step_id: + raise RuntimeError( + f"step_id mismatch: open step is {state['step_id']!r}, got {step_id!r}" + ) + return state + + @wrap_with_nvtx_name("megatron_policy_worker/begin_train_step") + def begin_train_step( + self, + step_id: str, + loss_fn: LossFunction, + gbs: Optional[int] = None, + mbs: Optional[int] = None, + ) -> None: + existing = getattr(self, "_train_step_state", None) + if existing is not None: + raise RuntimeError( + f"train step {existing['step_id']!r} is already open; " + f"call finish_train_step or abort_train_step before begin" + ) + # Match sync train() inference-state reset (line 332-340). + if hasattr(self.model, "inference_params"): + self.model.inference_params = None + for module in self.model.modules(): + if hasattr(module, "reset_inference_cache"): + module.reset_inference_cache() + if hasattr(module, "_inference_key_value_memory"): + module._inference_key_value_memory = None + + self.model.train() + self.model.zero_grad_buffer() + self.optimizer.zero_grad() + + state = self._split_step_state_init( + step_id=step_id, loss_fn=loss_fn, gbs=gbs, mbs=mbs + ) + + # Null both mcore hooks that would fire a mid-step DP reduce: + # grad_sync_func — PP scheduler's direct call on last-MB boundaries. + # no_sync_func — forward_backward_no_pipelining wraps inner MBs + # in it and runs the LAST MB OUTSIDE, leaking + # per_param_grad_ready_counts past our outer + # no_sync. Override to nullcontext so only the + # outer no_sync in train_microbatch governs + # is_last_microbatch. + # getattr-by-string keeps "config" out of __code__.co_names so + # cloudpickle doesn't grab torch.distributed.config. + model_config = getattr(self.model, "config", None) + if model_config is not None: + state["saved_grad_sync_func"] = getattr( + model_config, "grad_sync_func", None + ) + state["saved_no_sync_func"] = getattr(model_config, "no_sync_func", None) + model_config.grad_sync_func = None + model_config.no_sync_func = nullcontext + else: + state["saved_grad_sync_func"] = None + state["saved_no_sync_func"] = None + + self._train_step_state = state + + @wrap_with_nvtx_name("megatron_policy_worker/train_microbatch") + def train_microbatch( + self, + step_id: str, + data: BatchedDataDict[Any], + ) -> dict[str, Any]: + """One DP slice of data → one ``forward_backward_func`` invocation. + + Wrapped in ``self.model.no_sync()`` so the mcore DDP hooks + accumulate ``param.main_grad`` locally on each rank without + dispatching a per-call DP reduce. The single true reduce is done + explicitly in ``finish_train_step``. + """ + state = self._assert_step_open(step_id) + loss_fn = state["loss_fn"] + + # Accumulate local mask sums for the finish-time all_reduce. + # Inlined from process_global_batch (data.py:319-332) — we can't + # call process_global_batch directly because it eagerly all_reduces + # the local sums, which is exactly what we're trying to defer. + assert "sample_mask" in data, "sample_mask required on microbatch data" + sample_mask = data["sample_mask"] + call_local_seqs = torch.sum(sample_mask).to(torch.float64) + if "token_mask" in data: + token_mask = data["token_mask"] + call_local_toks = torch.sum( + token_mask[:, 1:] * sample_mask.unsqueeze(-1) + ).to(torch.float64) + else: + call_local_toks = call_local_seqs * data["input_ids"].shape[1] + + state["local_valid_seqs"] = state["local_valid_seqs"] + call_local_seqs + state["local_valid_toks"] = state["local_valid_toks"] + call_local_toks + + # Build the per-call iterator. Each ``train_microbatch_from_meta`` + # call carries one DP slice; the iterator subdivides into pipeline + # microbatches. + ( + data_iterator, + num_microbatches, + micro_batch_size, + seq_length, + padded_seq_length, + ) = get_microbatch_iterator( + data, + self.cfg, + state["mbs"], + straggler_timer=self.mcore_state.straggler_timer, + ) + state["total_num_microbatches"] += int(num_microbatches) + + loss_post_processor = LossPostProcessor( + loss_fn=loss_fn, + cfg=self.cfg, + num_microbatches=num_microbatches, + sampling_params=self.sampling_params, + draft_model=self.draft_model, + ) + + # Placeholder N=1: loss returns un-normalized sums. ``backward`` + # deposits raw ``d(sum)/dθ`` into ``param.main_grad`` via the DDP + # hooks. The 1/N rescale happens once at finish. + placeholder_n = torch.tensor(1.0, device="cuda") + + draft_enabled = "draft" in self.cfg and self.cfg["draft"]["enabled"] + + # The critical wrap: hooks fire (accumulate main_grad) but the + # per-call reduce dispatch is gated off. + with self.model.no_sync(): + rerun_state_machine = get_rerun_state_machine() + while rerun_state_machine.should_run_forward_backward(data_iterator): + losses_reduced = megatron_forward_backward( + model=self.model, + data_iterator=data_iterator, + num_microbatches=num_microbatches, + seq_length=padded_seq_length, + mbs=micro_batch_size, + post_processing_fn=loss_post_processor, + forward_only=False, + defer_fp32_logits=self.defer_fp32_logits, + global_valid_seqs=placeholder_n, + global_valid_toks=placeholder_n, + sampling_params=self.sampling_params, + straggler_timer=self.mcore_state.straggler_timer, + draft_model=self.draft_model, + enable_hidden_capture=draft_enabled, + use_linear_ce_fusion_loss=self.cfg["megatron_cfg"].get( + "use_linear_ce_fusion_loss", False + ), + ) + + if self.cfg["megatron_cfg"]["empty_unused_memory_level"] >= 1: + torch.cuda.empty_cache() + + # Collect per-mb metrics from the last PP stage; broadcast to all + # PP ranks so non-last-stage ranks have something to all_reduce + # against at finish. Metrics carry the N=1 placeholder for now — + # ``finish_train_step`` rescales by the true 1/N. + if parallel_state.is_pipeline_last_stage(ignore_virtual=True): + mb_metrics_collected = [] + for x in losses_reduced: + mb_metrics_collected.append(dict(x)) + else: + mb_metrics_collected = None + + mb_metrics_collected = broadcast_loss_metrics_from_last_stage( + mb_metrics_collected + ) + + for m in mb_metrics_collected: + state["all_mb_metrics"].append(m) + # ``loss`` key is the un-normalized per-mb scalar; collect for + # the global_loss aggregation at finish. + if "loss" in m: + state["mb_losses"].append(m["loss"]) + + return { + "local_valid_seqs_mb": float(call_local_seqs.item()), + "local_valid_toks_mb": float(call_local_toks.item()), + "num_pipeline_microbatches": int(num_microbatches), + } + + @wrap_with_nvtx_name("megatron_policy_worker/finish_train_step") + def finish_train_step(self, step_id: str) -> dict[str, Any]: + from nemo_rl.algorithms.loss.interfaces import LossType + + state = self._assert_step_open(step_id) + + # All-reduce accumulated mask sums across DP to recover true N. + to_reduce = torch.stack( + [state["local_valid_seqs"], state["local_valid_toks"]] + ).to(torch.float64) + torch.distributed.all_reduce( + to_reduce, group=parallel_state.get_data_parallel_group() + ) + global_valid_seqs = to_reduce[0] + global_valid_toks = to_reduce[1] + + if state["loss_type"] == LossType.TOKEN_LEVEL: + n_true = global_valid_toks + else: + n_true = global_valid_seqs + n_safe = n_true if n_true.item() > 0 else torch.tensor(1.0, device="cuda") + inv_n = float((1.0 / n_safe).item()) + + # Rescale all locally-accumulated gradients by 1/N. The reduce + # below sees the rescaled grads; for all_reduce the result is the + # global mean grad; for reduce_scatter (dist-opt) it's the shard. + # Either way, opt.step sees the right-normalized gradient. + self.model.scale_gradients(inv_n) + + # The ONE true cross-DP reduce for the entire step. + self.model.start_grad_sync() + self.model.finish_grad_sync() + + # opt.step clips internally (clip_grad config); operates on the + # already-rescaled grad. Returns (success, grad_norm, num_zeros). + update_successful, grad_norm, num_zeros_in_grad = self.optimizer.step() + + pg_collection = get_pg_collection(self.model) + update_successful = logical_and_across_model_parallel_group( + update_successful, mp_group=pg_collection.mp + ) + grad_norm = reduce_max_stat_across_model_parallel_group( + grad_norm, mp_group=pg_collection.mp + ) + num_zeros_in_grad = reduce_max_stat_across_model_parallel_group( + num_zeros_in_grad, mp_group=pg_collection.mp + ) + + if self.cfg["megatron_cfg"]["empty_unused_memory_level"] >= 2: + torch.cuda.empty_cache() + + # Restore the mcore hooks we nulled in begin_train_step. + # See begin_train_step for why .config is accessed by string. + finish_model_config = getattr(self.model, "config", None) + if finish_model_config is not None: + finish_model_config.grad_sync_func = state["saved_grad_sync_func"] + finish_model_config.no_sync_func = state["saved_no_sync_func"] + + # Record the LR/WD before self.scheduler.step() is called. + curr_lr = self.scheduler.get_lr(self.optimizer.param_groups[0]) + curr_wd = self.scheduler.get_wd() + + # Scheduler increment matches sync path's ``increment=gbs``. + self.scheduler.step(increment=state["gbs"]) + + # Per-mb metrics were computed with N=1; rescale to match what the + # sync path produces. ``masked_mean`` is linear in 1/N so a single + # scalar multiply per metric recovers the normalized value. + rescaled_metrics: list[dict[str, Any]] = [] + global_valid_seqs_f = float(global_valid_seqs.item()) + global_valid_toks_f = float(global_valid_toks.item()) + + for m in state["all_mb_metrics"]: + out: dict[str, Any] = {} + for k, v in m.items(): + if "_min" in k or "_max" in k: + out[k] = v + elif isinstance(v, torch.Tensor): + out[k] = v.detach() * inv_n + else: + out[k] = v * inv_n + out["lr"] = curr_lr + out["wd"] = curr_wd + out["global_valid_seqs"] = global_valid_seqs_f + out["global_valid_toks"] = global_valid_toks_f + rescaled_metrics.append(out) + + # Scale per-mb losses by 1/N and reduce per-call sums. + scaled_losses = [lv * inv_n for lv in state["mb_losses"]] + losses_to_aggregate = [torch.tensor(scaled_losses).sum().item()] + + mb_metrics, global_loss = aggregate_training_statistics( + all_mb_metrics=rescaled_metrics, + losses=losses_to_aggregate, + data_parallel_group=parallel_state.get_data_parallel_group(), + ) + + metrics = { + "global_loss": global_loss.cpu(), + "rank": torch.distributed.get_rank(), + "gpu_name": torch.cuda.get_device_name(), + "model_dtype": self.dtype, + "all_mb_metrics": mb_metrics, + "grad_norm": torch.tensor([grad_norm]), + } + + # MoE aux-loss metrics: same convention as sync train() — scale + # by the total pipeline-microbatch count accumulated across all + # train_microbatch calls. + model_config = getattr(self.model, "config", None) + num_moe_experts = getattr(model_config, "num_moe_experts", None) + if num_moe_experts is not None and num_moe_experts > 1: + moe_loss_scale = 1.0 / max(1, state["total_num_microbatches"]) + moe_metrics = get_moe_metrics( + loss_scale=moe_loss_scale, + per_layer_logging=self.cfg["megatron_cfg"]["moe_per_layer_logging"], + ) + if moe_metrics: + metrics["moe_metrics"] = moe_metrics + + self._train_step_state = None + return metrics + + @wrap_with_nvtx_name("megatron_policy_worker/abort_train_step") + def abort_train_step(self, step_id: str) -> None: + state = getattr(self, "_train_step_state", None) + if state is None: + return + if state["step_id"] != step_id: + raise RuntimeError( + f"abort_train_step({step_id!r}) does not match open step " + f"{state['step_id']!r}" + ) + # Restore the mcore hooks we nulled in begin_train_step before + # zero_grad_buffer touches anything. + # See begin_train_step for why .config is accessed by string. + abort_model_config = getattr(self.model, "config", None) + if abort_model_config is not None: + abort_model_config.grad_sync_func = state["saved_grad_sync_func"] + abort_model_config.no_sync_func = state["saved_no_sync_func"] + self.model.zero_grad_buffer() + self.optimizer.zero_grad() + self._train_step_state = None + @wrap_with_nvtx_name("megatron_policy_worker/get_reference_policy_logprobs") def get_reference_policy_logprobs( self, diff --git a/nemo_rl/utils/http_byte_counter.py b/nemo_rl/utils/http_byte_counter.py new file mode 100644 index 00000000000..7038d35b053 --- /dev/null +++ b/nemo_rl/utils/http_byte_counter.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Env-gated per-route HTTP byte counter (``NRL_HTTP_BYTES_DIR``). + +Measurement tooling for the token-capture perf comparison: sums request-body +and response-body bytes per path on the vLLM worker's in-process HTTP server +and periodically flushes an aggregate JSON. Mirrors Gym's +``HttpByteCounterMiddleware`` (separate copy: worker venvs cannot assume +``nemo_gym`` is installed on the legacy path). Never installed unless the env +var is set. +""" + +import json +import os +from typing import Any, Awaitable, Callable + +Scope = dict[str, Any] +Message = dict[str, Any] +Receive = Callable[[], Awaitable[Message]] +Send = Callable[[Message], Awaitable[None]] + + +class HttpByteCounterMiddleware: + """Pure ASGI wrapper counting per-path request/response body bytes.""" + + FLUSH_EVERY = 25 + + def __init__(self, app: Any, server_name: str, out_dir: str) -> None: + self.app = app + self.out_path = os.path.join(out_dir, f"{server_name}_{os.getpid()}.json") + os.makedirs(out_dir, exist_ok=True) + self.counts: dict[str, list[int]] = {} + self._events = 0 + + def _flush(self) -> None: + with open(self.out_path, "w") as f: + json.dump( + { + path: { + "requests": c[0], + "req_bytes": c[1], + "resp_bytes": c[2], + } + for path, c in self.counts.items() + }, + f, + ) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + entry = self.counts.setdefault(scope["path"], [0, 0, 0]) + entry[0] += 1 + + async def counting_receive() -> Message: + message = await receive() + if message["type"] == "http.request": + entry[1] += len(message.get("body", b"")) + return message + + async def counting_send(message: Message) -> None: + if message["type"] == "http.response.body": + entry[2] += len(message.get("body", b"")) + await send(message) + + try: + await self.app(scope, counting_receive, counting_send) + finally: + self._events += 1 + if self._events % self.FLUSH_EVERY == 0: + self._flush() diff --git a/nemo_rl/utils/logger.py b/nemo_rl/utils/logger.py index c00fa96b9b4..caf973a3c54 100644 --- a/nemo_rl/utils/logger.py +++ b/nemo_rl/utils/logger.py @@ -393,6 +393,16 @@ def log_plot(self, figure: plt.Figure, step: int, name: str) -> None: """ self.run.log({name: figure}, step=step) + def finish(self) -> None: + """Flush queued metrics and close the wandb service. + + Required when the run lives inside a Ray actor: Ray tears the worker + down before wandb's atexit hook can drain the IPC queue to the service. + """ + if self.run is not None: + self.run.finish() + self.run = None + def log_histogram(self, histogram: list[Any], step: int, name: str) -> None: """Log histogram metrics to wandb. @@ -1038,6 +1048,13 @@ def log_hyperparams(self, params: Mapping[str, Any]) -> None: for logger in self.loggers: logger.log_hyperparams(params) + def finish(self) -> None: + """Flush and close backends that need explicit teardown (e.g. wandb).""" + for logger in self.loggers: + finish = getattr(logger, "finish", None) + if callable(finish): + finish() + def log_batched_dict_as_jsonl( self, to_log: BatchedDataDict[Any] | dict[str, Any], filename: str ) -> None: diff --git a/pyrefly.toml b/pyrefly.toml index bca7713e542..9806035e376 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -8,6 +8,7 @@ replace-imports-with-any = [ "transformers.*", "vllm.*", "math_verify.*", + "nemo_gym.*", "sympy.*", "torchdata.*", "nemo.*", @@ -45,12 +46,14 @@ project-includes = [ "nemo_rl/algorithms/async_utils/__init__.py", "nemo_rl/algorithms/async_utils/interfaces.py", "nemo_rl/algorithms/async_utils/replay_buffer.py", + "nemo_rl/algorithms/async_utils/staleness_sampler.py", "nemo_rl/algorithms/logits_sampling_utils.py", "nemo_rl/algorithms/loss/__init__.py", "nemo_rl/algorithms/loss/interfaces.py", "nemo_rl/algorithms/loss/utils.py", "nemo_rl/algorithms/opd.py", "nemo_rl/algorithms/reward_functions.py", + "nemo_rl/algorithms/single_controller.py", "nemo_rl/algorithms/utils.py", "nemo_rl/algorithms/x_token/__init__.py", "nemo_rl/algorithms/x_token/utils.py", @@ -109,6 +112,7 @@ project-includes = [ "nemo_rl/data_plane/observability.py", "nemo_rl/data_plane/preshard.py", "nemo_rl/data_plane/schema.py", + "nemo_rl/data_plane/tq_token_sink.py", "nemo_rl/data_plane/worker_mixin.py", "nemo_rl/distributed/__init__.py", "nemo_rl/distributed/collectives.py", @@ -128,6 +132,8 @@ project-includes = [ "nemo_rl/evals/__init__.py", "nemo_rl/evals/answer_parsing.py", "nemo_rl/experience/__init__.py", + "nemo_rl/experience/blackbox_finalizer.py", + "nemo_rl/experience/row_dump.py", "nemo_rl/experience/interfaces.py", "nemo_rl/experience/rollout_manager.py", "nemo_rl/experience/rollouts.py", @@ -164,6 +170,7 @@ project-includes = [ "nemo_rl/models/generation/vllm/vllm_backend.py", "nemo_rl/models/huggingface/__init__.py", "nemo_rl/models/megatron/__init__.py", + "nemo_rl/models/megatron/draft/__init__.py", "nemo_rl/models/policy/__init__.py", "nemo_rl/models/policy/interfaces.py", "nemo_rl/models/policy/utils.py", @@ -175,6 +182,7 @@ project-includes = [ "nemo_rl/utils/__init__.py", "nemo_rl/utils/checkpoint.py", "nemo_rl/utils/config.py", + "nemo_rl/utils/http_byte_counter.py", "nemo_rl/utils/native_checkpoint.py", "nemo_rl/utils/nsys.py", "nemo_rl/utils/nvml.py", diff --git a/swe/EXPERIMENT_LAUNCH.md b/swe/EXPERIMENT_LAUNCH.md new file mode 100644 index 00000000000..f9f3e9801f4 --- /dev/null +++ b/swe/EXPERIMENT_LAUNCH.md @@ -0,0 +1,129 @@ +# Experiment Launch Plan — SWE Token-Capture A/B (2026-07-28) + +Concrete launch record for the run described generically in `SWE_RUN.md`. +Everything here is pinned to what will actually execute; update this file if +any pin changes before submission. **Status: READY — not yet submitted.** + +## 1. Hypothesis and expected outcome + +The gate-authoritative token-capture pipeline +(`docs/design-docs/tq-gym-gate-authoritative.md`) reduces HTTP bytes per +trained token and wall-clock step time vs the legacy token-echo path on a +real multi-turn agentic workload, at equal training quality. + +| Expectation | Basis | +|---|---| +| Bytes/token reduction ≥ 35.6 % (the single-turn floor), toward/beyond −46.9 % (sync-prototype multi-turn) | S5 instrumented A/B; legacy pays per-turn history re-echo (~quadratic in turns), capture stays O(generated tokens) | +| `exposed_generation` and `total_step_time` at or below legacy | S5 saw −16.8 % / −9.0 % single-turn; expect relative timing win to compress on a 30B MoE (GPU decode dominates) while the bytes win grows | +| `gate/token_in_rate` → 1.0 for continuation calls | First real measurement — OpenHands marker survival is the biggest unknown (risk #1); a low rate is a correctness-preserving perf bug, not a training bug | +| Reward / KL curves in the same band on both arms | S5: identical rewards 10/10, same KL band | + +## 2. Exact pins + +| What | Value | +|---|---| +| RL repo | branch `yukih/sc-entrypoint` @ `e71055650` (local-only; no remote has it) | +| Gym fork (submodule) | branch `tq-gate-capture` @ `e3b3eac6` (= upstream #2124 head `32b555f04` + S1–S3 capture commits + S5 byte-counter middleware; local-only) | +| Code snapshot | `code_snapshots/sc-swe-capture` @ `e710556` (immutable run tree; both arms run from it) | +| Container | `nemo-rl:sc-swe-baked.sqsh` (biguo; nightly-062526 + baked venvs — **stale vs branch lock**, hence forced venv rebuild below) | +| Model | Qwen3-30B-A3B-Thinking-2507, from SWE1 `step_230_hf` checkpoint (bihu, run dc3m70us lineage) | +| Train data | R2E-Gym subset jsonl (sdevare), 4,518 samples | +| Site config | `test_assets/SWE/grpo_qwen3_30b_async_swe.yaml` (real sandbox-image paths; snapshot copy is authoritative at run time) | +| Derived arm configs | `test_assets/SWE/derived_configs/grpo_swe_ab_{legacy,capture}.yaml` — regenerated at each submit by `make_capture_config.py` | + +Verified diff between derived arm configs: **only** `token_capture.enabled: +true` on the capture arm; `rollout_max_attempts_to_avoid_lp_nan: 1` pinned on +**both** arms (capture hard-errors without it; pinning both removes the +NaN-retry confound). + +## 3. Compute shape (per job; one job per arm) + +- 16 nodes × 8 GPUs (128 GPUs), SLURM `batch` partition, account + `coreai_dlalgo_genai`, 4 h wall time, `--exclusive`, + `--dependency=singleton` (arms queue back-to-back on comparable + allocations), idle-GPU-reaper exemption (180 min) in the job comment. +- 8 training nodes: Megatron-Core, TP=4 EP=8 CP=4 PP=2, pad 32. +- 8 generation nodes: async vLLM, TP=2, non-colocated. +- Single-controller entrypoint (`SC_MODE=1`, + `examples/run_grpo_single_controller.py`) + TransferQueue data plane. +- Sequence budget 131,072; GBS=64 (8 prompts/step × 8 generations/prompt); + LR 1e-6; staleness `age=1`, `force_in_order=true`, `over_sampling=false`. +- Agent: NeMo-Gym OpenHands, per-instance Singularity sandboxes (`.sif` + under igitman/sdevare lustre trees), up to ~100–200 turns/rollout. + +## 4. Environment posture (identical on both arms) + +- `NRL_FORCE_REBUILD_VENVS=true` — container venvs predate the branch's + `uv.lock`; capture arm additionally needs the unbaked `VLLM_GYM` worker + venv. First node-local build may take ~30 min (reaper exemption covers it). +- `GYM_VENV_DIR=/tmp/nemo_gym_venvs` — baked `/opt/gym_venvs` deps predate + the Gym fork's floors (aiohttp bump). +- `BYTES=1` (capture-arm measurement run): `NG_HTTP_BYTES_DIR` + + `NRL_HTTP_BYTES_DIR` → per-server JSON under `http_bytes//`; + forwarding into the job env added in `e71055650`. +- Secrets: `profiles/env.sh` (mode 600) sourced by the site wrapper; wrapper + fails loudly if `WANDB_API_KEY`/`HF_TOKEN` are unset. + +## 5. Run sequence and gates + +All submissions via (from repo root): + +```bash +SITE_WRAPPER=$PWD/test_assets/SWE/grpo_swe_tests.sh MAX_NUM_STEPS= ARM= [BYTES=1] bash swe/launch_swe_ab.sh +``` + +| # | Run | Purpose | Gate to proceed | +|---|---|---|---| +| 1 | `ARM=legacy MAX_NUM_STEPS=3` | Flag-off smoke: branch + Gym pin reproduce the known-good legacy SWE run | Job completes 3 steps; reward/KL sane; no venv/import failures | +| 2 | `ARM=legacy MAX_NUM_STEPS=25` | Measurement baseline | Completes ≥ 20 steps cleanly | +| 3 | `ARM=capture MAX_NUM_STEPS=25 BYTES=1` | Measurement + mechanism + byte accounting | — | +| 4 | (optional) repeat pair ×2 | Variance bars; S5 assessment puts run-to-run variance at ±5–10 % | 1 pair = directional, 3 pairs = quotable | + +W&B: project `pthombre-swe-capture-ab`, runs named +`swe-ab--`. Checkpoints: `/results//`. + +## 6. Analysis plan + +- Medians over steps 5..N (skip warm-up): `timing/train/total_step_time`, + `timing/train/exposed_generation`, `timing/train/valid_tokens_per_sec_per_gpu`. +- Mechanism: `gate/token_in_rate` (the headline first-time measurement), + `gate/fallback_{no_marker,fingerprint_miss,unknown_marker}` — sustained + `fingerprint_miss` ⇒ OpenHands rewrites history (marker-survival risk #1; + report back to the design doc §11). +- Health: `gate/capture_failed`, `train/global_valid_seqs` == GBS both arms; + `finalize/*` latency (if it dominates dispatch → argues for H5 finalizer pool). +- Equivalence: overlay `train/reward` + `train/gen_kl_error` across arms. +- Bytes: `python swe/aggregate_perf.py ` + → total bytes, bytes/trained-token, per-hop table. +- Results recorded in + `docs/design-docs/tq-gym-gate-authoritative-implementation-log.md` (post-MVP + perf report per design § 10). + +## 7. Known risks (accepted, watched) + +- **Gate death = silent stall** (S5 chaos finding; H1 fix queued): control + plane retries a dead gate unboundedly → job hangs, doesn't fail. + Supervise via `squeue` + driver log in + `code_snapshots/sc-swe-capture/logs/slurm/`. +- **Gate buffer growth uncapped** (H5): ids-only delta forests; at 131k × + in-flight concurrency raw ids are ~100s MB but Python overhead is real — + watch policy-model server RSS. +- **OpenHands marker survival unknown** — measured, not assumed + (`token_in_rate`); low rate degrades perf, never correctness. +- **Secrets in job env**: the launcher embeds `WANDB_API_KEY`/HF token in + the sbatch command (visible in SLURM logs; pre-existing behavior). + Rotate tokens if logs are shared. +- Snapshot is immutable but the **derived configs are re-generated per + submit** — do not edit the site yaml between arms of a pair. + +## 8. Pre-flight checklist (all verified 2026-07-28) + +- [x] S1–S5 + launcher committed (RL `e71055650`, Gym `e3b3eac6`) +- [x] Snapshot `sc-swe-capture` @ `e710556`: S5 files, `examples/swe_bench/` + launcher, site yaml, Gym gate + byte middleware present +- [x] Dry-run both arms end-to-end (`DRY_RUN=1`): correct account, snapshot + paths, singleton, reaper comment; derived-config diff = capture flag only +- [x] Container / model / data / sandbox images readable by pthombre +- [x] SLURM account `coreai_dlalgo_genai` valid for user (nemotron_sw_post is not) +- [x] Secrets profile in place; wrapper verified with tokens stripped from env +- [ ] SUBMIT (awaiting explicit go) diff --git a/swe/SWE_RUN.md b/swe/SWE_RUN.md new file mode 100644 index 00000000000..335eff4e6db --- /dev/null +++ b/swe/SWE_RUN.md @@ -0,0 +1,193 @@ +# SWE Run — Token-Capture vs Legacy Perf A/B at Scale + +Runbook for measuring the performance of the gate-authoritative token-capture +pipeline (`token_capture.enabled=true`, design: +`docs/design-docs/tq-gym-gate-authoritative.md`) against the legacy token-echo +path on a real workload: **async GRPO on SWE-bench** (Qwen3-30B-A3B thinking, +NeMo-Gym OpenHands agent, 16 nodes). This is the multi-turn run the 2-GPU S5 +experiments could not provide — SWE agents run up to 100–200 turns per +rollout, which is exactly where the legacy echo pays its per-turn token +re-transmission and where capture's `token_in_rate` and prefix serving are +finally exercised for real. + +Small-scale S5 results this run extends (implementation log, +`docs/design-docs/tq-gym-gate-authoritative-implementation-log.md` § S5): +−35.6 % HTTP bytes/trained token, −9 % step time, −16.8 % generation time on +a *single-turn* workload — the floor. The sync prototype measured −46.9 % +bytes/token multi-turn. + +--- + +## 1. The stack being launched + +``` +swe/launch_swe_ab.sh (this folder: picks the arm, derives the capture yaml) + └─ site wrapper e.g. .../nemo_rl-sc-test/test_assets/SWE/grpo_swe_tests.sh + (account, container, secrets, lustre paths — user-specific, gitignored) + └─ snapshot launcher /examples/swe_bench/run_grpo_qwen3_30b_async_swe.sh + (env-agnostic: parallelism, async_rl/data_plane overrides, sbatch) + └─ ray.sub → SLURM (16 nodes: 8 train TP4/EP8/CP4/PP2 + 8 gen vLLM TP2, + seqlen 131072, SC_MODE=1 single-controller + TQ) +``` + +Jobs run from an immutable **code snapshot** +(`test_assets/SWE/sync_code.sh`). The snapshot rsyncs `3rdparty/` wholesale, +so the **local-only Gym fork commits ride along** — the open gitlink/CI +question does not block this experiment. + +## 2. The A/B design + +One independent variable. Everything else — snapshot, container, data, model, +parallelism, async_rl knobs, node count — identical between arms. + +| | Arm A: `legacy` | Arm B: `capture` | +|---|---|---| +| Config | site yaml as-is | site yaml + `token_capture.enabled: true` (derived by `make_capture_config.py`) | +| Token path | gate string-parses ids + `/tokenize` per call; ids/logprobs echoed through agent messages every turn; tokens ride the Ray return | gate custodies lineage; worker stages delta ids+logprobs to TQ once; markers + token-free receipts | +| Rows into training | `TQReplayBuffer.commit` tensorize | `BlackboxFinalizer` verify → publish | + +Both arms use the launcher's defaults, which sit inside the capture MVP +matrix: `batch_selection_strategy=staleness_window`, age 1, +`mixed_weight_version_policy=allow` (default), async vLLM engine, +non-colocated generation. `make_capture_config.py` also pins +`rollout_max_attempts_to_avoid_lp_nan: 1` — capture setup hard-errors +otherwise (NaN-retry would re-register create-only rollout ids); it is set on +**both** arms' derived configs so retry behavior is not a confound. + +### Run protocol (perf reading) + +1. **Flag-off smoke first** (`MAX_NUM_STEPS=3`): proves the branch + Gym pin + reproduce the legacy SWE run before anything is compared. The standing + dormant-by-default discipline. +2. **Measurement runs**: `MAX_NUM_STEPS=20`–`30` per arm (long enough that + medians beat setup noise; short enough to iterate). `--dependency=singleton` + is already in the launcher — submit both arms back-to-back under the same + job name family so they land on comparable allocations. +3. **Repeats**: 1 pair = directional; 3 pairs = quotable (the S5 assessment: + run-to-run variance on this stack is plausibly ±5–10 %). + +## 3. Launching + +### Prerequisites (once) + +- The branch state you want measured is **committed** (RL repo and Gym + submodule) — snapshots are cut from a checkout. +- A checkout of `yukih/sc-entrypoint` (with the `tq-gate-capture` submodule + state) on lustre to snapshot from, e.g. clone + check out, or reuse an + existing sc-test workspace updated to this branch. +- A personal site wrapper (copy + `test_assets/SWE/grpo_swe_tests.sh` and edit ACCOUNT / CONTAINER / + secrets-profile / cache / W&B paths for your user). +- Snapshot it: + +```bash +cd # repo root, branch checked out +bash test_assets/SWE/sync_code.sh sc-swe-capture +``` + +### Submitting the arms + +```bash +# from this repo's swe/ folder +SITE_WRAPPER=/lustre/.../your_grpo_swe_tests.sh \ + MAX_NUM_STEPS=3 ARM=legacy bash swe/launch_swe_ab.sh # flag-off smoke + +SITE_WRAPPER=... MAX_NUM_STEPS=25 ARM=legacy bash swe/launch_swe_ab.sh +SITE_WRAPPER=... MAX_NUM_STEPS=25 ARM=capture bash swe/launch_swe_ab.sh + +# optional: HTTP byte accounting (writes per-server JSON to lustre) +SITE_WRAPPER=... MAX_NUM_STEPS=25 ARM=capture BYTES=1 bash swe/launch_swe_ab.sh +``` + +`launch_swe_ab.sh` does four things: derives the arm's config +(`make_capture_config.py` for the capture arm), stamps +`EXP_SUFFIX=swe-ab--...` so W&B separates the arms, forces the venv +posture below, and `exec`s your site wrapper (all launcher knobs still pass +through the environment: `TP`, `PPS`, `OVER_SAMPLING`, `DRY_RUN=1`, …). + +### Environment posture (why the wrapper forces these) + +- **`NRL_FORCE_REBUILD_VENVS=true` on BOTH arms.** The baked container's + `/opt/ray_venvs` predate this branch's `uv.lock` (S1 regenerated it), and + the capture arm additionally needs the `VLLM_GYM` worker venv + (`--extra vllm --extra nemo_gym`) that no image bakes yet. Forcing rebuild + on both arms keeps setup cost out of the A/B. Pre-warm `LUSTRE_UV_CACHE` + once (see the launcher header) or the first node-local build is ~30 min — + the 180-min idle-GPU reaper exemption in the site wrapper covers it. +- **`GYM_VENV_DIR=/opt/gym_venvs` (image-baked) on BOTH arms.** A node-local + `/tmp` rebuild does NOT work multi-node: the venv build runs only on the + NemoGym actor's node while Gym spawns servers cluster-wide (job 14542017 + failed on exactly this). The baked venvs are dependency-compatible with the + fork (verified: `swe_agents`/`vllm_model` requirements unchanged vs the old + pin; only the core aiohttp CVE floor moved, irrelevant at runtime), and the + editable install serves the fork's *code* on every node. If a future fork + commit does change server deps, rebake via + `test_assets/SWE/prebuild_gym_venvs.sh`. +- **Pre-warm `LUSTRE_UV_CACHE` for the branch's `uv.lock`** (single process, + inside the job container — the source builds need `nvcc`): without it the + 16-node venv rebuild exceeds the idle-GPU reaper's *de facto* 60-minute + kill (observed twice; the 180-min exemption comment is valid per the + exemption guide but not honored beyond 60 — escalate to + @job-reaper-support with jobs 14516316/14521677 if 180 is ever needed). + +## 4. What to compare (the perf read) + +All of these land in W&B (both arms) — the capture-arm `gate/*` block comes +from the per-step SC gate-metrics logging added in S5. + +**Headline perf (medians over steps 5..N, skipping warm-up):** + +| Metric (W&B key) | Expectation | +|---|---| +| `timing/train/total_step_time` | capture ≤ legacy; S5 saw −9 % single-turn | +| `timing/train/exposed_generation` | the mechanism lives here (no `/tokenize` round-trip, ~40 % smaller per-call payloads, no echo growth); S5 saw −16.8 % | +| `timing/train/valid_tokens_per_sec_per_gpu` | inverse of the above | + +**Capture-arm health / mechanism metrics:** + +| Metric | What it tells you | +|---|---| +| `gate/token_in_rate` | THE number this run finally measures. High (→1.0 for continuations) = OpenHands echoes markers faithfully and exact-prefix serving works at depth. Low = fallbacks (see next row) — runs stay *correct* but pay full re-renders. | +| `gate/fallback_no_marker` / `fallback_fingerprint_miss` / `fallback_unknown_marker` | Which § 3.3 fallback is firing if token_in_rate is low. `fingerprint_miss` at scale ⇒ the OpenHands history pipeline rewrites messages (report back — this is marker-survival risk #1 for agent frameworks). | +| `gate/capture_failed`, `train/global_valid_seqs` | staging failures → placeholder rows. valid_seqs should equal GBS on both arms. | +| `finalize/*` (rollout metrics) | finalizer latency rides the dispatch task (MVP placement) — SWE's 131k-token multi-call rollouts make this worth watching; if it dominates dispatch, that argues for the H5 finalizer pool. | + +**Training equivalence (guards the perf claim):** overlay `train/reward` +curves and `train/gen_kl_error` between arms — same band = the perf delta is +not bought with training drift. (S5: identical rewards 10/10, same KL band.) + +**Optional byte accounting (`BYTES=1`):** env-gated ASGI counters +(`NG_HTTP_BYTES_DIR` on every Gym server — the SWE config runs +`num_workers: 1`, which the counter supports — and `NRL_HTTP_BYTES_DIR` on +the vLLM workers) write per-server, per-route JSON to +`/http_bytes/`. Aggregate with `swe/aggregate_perf.py + ` → total bytes, bytes per trained token, per-hop +table. Expect the multi-turn reduction to land between the S5 floor +(−35.6 %) and beyond the prototype's −46.9 % as turn count grows. + +## 5. Known risks at this scale (accepted, watched) + +- **Gate death = silent stall, not loud failure** (S5 chaos finding, fix + queued for H1): Gym's control-plane client retries a dead gate unboundedly, + so if the policy-model server dies mid-run the job hangs rather than + failing. Supervise; `squeue` + driver log tell you which. +- **Gate buffer growth is uncapped** (H5 adds caps): per-rollout delta + forests, ids only. At seqlen 131k × concurrency 768 the raw ids are modest + (~100s of MB), but Python-object overhead is real — watch the policy-model + server RSS. +- **Marker survival through OpenHands** is the biggest unknown — it uses the + same message carrier the legacy token echo uses (so it *should* survive + wherever legacy works today), but `token_in_rate` is the proof either way, + and a low rate is a correctness-preserving perf bug, not a training bug. +- The idle-GPU reaper exemption and `checkpoint_must_save_by` are already + handled by the site wrapper / launcher. + +## 6. Files in this folder + +| File | Purpose | +|---|---| +| `SWE_RUN.md` | this runbook | +| `EXPERIMENT_LAUNCH.md` | pinned launch record for the 2026-07-28 A/B (exact commits, snapshot, gates, checklist) | +| `launch_swe_ab.sh` | arm selector: derives config, names the run, forces venv posture, delegates to your site wrapper | +| `make_capture_config.py` | derives the arm configs from the site yaml (`token_capture.enabled` + NaN-retry pin) | +| `aggregate_perf.py` | offline aggregation: per-hop HTTP bytes, bytes/trained-token, timing medians, token_in_rate | diff --git a/swe/aggregate_perf.py b/swe/aggregate_perf.py new file mode 100644 index 00000000000..c5490a60ebd --- /dev/null +++ b/swe/aggregate_perf.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Aggregate the instrumented legacy-vs-capture perf pair. + +Usage: python aggregate_perf.py + +Each run dir: http_bytes/*.json (per-server per-route byte counters), +metrics.json (TB dump), train_rows_*.jsonl (row dump for token counts). +""" + +import glob +import json +import os +import sys + + +def load_bytes(run_dir): + per_server = {} + for path in glob.glob(os.path.join(run_dir, "http_bytes", "*.json")): + name = os.path.basename(path).rsplit("_", 1)[0] + data = json.load(open(path)) + agg = per_server.setdefault(name, {}) + for route, c in data.items(): + e = agg.setdefault(route, {"requests": 0, "req_bytes": 0, "resp_bytes": 0}) + for k in e: + e[k] += c[k] + return per_server + + +def total_trained_tokens(run_dir): + total = 0 + for path in glob.glob(os.path.join(run_dir, "train_rows_*.jsonl")): + for line in open(path): + if line.strip(): + total += json.loads(line)["input_lengths"] + return total + + +def timing(run_dir): + d = json.load(open(os.path.join(run_dir, "metrics.json"))) + out = {} + for key in ( + "timing/train/total_step_time", + "timing/train/exposed_generation", + "timing/train/policy_training", + "timing/train/weight_sync", + "timing/train/valid_tokens_per_sec_per_gpu", + "train/gen_kl_error", + "train/reward", + "gate/token_in_rate", + "gate/token_in", + "gate/fallback_no_marker", + "gate/fallback_fingerprint_miss", + ): + s = d.get(key) + if isinstance(s, dict) and s: + vals = [v for _, v in sorted(s.items(), key=lambda kv: int(kv[0]))] + out[key] = vals + return out + + +def main(): + legacy_dir, capture_dir = sys.argv[1], sys.argv[2] + report = {} + for tag, run_dir in (("legacy", legacy_dir), ("capture", capture_dir)): + servers = load_bytes(run_dir) + toks = total_trained_tokens(run_dir) + grand = {"requests": 0, "req_bytes": 0, "resp_bytes": 0} + print(f"\n===== {tag} =====") + for name, routes in sorted(servers.items()): + s = {"requests": 0, "req_bytes": 0, "resp_bytes": 0} + for route, c in routes.items(): + for k in s: + s[k] += c[k] + for k in grand: + grand[k] += s[k] + print(f"{name:32s} req={s['requests']:6d} in={s['req_bytes']/1e6:8.2f} MB out={s['resp_bytes']/1e6:8.2f} MB") + for route, c in sorted(routes.items(), key=lambda kv: -(kv[1]["req_bytes"] + kv[1]["resp_bytes"]))[:4]: + print(f" {route:40s} n={c['requests']:5d} in={c['req_bytes']/1e6:7.2f} MB out={c['resp_bytes']/1e6:7.2f} MB") + total_bytes = grand["req_bytes"] + grand["resp_bytes"] + print(f"{'TOTAL':32s} req={grand['requests']:6d} bytes={total_bytes/1e6:.2f} MB trained_tokens={toks} bytes/token={total_bytes/max(toks,1):.1f}") + report[tag] = {"total_bytes": total_bytes, "tokens": toks, "timing": timing(run_dir)} + + lt, ct = report["legacy"], report["capture"] + print("\n===== comparison =====") + bt_l = lt["total_bytes"] / max(lt["tokens"], 1) + bt_c = ct["total_bytes"] / max(ct["tokens"], 1) + print(f"HTTP bytes/trained token: legacy {bt_l:.1f} -> capture {bt_c:.1f} ({(bt_c/bt_l-1)*100:+.1f}%)") + for key in ("timing/train/total_step_time", "timing/train/exposed_generation", "timing/train/valid_tokens_per_sec_per_gpu"): + lv, cv = lt["timing"].get(key), ct["timing"].get(key) + if lv and cv: + import statistics + ml, mc = statistics.median(lv), statistics.median(cv) + print(f"{key}: legacy median {ml:.3f} -> capture median {mc:.3f} ({(mc/ml-1)*100:+.1f}%)") + tir = ct["timing"].get("gate/token_in_rate") + if tir: + print(f"capture token_in_rate (cumulative, final): {tir[-1]:.3f}") + + +if __name__ == "__main__": + main() diff --git a/swe/launch_swe_ab.sh b/swe/launch_swe_ab.sh new file mode 100755 index 00000000000..aff2fc04f88 --- /dev/null +++ b/swe/launch_swe_ab.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +# SWE token-capture A/B arm launcher — see swe/SWE_RUN.md. +# +# Picks the arm, derives its config, names the run, forces the venv posture, +# then delegates to YOUR site wrapper (account/container/secrets live there). +# All launcher knobs (TP, PPS, MAX_NUM_STEPS, DRY_RUN=1, ...) pass through. +# +# Required: +# ARM=legacy|capture +# SITE_WRAPPER=/path/to/your grpo_swe_tests.sh-style wrapper +# Optional: +# BYTES=1 enable per-hop HTTP byte counters (JSON under the +# checkpoint dir; aggregate with swe/aggregate_perf.py) +# BASE_CONFIG site yaml to derive from (default: the wrapper's default, +# discovered via CONFIG_FILE after the wrapper sources it — +# set explicitly if your wrapper computes CONFIG_FILE late) +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ARM="${ARM:?set ARM=legacy|capture}" +SITE_WRAPPER="${SITE_WRAPPER:?set SITE_WRAPPER=/path/to/your site wrapper}" +[[ "${ARM}" == "legacy" || "${ARM}" == "capture" ]] || { echo "ARM must be legacy|capture" >&2; exit 1; } + +# ---- Derive the arm's config from the site yaml ----------------------------- +# Default BASE_CONFIG: the yaml sitting next to the site wrapper (the +# grpo_swe_tests.sh convention). Override for other layouts. +BASE_CONFIG="${BASE_CONFIG:-$(dirname "${SITE_WRAPPER}")/grpo_qwen3_30b_async_swe.yaml}" +[ -f "${BASE_CONFIG}" ] || { echo "BASE_CONFIG not found: ${BASE_CONFIG}" >&2; exit 1; } +DERIVED_DIR="${DERIVED_DIR:-$(dirname "${BASE_CONFIG}")/derived_configs}" +mkdir -p "${DERIVED_DIR}" +export CONFIG_FILE="${DERIVED_DIR}/grpo_swe_ab_${ARM}.yaml" +python3 "${SCRIPT_DIR}/make_capture_config.py" "${BASE_CONFIG}" "${CONFIG_FILE}" "${ARM}" + +# ---- Run naming (W&B separates the arms) ------------------------------------ +export EXP_SUFFIX="${EXP_SUFFIX:-swe-ab-${ARM}-$(date +%m%d%H%M)}" + +# ---- Venv posture: identical on both arms (see SWE_RUN.md § 3) -------------- +# Baked /opt/ray_venvs predate this branch's lock; the capture arm needs the +# unbaked VLLM_GYM worker venv. Forcing rebuild on both arms keeps setup cost +# out of the A/B. +export NRL_FORCE_REBUILD_VENVS=true +# Gym venvs: use the image-baked /opt/gym_venvs (present on EVERY node). +# A node-local GYM_VENV_DIR=/tmp/... does NOT work multi-node: the venv build +# runs only on the NemoGym actor's node while Gym spawns servers cluster-wide +# (learned from job 14542017). The fork's only dep-floor change vs the baked +# venvs is the aiohttp CVE bump (verified: swe_agents/vllm_model requirements +# unchanged), and editable installs serve the fork's *code* either way. +export GYM_VENV_DIR="${GYM_VENV_DIR:-/opt/gym_venvs}" + +# ---- Optional per-hop HTTP byte accounting ---------------------------------- +if [ "${BYTES:-0}" = "1" ]; then + BYTES_DIR="${BYTES_DIR:-${CHECKPOINT_ROOT:-$(dirname "${SITE_WRAPPER}")/../..}/http_bytes/${EXP_SUFFIX}}" + mkdir -p "${BYTES_DIR}" + export NG_HTTP_BYTES_DIR="${BYTES_DIR}" + export NRL_HTTP_BYTES_DIR="${BYTES_DIR}" + echo "[launch_swe_ab] byte counters on -> ${BYTES_DIR}" +fi + +echo "[launch_swe_ab] arm=${ARM} config=${CONFIG_FILE} exp=${EXP_SUFFIX}" +exec bash "${SITE_WRAPPER}" "$@" diff --git a/swe/make_capture_config.py b/swe/make_capture_config.py new file mode 100644 index 00000000000..5880a736655 --- /dev/null +++ b/swe/make_capture_config.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Derive the SWE A/B arm configs from a site yaml (see swe/SWE_RUN.md). + +Usage: + python swe/make_capture_config.py legacy|capture + +Both arms get `env.nemo_gym.rollout_max_attempts_to_avoid_lp_nan: 1` (the +capture arm hard-errors without it; pinning it on both keeps NaN-retry +behavior out of the A/B). The capture arm additionally gets +`token_capture.enabled: true` — the gate config injection into the Gym +policy-model server happens in code (`environments/nemo_gym.py:_spinup`), +so no other yaml change is needed. +""" + +import sys + +import yaml + + +def main() -> None: + src, dst, arm = sys.argv[1], sys.argv[2], sys.argv[3] + assert arm in ("legacy", "capture"), f"arm must be legacy|capture, got {arm}" + with open(src) as f: + config = yaml.safe_load(f) + + config.setdefault("env", {}).setdefault("nemo_gym", {})[ + "rollout_max_attempts_to_avoid_lp_nan" + ] = 1 + if arm == "capture": + config.setdefault("token_capture", {})["enabled"] = True + + with open(dst, "w") as f: + yaml.safe_dump(config, f, sort_keys=False) + print(f"[make_capture_config] arm={arm}: {src} -> {dst}") + + +if __name__ == "__main__": + main() diff --git a/tests/functional/L1_Functional_Tests_SingleController.sh b/tests/functional/L1_Functional_Tests_SingleController.sh new file mode 100755 index 00000000000..201ac4cf02a --- /dev/null +++ b/tests/functional/L1_Functional_Tests_SingleController.sh @@ -0,0 +1,46 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -xeuo pipefail # Exit immediately if a command exits with a non-zero status + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +PROJECT_ROOT=$(realpath ${SCRIPT_DIR}/../..) + +cd ${PROJECT_ROOT} + +# run_test [fast] +# - "run_test fast " = always runs (both fast and full modes) +# - "run_test " = only runs in full mode; skipped when FAST=1 +run_test() { + if [[ "$1" == "fast" ]]; then + shift + time "$@" + elif [[ "${FAST:-0}" == "1" ]]; then + echo "FAST: Skipping: $*" + else + time "$@" + fi +} + +run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh +# Token-capture (gate-authoritative) path: same SC+Gym smoke with the gate +# custodying token lineage and the finalizer publishing training rows. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true + +cd ${PROJECT_ROOT}/tests +if compgen -G ".coverage*" > /dev/null; then + coverage combine .coverage* +fi diff --git a/tests/functional/grpo_async_gym_single_controller.sh b/tests/functional/grpo_async_gym_single_controller.sh new file mode 100755 index 00000000000..f4f283bfffd --- /dev/null +++ b/tests/functional/grpo_async_gym_single_controller.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# SingleController + NeMo-Gym e2e smoke. Mirrors grpo_async_gym.sh but +# routes everything through the SC path (TransferQueue data plane + +# SingleControllerActor) instead of async_grpo_train. + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +CHECKPOINT_DIR=$EXP_DIR/checkpoints +DATA_DIR=$EXP_DIR/data +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR $CHECKPOINT_DIR $DATA_DIR + +# clean up checkpoint directory on exit +trap "rm -rf $CHECKPOINT_DIR" EXIT + +cd $PROJECT_ROOT + +# Follow nemo-gym instructions here to get this data: +# https://docs.nvidia.com/nemo/gym/0.1.0/tutorials/nemo-rl-grpo/setup.html#training-nemo-rl-grpo-setup +cd 3rdparty/Gym-workspace/Gym + +# We need HF_TOKEN to download the data from huggingface +if [[ ! -f env.yaml ]]; then + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "[ERROR] HF_TOKEN is not set" + exit 1 + fi + echo "hf_token: $HF_TOKEN" >> env.yaml +fi + +uv run ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml]" \ + +output_dirpath=data/workplace_assistant \ + +mode=train_preparation \ + +should_download=true \ + +data_source=huggingface +cd - + +# This trimming of the workplace assistant dataset is necessary b/c with all the tools the first prompt is >4000 tokens +# which will cause vllm to return nothing on the first prompt and crash RL. Since we want to keep this test short to +# smoke test, we trim all but the first tool +TRAIN_PATH=$DATA_DIR/workplace_assistant_train.jsonl +VALIDATION_PATH=$DATA_DIR/workplace_assistant_validation.jsonl +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/train.jsonl > $TRAIN_PATH +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/validation.jsonl > $VALIDATION_PATH + +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo_single_controller.py \ + --config $PROJECT_ROOT/examples/nemo_gym/grpo_qwen3_30ba3b_instruct.yaml \ + policy.model_name=Qwen/Qwen3-0.6B \ + policy.dtensor_cfg.enabled=false \ + policy.megatron_cfg.enabled=true \ + policy.megatron_cfg.tensor_model_parallel_size=1 \ + policy.megatron_cfg.pipeline_model_parallel_size=1 \ + policy.megatron_cfg.expert_model_parallel_size=1 \ + policy.megatron_cfg.context_parallel_size=1 \ + policy.megatron_cfg.sequence_parallel=false \ + policy.generation.vllm_cfg.tensor_parallel_size=1 \ + policy.generation.vllm_cfg.async_engine=true \ + policy.max_total_sequence_length=512 \ + policy.generation.colocated.enabled=false \ + policy.generation.colocated.resources.num_nodes=1 \ + policy.generation.colocated.resources.gpus_per_node=1 \ + grpo.num_prompts_per_step=4 \ + grpo.num_generations_per_prompt=2 \ + grpo.max_num_steps=10 \ + grpo.val_period=-1 \ + policy.train_global_batch_size=8 \ + policy.train_micro_batch_size=1 \ + cluster.gpus_per_node=2 \ + loss_fn.reference_policy_kl_penalty=0.01 \ + loss_fn.use_importance_sampling_correction=true \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=true \ + checkpointing.enabled=false \ + data.train.data_path=$TRAIN_PATH \ + data.validation.data_path=$VALIDATION_PATH \ + ++data_plane.enabled=true \ + ++data_plane.impl=transfer_queue \ + ++data_plane.backend=simple \ + ++data_plane.storage_capacity=1000000 \ + ++data_plane.num_storage_units=2 \ + ++data_plane.claim_meta_poll_interval_s=0.5 \ + ++data_plane.global_segment_size=549755813888 \ + ++data_plane.local_buffer_size=68719476736 \ + ++async_rl.batch_selection_strategy=strict_on_policy \ + ++async_rl.max_weight_staleness_versions=0 \ + ++async_rl.min_prompt_groups_per_batch=4 \ + ++async_rl.max_inflight_prompts=4 \ + ++async_rl.max_buffered_rollouts=4 \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Observed to be between 0.8-1.3 +uv run tests/check_metrics.py $JSON_METRICS \ + 'median(data["train/gen_kl_error"]) < 1.3' \ + 'max(data["train/reward"]) > 0' diff --git a/tests/functional/grpo_dp_single_controller.sh b/tests/functional/grpo_dp_single_controller.sh new file mode 100755 index 00000000000..abd590cc470 --- /dev/null +++ b/tests/functional/grpo_dp_single_controller.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Lightweight e2e for examples/run_grpo_single_controller.py — exercises +# setup_handle + setup_single_controller_component + SingleControllerActor +# end-to-end. Same shape as tests/functional/grpo_dp_simple.sh (Qwen3-0.6B, +# 2 GPUs, a handful of steps); data_plane.enabled=true is mandatory for SC. + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR + +cd $PROJECT_ROOT +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo_single_controller.py \ + policy.model_name=Qwen/Qwen3-0.6B \ + grpo.num_prompts_per_step=2 \ + grpo.num_generations_per_prompt=4 \ + policy.train_global_batch_size=8 \ + policy.train_micro_batch_size=1 \ + cluster.gpus_per_node=2 \ + grpo.max_num_steps=2 \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=false \ + checkpointing.enabled=false \ + data_plane.enabled=true \ + data_plane.impl=transfer_queue \ + data_plane.backend=simple \ + async_rl.batch_selection_strategy=strict_on_policy \ + async_rl.max_weight_staleness_versions=0 \ + async_rl.min_prompt_groups_per_batch=2 \ + async_rl.max_inflight_prompts=2 \ + async_rl.max_buffered_rollouts=2 \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/gen_kl_error"]) < 0.002' \ + 'min(data["train/probs_ratio_clamped_min"]) > 0.79' \ + 'max(data["train/probs_ratio_clamped_min"]) < 1.21' \ + 'min(data["train/probs_ratio_clamped_max"]) > 0.79' \ + 'max(data["train/probs_ratio_clamped_max"]) < 1.21' diff --git a/tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.sh b/tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.sh new file mode 100755 index 00000000000..e4c254c78f2 --- /dev/null +++ b/tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.sh @@ -0,0 +1,42 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=2 +STEPS_PER_RUN=100 +MAX_STEPS=100 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=240 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +# Run the experiment +cd $PROJECT_ROOT +uv run examples/run_grpo_single_controller.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=sc-yukih \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + $@ \ + 2>&1 | tee $RUN_LOG + +# Convert tensorboard logs to json +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Only run metrics if the target step is reached +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'median(data["train/token_mult_prob_error"]) < 1.1' \ + 'data["train/token_mult_prob_error"]["10"] < 1.1' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-fsdp2tp1.v3.sh b/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-fsdp2tp1.v3.sh index 353804958e5..298b6e0e9d3 100755 --- a/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-fsdp2tp1.v3.sh +++ b/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-fsdp2tp1.v3.sh @@ -17,13 +17,14 @@ cd $PROJECT_ROOT uv run examples/run_grpo.py \ --config $CONFIG_PATH \ grpo.max_num_steps=$MAX_STEPS \ + grpo.val_period=-1 \ logger.log_dir=$LOG_DIR \ logger.wandb_enabled=True \ - logger.wandb.project=nemo-rl \ + logger.wandb.project=sc-yukih \ logger.wandb.name=$EXP_NAME \ logger.monitor_gpus=True \ logger.tensorboard_enabled=True \ - checkpointing.enabled=True \ + checkpointing.enabled=false \ checkpointing.checkpoint_dir=$CKPT_DIR \ $@ \ 2>&1 | tee $RUN_LOG diff --git a/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.sh b/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.sh new file mode 100755 index 00000000000..978b8893ee5 --- /dev/null +++ b/tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.sh @@ -0,0 +1,43 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=1 +STEPS_PER_RUN=450 +MAX_STEPS=450 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=240 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +# Run the experiment +cd $PROJECT_ROOT +uv run examples/run_grpo_single_controller.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=sc-yukih \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + $@ \ + 2>&1 | tee $RUN_LOG + +# Convert tensorboard logs to json +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Only run metrics if the target step is reached +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'median(data["train/token_mult_prob_error"]) < 1.1' \ + 'data["train/token_mult_prob_error"]["450"] < 1.1' \ + 'mean(data["timing/train/total_step_time"], 2) < 25' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/llm/performance/grpo-llama3.1-8b-instruct-2n8g-async-1off.sh b/tests/test_suites/llm/performance/grpo-llama3.1-8b-instruct-2n8g-async-1off.sh index f9f561e3dcc..0555651b0af 100755 --- a/tests/test_suites/llm/performance/grpo-llama3.1-8b-instruct-2n8g-async-1off.sh +++ b/tests/test_suites/llm/performance/grpo-llama3.1-8b-instruct-2n8g-async-1off.sh @@ -4,10 +4,10 @@ source $SCRIPT_DIR/common.env # ===== BEGIN CONFIG ===== NUM_NODES=2 -STEPS_PER_RUN=10 -MAX_STEPS=10 +STEPS_PER_RUN=100 +MAX_STEPS=100 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up -NUM_MINUTES=100 +NUM_MINUTES=240 # ===== END CONFIG ===== exit_if_max_steps_reached @@ -17,13 +17,14 @@ cd $PROJECT_ROOT uv run examples/run_grpo.py \ --config $CONFIG_PATH \ grpo.max_num_steps=$MAX_STEPS \ + grpo.val_period=-1 \ logger.log_dir=$LOG_DIR \ logger.wandb_enabled=True \ - logger.wandb.project=nemo-rl \ + logger.wandb.project=sc-yukih \ logger.wandb.name=$EXP_NAME \ logger.monitor_gpus=True \ logger.tensorboard_enabled=True \ - checkpointing.enabled=True \ + checkpointing.enabled=false \ checkpointing.checkpoint_dir=$CKPT_DIR \ $@ \ 2>&1 | tee $RUN_LOG diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index 2c2fa417dd4..3653ae1a909 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -145,6 +145,10 @@ tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2-tq_mooncake.sh tests/test_suites/llm/grpo-qwen3-8B-base-1n8g-fsdp2-lora-tq_mooncake.sh tests/test_suites/llm/prorlv2-qwen2.5-math-1.5b-instruct-1n8g-fsdp2tp1.v2-tq_mooncake.sh +# Single Contoller (SC) +tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller.sh +tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller.sh + ######## # DAPO # ######## diff --git a/tests/unit/algorithms/test_advantage_validity.py b/tests/unit/algorithms/test_advantage_validity.py new file mode 100644 index 00000000000..64c1bc829b9 --- /dev/null +++ b/tests/unit/algorithms/test_advantage_validity.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""S4: validity-aware GRPO baseline (token-capture placeholder rows).""" + +import torch + +from nemo_rl.algorithms.advantage_estimator import GRPOAdvantageEstimator + + +def _estimator(**overrides) -> GRPOAdvantageEstimator: + config = {"use_leave_one_out_baseline": False, "normalize_rewards": False} + config.update(overrides) + return GRPOAdvantageEstimator(config, loss_config=None) + + +def test_invalid_rows_do_not_bias_the_baseline(): + prompt_ids = torch.zeros( + 4, 3, dtype=torch.long + ) # one shared prompt (2D, as prompt_ids_for_adv) + # The last row is a token-capture placeholder: reward 0, sample_mask 0. + rewards = torch.tensor([1.0, 3.0, 2.0, 0.0]) + valid_mask = torch.tensor([1.0, 1.0, 1.0, 0.0]) + mask = torch.ones(4, 5) + + adv = _estimator().compute_advantage( + prompt_ids, rewards, mask, valid_mask=valid_mask + ) + # Baseline over valid rows only: mean(1,3,2) = 2 (placeholder's 0 excluded). + assert torch.allclose(adv[0], torch.full((5,), -1.0)) + assert torch.allclose(adv[1], torch.full((5,), 1.0)) + assert torch.allclose(adv[2], torch.full((5,), 0.0)) + + +def test_none_valid_mask_keeps_legacy_all_valid_behavior(): + prompt_ids = torch.zeros(2, 3, dtype=torch.long) + rewards = torch.tensor([1.0, 3.0]) + mask = torch.ones(2, 3) + legacy = _estimator().compute_advantage(prompt_ids, rewards, mask) + explicit = _estimator().compute_advantage( + prompt_ids, rewards, mask, valid_mask=torch.ones(2) + ) + assert torch.equal(legacy, explicit) diff --git a/tests/unit/algorithms/test_async_utils.py b/tests/unit/algorithms/test_async_utils.py index b7cff7eecd7..127fc3dd25d 100644 --- a/tests/unit/algorithms/test_async_utils.py +++ b/tests/unit/algorithms/test_async_utils.py @@ -33,10 +33,7 @@ AsyncTrajectoryCollector, ReplayBuffer, ) -from nemo_rl.algorithms.async_utils.replay_buffer import ( - ReplayBufferImpl, - ReplayBufferNew, -) +from nemo_rl.algorithms.async_utils.replay_buffer import ReplayBufferImpl from nemo_rl.algorithms.grpo import ( MasterConfig, add_grpo_token_loss_masks_and_generation_logprobs, @@ -920,160 +917,6 @@ def test_replay_buffer_checkpoint_with_torch_save(self): ray.kill(buffer2) -class TestReplayBufferNew: - """Tests for ReplayBufferNew: staleness-window sampling via _evict + sample.""" - - def _make_traj(self, label: str) -> dict: - return {"batch": {"data": label}, "rollout_metrics": {}} - - def _add(self, buf, label: str, weight_version: int): - return ray.get( - buf.add.remote( - self._make_traj(label), - weight_version=weight_version, - target_weight_version=0, # unused in ReplayBufferNew - ) - ) - - def _sample(self, buf, num_groups: int, trainer_version: int): - return ray.get( - buf.sample.remote( - num_prompt_groups=num_groups, - current_weight_version=trainer_version, - max_age_steps=0, # unused in ReplayBufferNew - ) - ) - - # ------------------------------------------------------------------ - # Construction - # ------------------------------------------------------------------ - - def test_invalid_max_staleness_raises(self): - with pytest.raises(Exception): - buf = ReplayBufferNew.remote(max_size=10, max_staleness=-1) - ray.get(buf.size.remote()) - - # ------------------------------------------------------------------ - # _evict (via sample) - # ------------------------------------------------------------------ - - def test_stale_rows_evicted_before_sampling(self): - """Rows with age > max_staleness are removed before sample() selects.""" - buf = ReplayBufferNew.remote(max_size=10, max_staleness=2) - # age at trainer=4: gen_v=1 → 3 > 2 (stale), gen_v=3 → 1 ≤ 2 (valid) - self._add(buf, "stale", weight_version=1) - self._add(buf, "fresh", weight_version=3) - - result = self._sample(buf, num_groups=1, trainer_version=4) - - assert result is not None - assert result["trajectories"][0]["batch"]["data"] == "fresh" - assert ray.get(buf.size.remote()) == 0 # stale row also gone - ray.kill(buf) - - def test_all_stale_returns_none(self): - """sample() returns None when all rows are evicted as stale.""" - buf = ReplayBufferNew.remote(max_size=10, max_staleness=1) - self._add(buf, "a", weight_version=0) - self._add(buf, "b", weight_version=1) - - # trainer=5: both ages > 1 - result = self._sample(buf, num_groups=1, trainer_version=5) - - assert result is None - assert ray.get(buf.size.remote()) == 0 - ray.kill(buf) - - def test_eviction_frees_capacity(self): - """Evicting a stale row allows a subsequent add() to succeed.""" - buf = ReplayBufferNew.remote(max_size=1, max_staleness=1) - self._add(buf, "x", weight_version=1) - assert self._add(buf, "x", weight_version=1) == "full" - - # sample() at trainer=5 evicts the stale row (age 4 > 1) - self._sample(buf, num_groups=1, trainer_version=5) - - assert self._add(buf, "y", weight_version=4) == "success" - ray.kill(buf) - - def test_within_window_not_evicted(self): - """Rows whose age is within max_staleness are not evicted.""" - buf = ReplayBufferNew.remote(max_size=10, max_staleness=3) - self._add(buf, "x", weight_version=4) - - # trainer=6: age = 6 - 4 = 2 ≤ 3 → should survive - # should return None since there is only 1 row - result = self._sample(buf, num_groups=2, trainer_version=6) - assert result is None - - # this sample should still be there - assert ray.get(buf.size.remote()) == 1 - ray.kill(buf) - - # ------------------------------------------------------------------ - # sample() - # ------------------------------------------------------------------ - - @pytest.mark.parametrize("sample_freshest_first", [True, False]) - def test_sample_freshest_first(self, sample_freshest_first): - """sample() returns the freshest trajectories first.""" - buf = ReplayBufferNew.remote( - max_size=10, max_staleness=5, sample_freshest_first=sample_freshest_first - ) - for gen_v in [3, 4, 5]: - self._add(buf, f"v{gen_v}", weight_version=gen_v) - - result = self._sample(buf, num_groups=2, trainer_version=6) - - assert result is not None - data = [t["batch"]["data"] for t in result["trajectories"]] - if sample_freshest_first: - assert data == ["v5", "v4"] - else: - assert data == ["v3", "v4"] - ray.kill(buf) - - def test_sample_returns_none_when_insufficient(self): - """sample() returns None when fewer rows than requested remain after eviction.""" - buf = ReplayBufferNew.remote(max_size=10, max_staleness=5) - self._add(buf, "only", weight_version=1) - - result = self._sample(buf, num_groups=3, trainer_version=2) - - assert result is None - ray.kill(buf) - - def test_sample_returns_none_on_empty_buffer(self): - buf = ReplayBufferNew.remote(max_size=10, max_staleness=5) - result = self._sample(buf, num_groups=1, trainer_version=1) - assert result is None - ray.kill(buf) - - def test_sample_avg_trajectory_age(self): - """avg_trajectory_age is computed from the sampled generation versions.""" - buf = ReplayBufferNew.remote(max_size=10, max_staleness=5) - # freshest first: gen 8 (age 2), gen 6 (age 4) → avg = 3.0 - for gen_v in [6, 8]: - self._add(buf, f"v{gen_v}", weight_version=gen_v) - - result = self._sample(buf, num_groups=2, trainer_version=10) - - assert result is not None - assert abs(result["avg_trajectory_age"] - 3.0) < 1e-6 - ray.kill(buf) - - def test_sample_consumes_selected_rows(self): - """Rows returned by sample() are removed from the buffer.""" - buf = ReplayBufferNew.remote(max_size=10, max_staleness=5) - for gen_v in [1, 2, 3]: - self._add(buf, f"v{gen_v}", weight_version=gen_v) - - self._sample(buf, num_groups=2, trainer_version=4) - - assert ray.get(buf.size.remote()) == 1 - ray.kill(buf) - - class TestAsyncTrajectoryCollector: """Test cases for AsyncTrajectoryCollector.""" diff --git a/tests/unit/data_plane/test_blackbox_finalizer.py b/tests/unit/data_plane/test_blackbox_finalizer.py new file mode 100644 index 00000000000..58ecdae854c --- /dev/null +++ b/tests/unit/data_plane/test_blackbox_finalizer.py @@ -0,0 +1,247 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""S4: BlackboxFinalizer against a live TQ simple backend. + +Drives the S1 golden call sequences end to end: stage the fixture's delta +rows via TQTokenSink, hand the fixture receipt to the finalizer, and require +the published canonical rows to match the fixture's frozen training row. +Every rejection path (missing rows, digest corruption, poisoned receipts) +must yield a masked placeholder — always N rows — and the group publisher's +min/max weight versions and staging cleanup must hold. + +Marked nemo_gym (run with ``--nemo-gym-only``): the finalizer delegates +rebuild semantics to Gym's staging package. +""" + +from __future__ import annotations + +import pytest +import torch + +nemo_gym = pytest.importorskip("nemo_gym.token_id_capture.staging") + +from nemo_gym.token_id_capture.staging.conformance.kit import ( # noqa: E402 + build_fixture_artifacts, + f32, + load_fixture, +) + +from nemo_rl.data_plane.tq_token_sink import ( # noqa: E402 + STAGING_FIELDS, + TQTokenSink, +) +from nemo_rl.experience.blackbox_finalizer import BlackboxFinalizer # noqa: E402 + +pytestmark = pytest.mark.nemo_gym + +STAGING_PARTITION = "rollout_staging_fin_test" +CANONICAL_PARTITION = "rollout_data_fin_test" +PAD = 0 + + +@pytest.fixture() +def partitions(tq_client): + tq_client.register_partition( + partition_id=STAGING_PARTITION, + fields=list(STAGING_FIELDS), + num_samples=64, + consumer_tasks=["finalize"], + ) + tq_client.register_partition( + partition_id=CANONICAL_PARTITION, + fields=[ + "input_ids", + "input_lengths", + "generation_logprobs", + "token_mask", + "sample_mask", + "prompt_ids_for_adv", + "total_reward", + ], + num_samples=64, + consumer_tasks=["train"], + ) + yield + tq_client.clear_samples(sample_ids=None, partition_id=STAGING_PARTITION) + tq_client.clear_samples(sample_ids=None, partition_id=CANONICAL_PARTITION) + + +def _finalizer(tq_client, **overrides) -> BlackboxFinalizer: + kwargs = dict( + partition_id=CANONICAL_PARTITION, + staging_partition=STAGING_PARTITION, + pad_token_id=PAD, + mixed_weight_version_policy="allow", + min_valid_fraction_per_group=None, + ) + kwargs.update(overrides) + return BlackboxFinalizer(tq_client, **kwargs) + + +def _stage_fixture(tq_client, name: str, *, rollout_id: str | None = None): + """Stage one golden fixture's rows (optionally re-keyed to rollout_id) + and return (receipt_dict, expected LinearizedRow).""" + fixture = load_fixture(name) + if rollout_id is not None: + fixture = dict(fixture) + fixture["rollout_id"] = rollout_id + records, _, receipt, row = build_fixture_artifacts(fixture) + sink = TQTokenSink(tq_client, staging_partition=STAGING_PARTITION) + for record in records: + assert sink.stage(record).ok + return receipt.model_dump(), row + + +def test_finalize_rollout_reproduces_the_golden_row(tq_client, partitions): + receipt, expected = _stage_fixture(tq_client, "worked_example") + finalizer = _finalizer(tq_client) + row = finalizer.finalize_rollout("g7_r0", receipt, reward=1.0) + assert row.valid, row.rejection_reason + assert row.token_ids == expected.token_ids + assert row.token_mask == [f32(m) for m in expected.token_mask] + assert row.logprobs == [f32(p) for p in expected.logprobs] + assert row.prompt_len == expected.prompt_len + # The worked example spans a single weight version (wv 4 throughout). + assert (row.min_wv, row.max_wv) == (4, 4) + + +def test_finalize_rollout_rejections(tq_client, partitions): + finalizer = _finalizer(tq_client) + assert ( + finalizer.finalize_rollout("r", None, reward=0.0).rejection_reason + == "missing_receipt" + ) + + receipt, _ = _stage_fixture(tq_client, "single_call", rollout_id="rej_a") + poisoned = dict(receipt, capture_poisoned=True) + assert ( + finalizer.finalize_rollout("rej_a", poisoned, reward=0.0).rejection_reason + == "capture_poisoned" + ) + empty = dict(receipt, manifest=[], terminal_call_id=None) + assert ( + finalizer.finalize_rollout("rej_a", empty, reward=0.0).rejection_reason + == "empty_manifest" + ) + wrong_identity = finalizer.finalize_rollout("someone_else", receipt, reward=0.0) + assert (wrong_identity.rejection_reason or "").startswith("identity_mismatch") + + # A manifest naming rows that were never staged. + ghost = dict(receipt) + ghost["manifest"] = [ + {**entry, "staging_key": "ghost/row"} for entry in receipt["manifest"] + ] + missing = finalizer.finalize_rollout("rej_a", ghost, reward=0.0) + assert (missing.rejection_reason or "").startswith("missing_staging_row") + + # Digest corruption: break the manifest digest so recomputation misses. + corrupted = dict(receipt) + corrupted["manifest"] = [ + {**entry, "digest": "0" * 64} for entry in receipt["manifest"] + ] + bad = finalizer.finalize_rollout("rej_a", corrupted, reward=0.0) + assert (bad.rejection_reason or "").startswith("digest_mismatch") + + +def test_mixed_weight_version_policy_reject(tq_client, partitions): + receipt, _ = _stage_fixture(tq_client, "mixed_weight_versions", rollout_id="mix_r0") + receipt["rollout_id"] = "mix_r0" + allow_row = _finalizer(tq_client).finalize_rollout("mix_r0", receipt, reward=0.0) + assert allow_row.valid + assert allow_row.min_wv < allow_row.max_wv + reject_row = _finalizer( + tq_client, mixed_weight_version_policy="reject" + ).finalize_rollout("mix_r0", receipt, reward=0.0) + assert (reject_row.rejection_reason or "").startswith("mixed_weight_versions") + + +def _fetch_rows(tq_client, sample_ids): + return tq_client.get_samples( + sample_ids=sample_ids, + partition_id=CANONICAL_PARTITION, + select_fields=[ + "input_ids", + "input_lengths", + "generation_logprobs", + "token_mask", + "sample_mask", + "prompt_ids_for_adv", + "total_reward", + ], + ) + + +def test_finalize_group_publishes_n_rows_with_placeholder(tq_client, partitions): + group_id = "grp1" + receipt, expected = _stage_fixture( + tq_client, "worked_example", rollout_id=f"{group_id}_g0" + ) + receipt["rollout_id"] = f"{group_id}_g0" + rollout_ids = [f"{group_id}_g0", f"{group_id}_g1"] + + finalizer = _finalizer(tq_client) + finalized = finalizer.finalize_group( + group_id, + rollout_ids, + [receipt, None], # second rollout lost its receipt -> placeholder + [1.0, 0.0], + fallback_weight_version=9, + ) + assert not finalized.dropped + assert finalized.meta is not None + assert finalized.meta.sample_ids == rollout_ids + # Group staleness comes from the valid rollout's calls (wv 4), not the fallback. + assert (finalized.group_min_wv, finalized.group_max_wv) == (4, 4) + assert finalized.metrics["finalize/invalid_row_rate"] == 0.5 + + rows = _fetch_rows(tq_client, rollout_ids) + sample_mask = torch.as_tensor(rows["sample_mask"]).flatten() + assert sample_mask.tolist() == [1.0, 0.0] + valid_len = len(expected.token_ids) + input_ids = torch.as_tensor(rows["input_ids"][0]).flatten() + assert input_ids[:valid_len].tolist() == expected.token_ids + # Placeholder borrows the valid sibling's prompt for baseline grouping. + prompt = expected.token_ids[: expected.prompt_len] + adv_prompt_valid = torch.as_tensor(rows["prompt_ids_for_adv"][0]).flatten() + adv_prompt_placeholder = torch.as_tensor(rows["prompt_ids_for_adv"][1]).flatten() + assert adv_prompt_valid.tolist() == prompt + assert adv_prompt_placeholder.tolist() == prompt + placeholder_mask = torch.as_tensor(rows["token_mask"][1]).flatten() + assert placeholder_mask.sum().item() == 0.0 + rewards = torch.as_tensor(rows["total_reward"]).flatten() + assert rewards.tolist() == [1.0, 0.0] + + # The finalizer cleared its staged rows after publishing. + with pytest.raises(KeyError): + finalizer._source.fetch([receipt["manifest"][0]["staging_key"]]) + + +def test_finalize_group_min_valid_fraction_drops(tq_client, partitions): + group_id = "grp2" + rollout_ids = [f"{group_id}_g0", f"{group_id}_g1"] + finalizer = _finalizer(tq_client, min_valid_fraction_per_group=0.5) + finalized = finalizer.finalize_group( + group_id, + rollout_ids, + [None, None], + [0.0, 0.0], + fallback_weight_version=3, + ) + assert finalized.dropped + assert finalized.meta is None + assert (finalized.group_min_wv, finalized.group_max_wv) == (3, 3) + with pytest.raises((KeyError, RuntimeError, ValueError)): + rows = _fetch_rows(tq_client, rollout_ids) + assert not rows # nothing published diff --git a/tests/unit/data_plane/test_kvbatchmeta.py b/tests/unit/data_plane/test_kvbatchmeta.py index 4774c44f1e3..a8dc3bc822d 100644 --- a/tests/unit/data_plane/test_kvbatchmeta.py +++ b/tests/unit/data_plane/test_kvbatchmeta.py @@ -147,6 +147,96 @@ def test_tags_travel_with_subset_slice_concat(): assert joined.tags == m.tags +def test_drop_removes_indices_and_keeps_remaining_rows(): + """Rows at ``indices`` disappear; survivors keep their tags/seqlens.""" + m = KVBatchMeta( + partition_id="p", + task_name="t", + sample_ids=["a", "b", "c", "d"], + sequence_lengths=[1, 2, 3, 4], + tags=[{"i": 0}, {"i": 1}, {"i": 2}, {"i": 3}], + ) + remaining = m.drop([1, 3]) + assert remaining.sample_ids == ["a", "c"] + assert remaining.sequence_lengths == [1, 3] + assert remaining.tags == [{"i": 0}, {"i": 2}] + + +def test_drop_empty_indices_returns_identical_content(): + """``drop([])`` returns a fresh copy with the same rows.""" + m = KVBatchMeta( + partition_id="p", + task_name="t", + sample_ids=["a", "b"], + sequence_lengths=[1, 2], + ) + out = m.drop([]) + assert out.sample_ids == m.sample_ids + assert out.sequence_lengths == m.sequence_lengths + assert out is not m + + +def test_drop_all_returns_none(): + """Dropping every row returns ``None`` (callers chain into ``is None`` branches).""" + m = KVBatchMeta( + partition_id="p", + task_name="t", + sample_ids=["a", "b"], + tags=[{"i": 0}, {"i": 1}], + ) + assert m.drop([0, 1]) is None + + +def test_drop_indices_can_be_unordered_or_duplicated(): + """Duplicate / out-of-order indices collapse via the internal set.""" + m = KVBatchMeta( + partition_id="p", + task_name="t", + sample_ids=["a", "b", "c"], + ) + assert m.drop([2, 0, 0]).sample_ids == ["b"] + + +def test_with_fields_appends_and_dedupes_preserving_order(): + """Merges into ``fields`` with order-preserving dedup.""" + m = KVBatchMeta( + partition_id="p", + task_name="t", + sample_ids=["a"], + fields=["input_ids", "advantages"], + ) + out = m.with_fields(["advantages", "logprobs"]) + assert out.fields == ["input_ids", "advantages", "logprobs"] + + +def test_with_fields_initializes_from_none(): + """Seeds ``fields`` from the argument when previously ``None``.""" + m = KVBatchMeta(partition_id="p", task_name="t", sample_ids=["a"]) + out = m.with_fields(["x", "y", "x"]) + assert out.fields == ["x", "y"] + + +def test_with_fields_returns_independent_copy(): + """Returned meta does not share mutable references with the source.""" + m = KVBatchMeta( + partition_id="p", + task_name="t", + sample_ids=["a", "b"], + sequence_lengths=[1, 2], + extra_info={"step": 0}, + tags=[{"x": 1}, {"x": 2}], + ) + out = m.with_fields(["z"]) + out.sample_ids.append("c") + out.sequence_lengths.append(3) # type: ignore[union-attr] + out.extra_info["step"] = 99 + out.tags[0]["x"] = 999 # type: ignore[index] + assert m.sample_ids == ["a", "b"] + assert m.sequence_lengths == [1, 2] + assert m.extra_info == {"step": 0} + assert m.tags == [{"x": 1}, {"x": 2}] + + def test_tags_none_when_either_side_missing_in_concat(): """``concat`` drops tags if either side has none — symmetric with the ``sequence_lengths`` behavior.""" diff --git a/tests/unit/data_plane/test_tq_token_sink.py b/tests/unit/data_plane/test_tq_token_sink.py new file mode 100644 index 00000000000..d543756a084 --- /dev/null +++ b/tests/unit/data_plane/test_tq_token_sink.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TQTokenSink / TQTokenSource against a live TQ backend. + +Runs NeMo-Gym's installable conformance kit (golden call sequences → +byte-exact digests, manifests, and linearized rows) over the TransferQueue +implementations — the framework-CI half of the § 3.0 contract — plus the +protocol edges the kit does not cover (missing keys, stage failure shape). +""" + +from __future__ import annotations + +import pytest + +nemo_gym = pytest.importorskip("nemo_gym.token_id_capture.staging") + +from nemo_gym.token_id_capture.staging.conformance import ( # noqa: E402 + build_fixture_artifacts, + fixture_names, + load_fixture, + run_sink_source_conformance, +) +from nemo_gym.token_id_capture.staging.protocols import ( # noqa: E402 + StagingSink as TokenSinkProtocol, +) +from nemo_gym.token_id_capture.staging.protocols import ( # noqa: E402 + StagingSource as TokenSourceProtocol, +) + +from nemo_rl.data_plane.tq_token_sink import ( # noqa: E402 + STAGING_FIELDS, + TQTokenSink, + TQTokenSource, +) + +STAGING_PARTITION = "rollout_staging_test" + +pytestmark = pytest.mark.nemo_gym + + +@pytest.fixture() +def staging_partition(tq_client): + tq_client.register_partition( + partition_id=STAGING_PARTITION, + fields=list(STAGING_FIELDS), + num_samples=64, + consumer_tasks=["finalize"], + ) + yield STAGING_PARTITION + tq_client.clear_samples(sample_ids=None, partition_id=STAGING_PARTITION) + + +def test_implementations_satisfy_protocols(tq_client, staging_partition): + sink = TQTokenSink(tq_client, staging_partition=staging_partition) + source = TQTokenSource(tq_client, staging_partition=staging_partition) + assert isinstance(sink, TokenSinkProtocol) + assert isinstance(source, TokenSourceProtocol) + + +@pytest.mark.parametrize( + "fixture_name", ["worked_example", "single_call", "mixed_weight_versions"] +) +def test_tq_sink_source_passes_conformance(tq_client, staging_partition, fixture_name): + assert fixture_name in fixture_names() + sink = TQTokenSink(tq_client, staging_partition=staging_partition) + source = TQTokenSource(tq_client, staging_partition=staging_partition) + run_sink_source_conformance(load_fixture(fixture_name), sink, source) + + +def test_fetch_missing_key_raises_keyerror(tq_client, staging_partition): + source = TQTokenSource(tq_client, staging_partition=staging_partition) + with pytest.raises(KeyError): + source.fetch(["ghost_rollout/ghost_call"]) + + +def test_stage_failure_reports_not_raises(staging_partition): + class ExplodingClient: + def put_samples(self, **kwargs): + raise RuntimeError("controller down") + + sink = TQTokenSink(ExplodingClient(), staging_partition=staging_partition) + records, _, _, _ = build_fixture_artifacts(load_fixture("single_call")) + result = sink.stage(records[0]) + assert not result.ok + assert result.staging_key == records[0].staging_key + assert "controller down" in (result.error or "") + + +def test_sink_clear_drops_rows(tq_client, staging_partition): + sink = TQTokenSink(tq_client, staging_partition=staging_partition) + source = TQTokenSource(tq_client, staging_partition=staging_partition) + records, _, _, _ = build_fixture_artifacts(load_fixture("single_call")) + for record in records: + assert sink.stage(record).ok + keys = [record.staging_key for record in records] + assert len(source.fetch(keys)) == len(keys) + sink.clear(keys) + with pytest.raises(KeyError): + source.fetch(keys) diff --git a/tests/unit/experience/test_rollout_manager.py b/tests/unit/experience/test_rollout_manager.py new file mode 100644 index 00000000000..47b2b957451 --- /dev/null +++ b/tests/unit/experience/test_rollout_manager.py @@ -0,0 +1,1013 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for RolloutManager. + +Two groups: + +* TestGenerateAndPushFlow — lightweight unit tests for the reserve→run→commit + flow in generate_and_push (no Ray/vLLM; fakes for impl + tq_buffer). +* AsyncRollout / AsyncNemoGymRollout tests — vLLM/Ray-backed end-to-end checks + for the underlying run_rollout paths (AsyncRolloutImpl / AsyncNemoGymRolloutImpl). +""" + +from __future__ import annotations + +import asyncio +import json +import tempfile +import uuid +from copy import deepcopy + +import pytest +import torch + +from nemo_rl.data.collate_fn import rl_collate_fn +from nemo_rl.data.datasets.response_datasets import NemoGymDataset +from nemo_rl.data.interfaces import DatumSpec +from nemo_rl.data.processors import nemo_gym_data_processor +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.experience.interfaces import Completion, PromptGroupRecord +from nemo_rl.experience.rollout_manager import RolloutManager +from nemo_rl.experience.rollouts import ( + run_async_multi_turn_rollout, + run_async_nemo_gym_rollout, +) + +# Fixtures shared with the heavyweight rollout tests. +from tests.unit.environments.test_nemo_gym import ( + cluster, # noqa: F401 + nemo_gym, # noqa: F401 + nemo_gym_sanity_test_data, # noqa: F401 + nemo_gym_tokenizer, # noqa: F401 + nemo_gym_vllm_generation, # noqa: F401 +) +from tests.unit.experience.test_rollouts import ( + initial_multi_step_calculator_batch, # noqa: F401 + multi_step_calculator_environment, # noqa: F401 + multi_step_setup_vllm_async, # noqa: F401 + rollout_cluster, # noqa: F401 + rollout_tokenizer, # noqa: F401 +) +from tests.unit.test_envs import MultiStepCalcMetadata + + +def _run(coro): + return asyncio.run(coro) + + +class _FakeBuffer: + """Minimal TQReplayBuffer stand-in that records reserve/commit calls.""" + + def __init__(self) -> None: + self.reserve_calls: list[int] = [] # weight_versions passed to reserve + self.commit_calls: list[tuple[str, object, int, int]] = [] + self.abort_calls: list[str] = [] + # reserve(weight_version=X) -> group_id; commit fills the slot. + self._slots: list[str] = [] + + def reserve( + self, + *, + weight_version: int, + target_step: int | None = None, + group_id: str | None = None, + rollout_ids: list[str] | None = None, + ) -> str: + del target_step, rollout_ids + if group_id is None: + group_id = str(uuid.uuid4()) + self.reserve_calls.append(weight_version) + self._slots.append(group_id) + return group_id + + def abort(self, group_id: str) -> bool: + self.abort_calls.append(group_id) + if group_id in self._slots: + self._slots.remove(group_id) + return True + return False + + async def commit( + self, + group_id: str, + record, + start_weight_version: int, + end_weight_version: int, + ): + self.commit_calls.append( + (group_id, record, start_weight_version, end_weight_version) + ) + return record + + +class _FakeImpl: + """Stand-in for AsyncRolloutImpl that returns a sentinel record.""" + + def __init__(self, record="sentinel-record", on_run=None) -> None: + self._record = record + self._on_run = on_run + + async def run_rollout(self, input_sample): + if self._on_run is not None: + await self._on_run(input_sample) + return self._record + + +def _make_manager(buffer: _FakeBuffer, impl: _FakeImpl) -> RolloutManager: + """Build a RolloutManager without firing the real __init__.""" + mgr = object.__new__(RolloutManager) + mgr._impl = impl + mgr._tokenizer = None + mgr._num_generations_per_prompt = 1 + mgr._tq_buffer = buffer + mgr._finalizer = None + mgr._env_handles = {} + mgr._weight_version = 0 + return mgr + + +class TestGenerateAndPushFlow: + def test_reserves_then_runs_then_commits(self): + events: list[str] = [] + buf = _FakeBuffer() + + async def _track_run(_sample): + events.append("run") + + impl = _FakeImpl(record="r0", on_run=_track_run) + mgr = _make_manager(buf, impl) + + # Wrap reserve/commit to log ordering. + original_reserve = buf.reserve + original_commit = buf.commit + + def _logged_reserve(**kwargs): + events.append("reserve") + return original_reserve(**kwargs) + + async def _logged_commit(*args, **kwargs): + events.append("commit") + return await original_commit(*args, **kwargs) + + buf.reserve = _logged_reserve # type: ignore[method-assign] + buf.commit = _logged_commit # type: ignore[method-assign] + + _run(mgr.generate_and_push({"prompt": "p"})) + + assert events == ["reserve", "run", "commit"] + assert buf.reserve_calls == [0] + assert len(buf.commit_calls) == 1 + gid, record, start_v, end_v = buf.commit_calls[0] + assert gid in buf._slots + assert record == "r0" + assert start_v == 0 + assert end_v == 0 + + def test_start_weight_version_pinned_at_reserve_time(self): + """If set_weight_version is called mid-rollout, start != end.""" + buf = _FakeBuffer() + + async def _bump_weight_mid_rollout(_sample): + # Simulate a sync_weights bump during the rollout. + mgr.set_weight_version(5) + + impl = _FakeImpl(record="r0", on_run=_bump_weight_mid_rollout) + mgr = _make_manager(buf, impl) + mgr.set_weight_version(3) + + _run(mgr.generate_and_push({"prompt": "p"})) + + # reserve happened before run_rollout → captured weight 3. + assert buf.reserve_calls == [3] + # commit's start is the same dispatch-time value; end reflects the post-rollout weight. + _, _, start_v, end_v = buf.commit_calls[0] + assert start_v == 3 + assert end_v == 5 + + def test_no_weight_change_means_start_equals_end(self): + buf = _FakeBuffer() + impl = _FakeImpl(record="r0") + mgr = _make_manager(buf, impl) + mgr.set_weight_version(7) + + _run(mgr.generate_and_push({"prompt": "p"})) + + _, _, start_v, end_v = buf.commit_calls[0] + assert start_v == 7 + assert end_v == 7 + + def test_concurrent_dispatch_preserves_reserve_order(self): + """Two concurrent generate_and_push calls must reserve before either commits. + + The contract: reserve order == dispatch order, even if rollouts finish + out of order. Slot order in the buffer reflects the order reserve was + called (not the order run_rollout completed). + """ + buf = _FakeBuffer() + + # First call's rollout blocks until second call has reserved. + first_reserved = asyncio.Event() + second_reserved = asyncio.Event() + + async def _first_run(_sample): + first_reserved.set() + await second_reserved.wait() + + async def _second_run(_sample): + # Second is dispatched only after first reserves, so by the time + # second's reserve fires, slots[0] == first's gid. + second_reserved.set() + + first_impl = _FakeImpl(record="r0", on_run=_first_run) + second_impl = _FakeImpl(record="r1", on_run=_second_run) + + first_mgr = _make_manager(buf, first_impl) + # Share buffer across two managers (mimics two dispatches from one pump). + second_mgr = object.__new__(RolloutManager) + second_mgr._impl = second_impl + second_mgr._tokenizer = None + second_mgr._num_generations_per_prompt = 1 + second_mgr._tq_buffer = buf + second_mgr._finalizer = None + second_mgr._env_handles = {} + second_mgr._weight_version = 0 + + async def _drive(): + t1 = asyncio.create_task(first_mgr.generate_and_push({"prompt": "p1"})) + # Wait until first has reserved before kicking off second so the + # reserve ordering is deterministic. + await first_reserved.wait() + t2 = asyncio.create_task(second_mgr.generate_and_push({"prompt": "p2"})) + await asyncio.gather(t1, t2) + + _run(_drive()) + + # Slots in buffer == reserve order. + first_gid, second_gid = buf._slots + # Commit recorded both, in either order, but each maps to its own gid. + commit_gids = [c[0] for c in buf.commit_calls] + assert set(commit_gids) == {first_gid, second_gid} + assert buf.reserve_calls == [0, 0] + + def test_requires_tq_buffer(self): + mgr = _make_manager(_FakeBuffer(), _FakeImpl()) + mgr._tq_buffer = None + with pytest.raises(AssertionError, match="tq_buffer"): + _run(mgr.generate_and_push({"prompt": "p"})) + + def test_failed_rollout_aborts_reserved_slot(self): + """A dispatch that raises must not leave a phantom unready slot.""" + + async def _boom(_input_sample): + raise RuntimeError("rollout exploded") + + buf = _FakeBuffer() + mgr = _make_manager(buf, _FakeImpl(on_run=_boom)) + + with pytest.raises(RuntimeError, match="rollout exploded"): + _run(mgr.generate_and_push({"prompt": "p"})) + + assert len(buf.reserve_calls) == 1 + assert buf.commit_calls == [] + assert len(buf.abort_calls) == 1 + assert buf._slots == [] # the reserved slot was dropped + + def test_failed_commit_aborts_reserved_slot(self): + """Commit failures (e.g. evicted slot) also abort the reservation.""" + + class _CommitBoomBuffer(_FakeBuffer): + async def commit( + self, group_id, record, start_weight_version, end_weight_version + ): + raise ValueError("no live slot") + + buf = _CommitBoomBuffer() + mgr = _make_manager(buf, _FakeImpl()) + + with pytest.raises(ValueError, match="no live slot"): + _run(mgr.generate_and_push({"prompt": "p"})) + assert len(buf.abort_calls) == 1 + + +# --------------------------------------------------------------------------- +# Tests for RolloutManager +# --------------------------------------------------------------------------- + + +def test_rollout_manager_raises_without_impl_params(): + """RolloutManager raises AssertionError when required params are missing.""" + common = { + "tokenizer": None, + "env_handles": {}, + "num_generations_per_prompt": 1, + "max_seq_len": 1, + } + + with pytest.raises(AssertionError, match="num_generations_per_prompt must be >= 1"): + updated_common = common.copy() + updated_common["num_generations_per_prompt"] = 0 + RolloutManager(**updated_common, use_nemo_gym=False) + + with pytest.raises(AssertionError, match="policy_generation is required"): + RolloutManager(**common, use_nemo_gym=False) + + with pytest.raises(AssertionError, match="generation_config is required"): + RolloutManager(**common, use_nemo_gym=True) + + +# --------------------------------------------------------------------------- +# Tests for AsyncRolloutManager (native async path) +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="function") +def single_multi_step_calculator_input_sample(rollout_tokenizer): # noqa: F811 + """Returns a single DatumSpec prompt dict (problem 0) for AsyncRolloutManager tests.""" + problem_text = "(5 + 3) * 2" + expected_answer = 16.0 + max_steps = 5 + + tool_instructions = ( + "You have a calculator tool. To use it, respond with:\n" + "'[operand1, operand2, operation_name]'\n" + "The valid 'operation_name' values are exactly: 'sum', 'diff', 'prod', 'div'.\n" + "Example: [5, 3, sum]\n" + "You will receive the result of your calculation as ...\n" + "Use this result to make the next calculation if needed.\n" + "IMPORTANT: Only perform one calculation step (one tool call) before waiting for a result and making a new tool call.\n" + "IMPORTANT: Do not perform any other calculations or operations aside from the tool call and result. Doing so will result in failure.\n" + "To give the final answer, just output the number. numbers inside of don't count, so output just the final number yourself outside of this.\n" + "Example full output: [2, 4, sum]\n6.0\n[6, 6, diff]\n0.0 0\n(note how you have to output the final 0 outside of the tags)" + "------\n" + f"Solve: {problem_text}" + ) + + initial_prompt_content = rollout_tokenizer.apply_chat_template( + [{"role": "user", "content": tool_instructions}], + tokenize=False, + add_system_prompt=False, + add_generation_prompt=True, + add_special_tokens=False, + ) + tokenized_prompt = rollout_tokenizer( + initial_prompt_content, return_tensors="pt", add_special_tokens=False + )["input_ids"][0] + message_log = [ + { + "role": "user", + "content": initial_prompt_content, + "token_ids": tokenized_prompt, + } + ] + metadata = MultiStepCalcMetadata( + problem=problem_text, + expected_final_answer=expected_answer, + max_steps=max_steps, + current_step=0, + ) + return { + "message_log": message_log, + "extra_env_info": metadata, + "task_name": "multi_step_calculator_game", + "stop_strings": [""], + "idx": 0, + } + + +@pytest.mark.vllm +def test_async_rollout_manager( + multi_step_setup_vllm_async, # noqa: F811 + single_multi_step_calculator_input_sample, +): + """Standalone test for AsyncRolloutManager. + + Given 1 prompt with num_generations_per_prompt=N, asserts: + - output is a PromptGroupRecord with N Completion objects + - each Completion has a reward (float) and a non-empty message_log + - rollout_metrics has the expected keys with correct types + - completions hold independent (not aliased) message_log objects + """ + vllm_generation, tokenizer, env_handles, _, _ = multi_step_setup_vllm_async + input_sample = single_multi_step_calculator_input_sample + num_generations = 2 + max_seq_len = 1024 + max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 + + manager = RolloutManager( + use_nemo_gym=False, + tokenizer=tokenizer, + env_handles=env_handles, + num_generations_per_prompt=num_generations, + max_seq_len=max_seq_len, + max_rollout_turns=max_rollout_turns, + policy_generation=vllm_generation, + ) + + vllm_generation.prepare_for_generation() + record = asyncio.run(manager.run_rollout(input_sample)) + vllm_generation.finish_generation() + + assert isinstance(record, PromptGroupRecord) + assert len(record.completions) == num_generations, ( + f"Expected {num_generations} completions, got {len(record.completions)}" + ) + assert record.prompt_idx == input_sample["idx"] + + for i, completion in enumerate(record.completions): + assert isinstance(completion, Completion) + + # 1. message_log length + assert len(completion.message_log) >= 4, ( + f"Completion {i}: expected >= 4 messages, got {len(completion.message_log)}" + ) + + # 2. last assistant content + last_assistant = next( + (m for m in reversed(completion.message_log) if m["role"] == "assistant"), + None, + ) + assert last_assistant is not None, f"Completion {i}: no assistant message found" + assert last_assistant["content"].strip() == "16", ( + f"Completion {i}: last assistant content {last_assistant['content']!r} != '16'" + ) + + # 3. reward + assert completion.reward == 1.0, ( + f"Completion {i}: reward {completion.reward} != 1.0" + ) + + # completions must be independent objects + assert record.completions[0].message_log is not record.completions[1].message_log + + +@pytest.mark.vllm +def test_async_rollout_manager_truncation( + multi_step_setup_vllm_async, # noqa: F811 + single_multi_step_calculator_input_sample, +): + """Small max_seq_len forces truncation and truncation_rate=1.0.""" + vllm_generation, tokenizer, env_handles, _, _ = multi_step_setup_vllm_async + input_sample = single_multi_step_calculator_input_sample + num_generations = 2 + max_seq_len = 290 + max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 + + manager = RolloutManager( + use_nemo_gym=False, + tokenizer=tokenizer, + env_handles=env_handles, + num_generations_per_prompt=num_generations, + max_seq_len=max_seq_len, + max_rollout_turns=max_rollout_turns, + policy_generation=vllm_generation, + ) + vllm_generation.prepare_for_generation() + record = asyncio.run(manager.run_rollout(input_sample)) + vllm_generation.finish_generation() + + assert len(record.completions) == num_generations + assert all(c.truncated for c in record.completions) + assert record.rollout_metrics["truncation_rate"] == 1.0 + assert record.rollout_metrics["natural_termination_rate"] == 0.0 + + +@pytest.mark.vllm +def test_async_rollout_manager_matches_original( + multi_step_setup_vllm_async, # noqa: F811 + single_multi_step_calculator_input_sample, +): + """Comparison test: AsyncRolloutManager output is structurally equivalent to the original. + + Calls run_async_multi_turn_rollout with a batch of N identical prompts, + then calls AsyncRolloutManager with 1 prompt and N generations. + Asserts that both produce N results with matching message-log depth, rewards, + and rollout_metrics numeric values. + + TODO: remove this test together with run_async_multi_turn_rollout when the legacy path is deleted. + """ + vllm_generation, tokenizer, env_handles, _, _ = multi_step_setup_vllm_async + input_sample = single_multi_step_calculator_input_sample + num_generations = 2 + max_seq_len = 1024 + max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 + + # Build a batch of N identical prompts for the original function + batch = BatchedDataDict( + { + "message_log": [ + deepcopy(input_sample["message_log"]) for _ in range(num_generations) + ], + "extra_env_info": [ + deepcopy(input_sample["extra_env_info"]) for _ in range(num_generations) + ], + "task_name": [input_sample["task_name"]] * num_generations, + "stop_strings": [input_sample["stop_strings"]] * num_generations, + "idx": list(range(num_generations)), + "loss_multiplier": [1.0] * num_generations, + } + ) + + vllm_generation.prepare_for_generation() + original_batch, original_metrics = run_async_multi_turn_rollout( + policy_generation=vllm_generation, + input_batch=batch, + tokenizer=tokenizer, + task_to_env=env_handles, + max_seq_len=max_seq_len, + max_rollout_turns=max_rollout_turns, + ) + + manager = RolloutManager( + use_nemo_gym=False, + tokenizer=tokenizer, + env_handles=env_handles, + num_generations_per_prompt=num_generations, + max_seq_len=max_seq_len, + max_rollout_turns=max_rollout_turns, + policy_generation=vllm_generation, + ) + record = asyncio.run(manager.run_rollout(input_sample)) + vllm_generation.finish_generation() + + # Both should produce N results + assert len(original_batch["message_log"]) == num_generations + assert len(record.completions) == num_generations + + for i in range(num_generations): + orig_msg_log = original_batch["message_log"][i] + new_msg_log = record.completions[i].message_log + + # 1. message_log length matches + assert len(orig_msg_log) == len(new_msg_log), ( + f"Completion {i}: message_log length {len(new_msg_log)} != original {len(orig_msg_log)}" + ) + + # 2. last assistant content matches + def _last_assistant_content(msg_log): + for m in reversed(msg_log): + if m["role"] == "assistant": + return m.get("content", "") + return "" + + orig_last = _last_assistant_content(orig_msg_log) + new_last = _last_assistant_content(new_msg_log) + assert orig_last == new_last, ( + f"Completion {i}: last assistant content mismatch\n" + f" original: {orig_last!r}\n" + f" manager: {new_last!r}" + ) + + # 3. reward matches + orig_reward = original_batch["total_reward"][i].item() + new_reward = record.completions[i].reward + assert orig_reward == new_reward, ( + f"Completion {i}: reward mismatch — original {orig_reward}, manager {new_reward}" + ) + + # 4. rollout_metrics numeric values match (timing and histogram fields are excluded). + # The new impl emits slash-style keys (X/mean, X/max, X/min) via _calculate_single_metric; + # translate the legacy prefix-style keys before comparing. + def _translate_legacy_key(key: str) -> str: + if key == "avg_turns_per_sample": + return "turns_per_sample/mean" + if key == "max_turns_reached_rate": + return key + for prefix, suffix in (("mean_", "/mean"), ("max_", "/max"), ("min_", "/min")): + if key.startswith(prefix): + return f"{key[len(prefix) :]}{suffix}" + return key + + new_metrics = record.rollout_metrics + for key in original_metrics.keys(): + if key.startswith("timing/") or key.startswith("histogram/"): + continue + + new_key = _translate_legacy_key(key) + assert new_key in new_metrics, ( + f"rollout_metrics[{new_key!r}] missing from manager" + ) + + orig_val = original_metrics[key] + new_val = new_metrics[new_key] + + assert type(orig_val) == type(new_val), ( + f"rollout_metrics[{key!r}] type mismatch: {type(orig_val)} != {type(new_val)}" + ) + if not isinstance(orig_val, (bool, int, float)): + continue + + assert orig_val == pytest.approx(new_val), ( + f"rollout_metrics[{key!r}] mismatch — original {orig_val}, manager {new_val}" + ) + + +# --------------------------------------------------------------------------- +# Tests for AsyncNemoGymRolloutManager +# --------------------------------------------------------------------------- + + +@pytest.mark.nemo_gym +def test_async_nemo_gym_rollout_manager( + nemo_gym, # noqa: F811 + nemo_gym_vllm_generation, # noqa: F811 + nemo_gym_sanity_test_data, # noqa: F811 + nemo_gym_tokenizer, # noqa: F811 +): + """Standalone test for AsyncNemoGymRolloutManager. + + Given 1 prompt with num_generations_per_prompt=N, asserts: + - output is a PromptGroupRecord with N Completion objects + - each Completion has a reward (float) and a non-empty message_log + - completions hold independent message_log objects + + If the result here does not match, please check the following: + 1. Test data changed: re-run test_nemo_gym_sanity (tests/unit/environments/test_nemo_gym.py) + and use _write_actual_test_data output to refresh test_nemo_gym_sanity.json. + 2. Logic changed: inspect recent changes to AsyncNemoGymRolloutManager or the gym env. + """ + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: + for data in nemo_gym_sanity_test_data["input"]: + f.write(json.dumps(data) + "\n") + data_path = f.name + + dataset = NemoGymDataset(data_path) + examples = [ + nemo_gym_data_processor(dataset.dataset[idx], None, None, None, idx) + for idx in range(len(dataset.dataset)) + ] + input_batch: BatchedDataDict[DatumSpec] = rl_collate_fn(examples) + + # Use only the first prompt + single_prompt = { + "message_log": input_batch["message_log"][0], + "extra_env_info": input_batch["extra_env_info"][0], + "task_name": "nemo_gym", + "idx": 0, + "loss_multiplier": float(input_batch["loss_multiplier"][0]), + } + num_generations = 2 + + manager = RolloutManager( + use_nemo_gym=True, + tokenizer=nemo_gym_tokenizer, + env_handles={"nemo_gym": nemo_gym}, + num_generations_per_prompt=num_generations, + max_seq_len=nemo_gym_vllm_generation.cfg["vllm_cfg"]["max_model_len"], + generation_config=nemo_gym_vllm_generation.cfg, + ) + record = asyncio.run(manager.run_rollout(single_prompt)) + + assert isinstance(record, PromptGroupRecord) + assert len(record.completions) == num_generations, ( + f"Expected {num_generations} completions, got {len(record.completions)}" + ) + assert record.prompt_idx == 0 + + for i, completion in enumerate(record.completions): + assert isinstance(completion, Completion) + + # 1. message_log length + assert len(completion.message_log) == 2, ( + f"Completion {i}: expected 2 messages, got {len(completion.message_log)}" + ) + + # 2. last assistant token_ids + last_assistant = next( + (m for m in reversed(completion.message_log) if m["role"] == "assistant"), + None, + ) + assert last_assistant is not None, f"Completion {i}: no assistant message found" + assert torch.equal( + last_assistant["token_ids"], + torch.tensor([151667, 198, 32313, 11, 1077]), + ), ( + f"Completion {i}: last assistant token_ids {last_assistant['token_ids'].tolist()} " + f"!= [151667, 198, 32313, 11, 1077]" + ) + + # 3. reward + assert completion.reward == 0.0, ( + f"Completion {i}: reward {completion.reward} != 0.0" + ) + + # completions must be independent objects + assert record.completions[0].message_log is not record.completions[1].message_log + + +@pytest.mark.nemo_gym +def test_async_nemo_gym_rollout_manager_matches_original( + nemo_gym, # noqa: F811 + nemo_gym_vllm_generation, # noqa: F811 + nemo_gym_sanity_test_data, # noqa: F811 + nemo_gym_tokenizer, # noqa: F811 +): + """Comparison test: AsyncNemoGymRolloutManager output is structurally equivalent to the original. + + Calls run_async_nemo_gym_rollout with a batch of N identical rows, + then calls AsyncNemoGymRolloutManager with 1 prompt, N generations. + Asserts that both produce N results and rewards are in the same numeric domain. + + TODO: remove this test together with run_async_nemo_gym_rollout when the legacy path is deleted. + """ + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: + for data in nemo_gym_sanity_test_data["input"]: + f.write(json.dumps(data) + "\n") + data_path = f.name + + dataset = NemoGymDataset(data_path) + examples = [ + nemo_gym_data_processor(dataset.dataset[idx], None, None, None, idx) + for idx in range(len(dataset.dataset)) + ] + input_batch: BatchedDataDict[DatumSpec] = rl_collate_fn(examples) + + num_generations = 2 + single_prompt = { + "message_log": input_batch["message_log"][0], + "extra_env_info": input_batch["extra_env_info"][0], + "task_name": "nemo_gym", + "idx": 0, + "loss_multiplier": float(input_batch["loss_multiplier"][0]), + } + + # Build a batch of N identical rows for the original function + repeated_batch = BatchedDataDict( + { + "message_log": [ + deepcopy(input_batch["message_log"][0]) for _ in range(num_generations) + ], + "extra_env_info": [ + deepcopy(input_batch["extra_env_info"][0]) + for _ in range(num_generations) + ], + "loss_multiplier": input_batch["loss_multiplier"][0:1].repeat( + num_generations + ), + "idx": list(range(num_generations)), + "task_name": ["nemo_gym"] * num_generations, + } + ) + + original_result = run_async_nemo_gym_rollout( + policy_generation=nemo_gym_vllm_generation, + input_batch=repeated_batch, + tokenizer=nemo_gym_tokenizer, + task_to_env={"nemo_gym": nemo_gym}, + generation_config=nemo_gym_vllm_generation.cfg, + max_seq_len=nemo_gym_vllm_generation.cfg["vllm_cfg"]["max_model_len"], + max_rollout_turns=None, + ) + + manager = RolloutManager( + use_nemo_gym=True, + tokenizer=nemo_gym_tokenizer, + env_handles={"nemo_gym": nemo_gym}, + num_generations_per_prompt=num_generations, + max_seq_len=nemo_gym_vllm_generation.cfg["vllm_cfg"]["max_model_len"], + generation_config=nemo_gym_vllm_generation.cfg, + ) + record = asyncio.run(manager.run_rollout(single_prompt)) + + # Both should produce N completions + assert len(original_result.final_batch["message_log"]) == num_generations + assert len(record.completions) == num_generations + + for i in range(num_generations): + orig_msg_log = original_result.final_batch["message_log"][i] + new_msg_log = record.completions[i].message_log + + # 1. message_log length matches + assert len(orig_msg_log) == len(new_msg_log), ( + f"Completion {i}: message_log length {len(new_msg_log)} != original {len(orig_msg_log)}" + ) + + # 2. last assistant token_ids match + def _last_assistant_token_ids(msg_log): + for m in reversed(msg_log): + if m["role"] == "assistant": + return m.get("token_ids") + return None + + orig_token_ids = _last_assistant_token_ids(orig_msg_log) + new_token_ids = _last_assistant_token_ids(new_msg_log) + assert orig_token_ids is not None, ( + f"Completion {i}: no assistant message in original" + ) + assert new_token_ids is not None, ( + f"Completion {i}: no assistant message in manager" + ) + assert torch.equal(orig_token_ids, new_token_ids), ( + f"Completion {i}: last assistant token_ids mismatch\n" + f" original: {orig_token_ids.tolist()}\n" + f" manager: {new_token_ids.tolist()}" + ) + + # 3. reward matches + orig_reward = original_result.final_batch["total_reward"][i].item() + new_reward = record.completions[i].reward + assert orig_reward == new_reward, ( + f"Completion {i}: reward mismatch — original {orig_reward}, manager {new_reward}" + ) + + # 4. rollout_metrics numeric values match (timing and Table fields are excluded) + orig_metrics = original_result.rollout_metrics + new_metrics = record.rollout_metrics + for key in orig_metrics.keys(): + # Skip timing and full_result fields + if key.startswith("timing/") or key.endswith("/full_result"): + continue + + # Check that the key is present in the new metrics + assert key in new_metrics, f"rollout_metrics[{key!r}] missing from manager" + + orig_val = orig_metrics[key] + new_val = new_metrics[key] + + # Skip non-numeric fields + assert type(orig_val) == type(new_val), ( + f"rollout_metrics[{key!r}] type mismatch: {type(orig_val)} != {type(new_val)}" + ) + if not isinstance(orig_val, (bool, int, float)): + continue + + # Check equal + assert orig_val == pytest.approx(new_val), ( + f"rollout_metrics[{key!r}] mismatch — original {orig_val}, manager {new_val}" + ) + + +class _FakeFinalizedGroup: + def __init__(self, *, dropped=False): + self.meta = None if dropped else "meta-sentinel" + self.group_min_wv = 3 + self.group_max_wv = 4 + self.staging_keys = [] + self.metrics = {"finalize/invalid_row_rate": 0.0} + self.dropped = dropped + + +class _FakeFinalizer: + def __init__(self, *, dropped=False): + self.calls: list[tuple] = [] + self._dropped = dropped + + def finalize_group( + self, group_id, rollout_ids, receipts, rewards, *, fallback_weight_version + ): + self.calls.append( + (group_id, rollout_ids, receipts, rewards, fallback_weight_version) + ) + return _FakeFinalizedGroup(dropped=self._dropped) + + +class _FakeCaptureBuffer(_FakeBuffer): + def __init__(self): + super().__init__() + self.reserve_rollout_ids: list[list[str] | None] = [] + self.commit_finalized_calls: list[tuple] = [] + + def reserve( + self, *, weight_version, target_step=None, group_id=None, rollout_ids=None + ): + self.reserve_rollout_ids.append(rollout_ids) + return super().reserve( + weight_version=weight_version, + target_step=target_step, + group_id=group_id, + rollout_ids=rollout_ids, + ) + + async def commit_finalized( + self, group_id, meta, group_min_wv, group_max_wv, *, staging_keys=None + ): + self.commit_finalized_calls.append( + (group_id, meta, group_min_wv, group_max_wv, staging_keys) + ) + return meta + + +class _FakeGymEnvHandle: + """NemoGym actor stand-in exposing fail_rollouts.remote.""" + + def __init__(self): + self.failed: list[tuple[list[str], str]] = [] + outer = self + + class _FailRollouts: + def remote(self, rollout_ids, reason): + outer.failed.append((list(rollout_ids), reason)) + + async def _done(): + return None + + return _done() + + self.fail_rollouts = _FailRollouts() + + +def _receipt_record(rollout_ids, receipts): + completions = [ + Completion( + message_log=[], + env_extras={"reward": 0.5, "ng_receipt": receipt, "ng_rollout_id": rid}, + truncated=False, + reward=0.5, + ) + for rid, receipt in zip(rollout_ids, receipts) + ] + return PromptGroupRecord( + prompt_idx=0, + prompt=[], + extra_env_info={}, + metadata={"task_name": "nemo_gym"}, + completions=completions, + rollout_metrics={}, + ) + + +def _make_capture_manager(buf, finalizer, *, on_run=None, num_generations=2): + mgr = object.__new__(RolloutManager) + mgr._tokenizer = None + mgr._num_generations_per_prompt = num_generations + mgr._tq_buffer = buf + mgr._finalizer = finalizer + mgr._env_handles = {"nemo_gym": _FakeGymEnvHandle()} + mgr._weight_version = 7 + + class _CaptureImpl: + def __init__(self): + self.seen_rollout_ids = None + + async def run_rollout(self, _sample, *, rollout_ids=None): + self.seen_rollout_ids = rollout_ids + if on_run is not None: + await on_run(_sample) + return _receipt_record( + rollout_ids, [{"rollout_id": rid} for rid in rollout_ids] + ) + + mgr._impl = _CaptureImpl() + return mgr + + +class TestGenerateAndFinalizeFlow: + def test_mints_ids_finalizes_and_commits(self): + buf = _FakeCaptureBuffer() + finalizer = _FakeFinalizer() + mgr = _make_capture_manager(buf, finalizer) + + _run(mgr.generate_and_push({"prompt": "p"}, target_step=5)) + + # Rollout ids were minted from the reserved group id and threaded + # end to end: reserve -> impl -> finalizer. + (group_id,) = buf._slots + expected_ids = [f"{group_id}_g0", f"{group_id}_g1"] + assert buf.reserve_rollout_ids == [expected_ids] + assert mgr._impl.seen_rollout_ids == expected_ids + (fin_group_id, fin_ids, receipts, rewards, fallback_wv) = finalizer.calls[0] + assert fin_group_id == group_id + assert fin_ids == expected_ids + assert [r["rollout_id"] for r in receipts] == expected_ids + assert rewards == [0.5, 0.5] + assert fallback_wv == 7 + # commit_finalized carried the group's min/max call versions. + assert buf.commit_finalized_calls == [(group_id, "meta-sentinel", 3, 4, [])] + # The legacy commit path was not used and nothing failed at the gate. + assert buf.commit_calls == [] + assert mgr._env_handles["nemo_gym"].failed == [] + + def test_dropped_group_aborts_slot(self): + buf = _FakeCaptureBuffer() + mgr = _make_capture_manager(buf, _FakeFinalizer(dropped=True)) + with pytest.raises(RuntimeError, match="min_valid_fraction"): + _run(mgr.generate_and_push({"prompt": "p"})) + assert buf.commit_finalized_calls == [] + assert len(buf.abort_calls) >= 1 + + def test_failed_dispatch_aborts_and_fails_gate_rollouts(self): + buf = _FakeCaptureBuffer() + + async def _boom(_sample): + raise RuntimeError("rollout exploded") + + mgr = _make_capture_manager(buf, _FakeFinalizer(), on_run=_boom) + with pytest.raises(RuntimeError, match="rollout exploded"): + _run(mgr.generate_and_push({"prompt": "p"})) + assert buf.commit_finalized_calls == [] + assert len(buf.abort_calls) == 1 + (failed_ids, reason) = mgr._env_handles["nemo_gym"].failed[0] + (group_id,) = [buf.abort_calls[0]] + assert failed_ids == [f"{group_id}_g0", f"{group_id}_g1"] + assert reason == "dispatch_failed" diff --git a/tests/unit/experience/test_rollouts.py b/tests/unit/experience/test_rollouts.py index 5bd124d935a..91cf854cde9 100644 --- a/tests/unit/experience/test_rollouts.py +++ b/tests/unit/experience/test_rollouts.py @@ -37,8 +37,6 @@ SlidingPuzzleGameLogic, SlidingPuzzleMetadata, ) -from nemo_rl.experience.interfaces import Completion, PromptGroupRecord -from nemo_rl.experience.rollout_manager import RolloutManager from nemo_rl.experience.rollouts import ( _calculate_single_metric, generate_responses_async, @@ -1033,550 +1031,3 @@ def _standardize(d: dict) -> dict: 1. In nemo_rl/experience/rollouts.py::run_async_nemo_gym_rollout, the sampling params are passed appropriately 2. In nemo_rl/models/generation/vllm/vllm_worker_async.py::VllmAsyncGenerationWorker::_setup_vllm_server::create_chat_completion, the sampling params (like top_k) are set as appropriate """ - - -# --------------------------------------------------------------------------- -# Tests for RolloutManager -# --------------------------------------------------------------------------- - - -def test_rollout_manager_raises_without_impl_params(): - """RolloutManager raises AssertionError when required params are missing.""" - common = { - "tokenizer": None, - "task_to_env": {}, - "num_generations_per_prompt": 1, - "max_seq_len": 1, - } - - with pytest.raises(AssertionError, match="num_generations_per_prompt must be >= 1"): - updated_common = common.copy() - updated_common["num_generations_per_prompt"] = 0 - RolloutManager(**updated_common, use_nemo_gym=False) - - with pytest.raises(AssertionError, match="policy_generation is required"): - RolloutManager(**common, use_nemo_gym=False) - - with pytest.raises(AssertionError, match="generation_config is required"): - RolloutManager(**common, use_nemo_gym=True) - - -# --------------------------------------------------------------------------- -# Tests for AsyncRolloutManager (native async path) -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope="function") -def single_multi_step_calculator_input_sample(rollout_tokenizer): - """Returns a single DatumSpec prompt dict (problem 0) for AsyncRolloutManager tests.""" - problem_text = "(5 + 3) * 2" - expected_answer = 16.0 - max_steps = 5 - - tool_instructions = ( - "You have a calculator tool. To use it, respond with:\n" - "'[operand1, operand2, operation_name]'\n" - "The valid 'operation_name' values are exactly: 'sum', 'diff', 'prod', 'div'.\n" - "Example: [5, 3, sum]\n" - "You will receive the result of your calculation as ...\n" - "Use this result to make the next calculation if needed.\n" - "IMPORTANT: Only perform one calculation step (one tool call) before waiting for a result and making a new tool call.\n" - "IMPORTANT: Do not perform any other calculations or operations aside from the tool call and result. Doing so will result in failure.\n" - "To give the final answer, just output the number. numbers inside of don't count, so output just the final number yourself outside of this.\n" - "Example full output: [2, 4, sum]\n6.0\n[6, 6, diff]\n0.0 0\n(note how you have to output the final 0 outside of the tags)" - "------\n" - f"Solve: {problem_text}" - ) - - initial_prompt_content = rollout_tokenizer.apply_chat_template( - [{"role": "user", "content": tool_instructions}], - tokenize=False, - add_system_prompt=False, - add_generation_prompt=True, - add_special_tokens=False, - ) - tokenized_prompt = rollout_tokenizer( - initial_prompt_content, return_tensors="pt", add_special_tokens=False - )["input_ids"][0] - message_log = [ - { - "role": "user", - "content": initial_prompt_content, - "token_ids": tokenized_prompt, - } - ] - metadata = MultiStepCalcMetadata( - problem=problem_text, - expected_final_answer=expected_answer, - max_steps=max_steps, - current_step=0, - ) - return { - "message_log": message_log, - "extra_env_info": metadata, - "task_name": "multi_step_calculator_game", - "stop_strings": [""], - "idx": 0, - } - - -@pytest.mark.vllm -def test_async_rollout_manager( - multi_step_setup_vllm_async, - single_multi_step_calculator_input_sample, -): - """Standalone test for AsyncRolloutManager. - - Given 1 prompt with num_generations_per_prompt=N, asserts: - - output is a PromptGroupRecord with N Completion objects - - each Completion has a reward (float) and a non-empty message_log - - rollout_metrics has the expected keys with correct types - - completions hold independent (not aliased) message_log objects - """ - vllm_generation, rollout_tokenizer, task_to_env, _, _ = multi_step_setup_vllm_async - input_sample = single_multi_step_calculator_input_sample - num_generations = 2 - max_seq_len = 1024 - max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 - - manager = RolloutManager( - use_nemo_gym=False, - tokenizer=rollout_tokenizer, - task_to_env=task_to_env, - num_generations_per_prompt=num_generations, - max_seq_len=max_seq_len, - max_rollout_turns=max_rollout_turns, - policy_generation=vllm_generation, - ) - - vllm_generation.prepare_for_generation() - record = asyncio.run(manager.run_rollout(input_sample)) - vllm_generation.finish_generation() - - assert isinstance(record, PromptGroupRecord) - assert len(record.completions) == num_generations, ( - f"Expected {num_generations} completions, got {len(record.completions)}" - ) - assert record.prompt_idx == input_sample["idx"] - - for i, completion in enumerate(record.completions): - assert isinstance(completion, Completion) - - # 1. message_log length - assert len(completion.message_log) >= 4, ( - f"Completion {i}: expected >= 4 messages, got {len(completion.message_log)}" - ) - - # 2. last assistant content - last_assistant = next( - (m for m in reversed(completion.message_log) if m["role"] == "assistant"), - None, - ) - assert last_assistant is not None, f"Completion {i}: no assistant message found" - assert last_assistant["content"].strip() == "16", ( - f"Completion {i}: last assistant content {last_assistant['content']!r} != '16'" - ) - - # 3. reward - assert completion.reward == 1.0, ( - f"Completion {i}: reward {completion.reward} != 1.0" - ) - - # completions must be independent objects - assert record.completions[0].message_log is not record.completions[1].message_log - - -@pytest.mark.vllm -def test_async_rollout_manager_truncation( - multi_step_setup_vllm_async, - single_multi_step_calculator_input_sample, -): - """Small max_seq_len forces truncation and truncation_rate=1.0.""" - vllm_generation, rollout_tokenizer, task_to_env, _, _ = multi_step_setup_vllm_async - input_sample = single_multi_step_calculator_input_sample - num_generations = 2 - max_seq_len = 290 - max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 - - manager = RolloutManager( - use_nemo_gym=False, - tokenizer=rollout_tokenizer, - task_to_env=task_to_env, - num_generations_per_prompt=num_generations, - max_seq_len=max_seq_len, - max_rollout_turns=max_rollout_turns, - policy_generation=vllm_generation, - ) - vllm_generation.prepare_for_generation() - record = asyncio.run(manager.run_rollout(input_sample)) - vllm_generation.finish_generation() - - assert len(record.completions) == num_generations - assert all(c.truncated for c in record.completions) - assert record.rollout_metrics["truncation_rate"] == 1.0 - assert record.rollout_metrics["natural_termination_rate"] == 0.0 - - -@pytest.mark.vllm -def test_async_rollout_manager_matches_original( - multi_step_setup_vllm_async, - single_multi_step_calculator_input_sample, -): - """Comparison test: AsyncRolloutManager output is structurally equivalent to the original. - - Calls run_async_multi_turn_rollout with a batch of N identical prompts, - then calls AsyncRolloutManager with 1 prompt and N generations. - Asserts that both produce N results with matching message-log depth, rewards, - and rollout_metrics numeric values. - - TODO: remove this test together with run_async_multi_turn_rollout when the legacy path is deleted. - """ - vllm_generation, rollout_tokenizer, task_to_env, _, _ = multi_step_setup_vllm_async - input_sample = single_multi_step_calculator_input_sample - num_generations = 2 - max_seq_len = 1024 - max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 - - # Build a batch of N identical prompts for the original function - batch = BatchedDataDict( - { - "message_log": [ - deepcopy(input_sample["message_log"]) for _ in range(num_generations) - ], - "extra_env_info": [ - deepcopy(input_sample["extra_env_info"]) for _ in range(num_generations) - ], - "task_name": [input_sample["task_name"]] * num_generations, - "stop_strings": [input_sample["stop_strings"]] * num_generations, - "idx": list(range(num_generations)), - "loss_multiplier": [1.0] * num_generations, - } - ) - - vllm_generation.prepare_for_generation() - original_batch, original_metrics = run_async_multi_turn_rollout( - policy_generation=vllm_generation, - input_batch=batch, - tokenizer=rollout_tokenizer, - task_to_env=task_to_env, - max_seq_len=max_seq_len, - max_rollout_turns=max_rollout_turns, - ) - - manager = RolloutManager( - use_nemo_gym=False, - tokenizer=rollout_tokenizer, - task_to_env=task_to_env, - num_generations_per_prompt=num_generations, - max_seq_len=max_seq_len, - max_rollout_turns=max_rollout_turns, - policy_generation=vllm_generation, - ) - record = asyncio.run(manager.run_rollout(input_sample)) - vllm_generation.finish_generation() - - # Both should produce N results - assert len(original_batch["message_log"]) == num_generations - assert len(record.completions) == num_generations - - for i in range(num_generations): - orig_msg_log = original_batch["message_log"][i] - new_msg_log = record.completions[i].message_log - - # 1. message_log length matches - assert len(orig_msg_log) == len(new_msg_log), ( - f"Completion {i}: message_log length {len(new_msg_log)} != original {len(orig_msg_log)}" - ) - - # 2. last assistant content matches - def _last_assistant_content(msg_log): - for m in reversed(msg_log): - if m["role"] == "assistant": - return m.get("content", "") - return "" - - orig_last = _last_assistant_content(orig_msg_log) - new_last = _last_assistant_content(new_msg_log) - assert orig_last == new_last, ( - f"Completion {i}: last assistant content mismatch\n" - f" original: {orig_last!r}\n" - f" manager: {new_last!r}" - ) - - # 3. reward matches - orig_reward = original_batch["total_reward"][i].item() - new_reward = record.completions[i].reward - assert orig_reward == new_reward, ( - f"Completion {i}: reward mismatch — original {orig_reward}, manager {new_reward}" - ) - - # 4. rollout_metrics numeric values match (timing and histogram fields are excluded). - # The new impl emits slash-style keys (X/mean, X/max, X/min) via _calculate_single_metric; - # translate the legacy prefix-style keys before comparing. - def _translate_legacy_key(key: str) -> str: - if key == "avg_turns_per_sample": - return "turns_per_sample/mean" - if key == "max_turns_reached_rate": - return key - for prefix, suffix in (("mean_", "/mean"), ("max_", "/max"), ("min_", "/min")): - if key.startswith(prefix): - return f"{key[len(prefix) :]}{suffix}" - return key - - new_metrics = record.rollout_metrics - for key in original_metrics.keys(): - if key.startswith("timing/") or key.startswith("histogram/"): - continue - - new_key = _translate_legacy_key(key) - assert new_key in new_metrics, ( - f"rollout_metrics[{new_key!r}] missing from manager" - ) - - orig_val = original_metrics[key] - new_val = new_metrics[new_key] - - assert type(orig_val) == type(new_val), ( - f"rollout_metrics[{key!r}] type mismatch: {type(orig_val)} != {type(new_val)}" - ) - if not isinstance(orig_val, (bool, int, float)): - continue - - assert orig_val == pytest.approx(new_val), ( - f"rollout_metrics[{key!r}] mismatch — original {orig_val}, manager {new_val}" - ) - - -# --------------------------------------------------------------------------- -# Tests for AsyncNemoGymRolloutManager -# --------------------------------------------------------------------------- - - -@pytest.mark.nemo_gym -def test_async_nemo_gym_rollout_manager( - nemo_gym, # noqa: F811 - nemo_gym_vllm_generation, # noqa: F811 - nemo_gym_sanity_test_data, # noqa: F811 - nemo_gym_tokenizer, # noqa: F811 -): - """Standalone test for AsyncNemoGymRolloutManager. - - Given 1 prompt with num_generations_per_prompt=N, asserts: - - output is a PromptGroupRecord with N Completion objects - - each Completion has a reward (float) and a non-empty message_log - - completions hold independent message_log objects - - If the result here does not match, please check the following: - 1. Test data changed: re-run test_nemo_gym_sanity (tests/unit/environments/test_nemo_gym.py) - and use _write_actual_test_data output to refresh test_nemo_gym_sanity.json. - 2. Logic changed: inspect recent changes to AsyncNemoGymRolloutManager or the gym env. - """ - with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: - for data in nemo_gym_sanity_test_data["input"]: - f.write(json.dumps(data) + "\n") - data_path = f.name - - dataset = NemoGymDataset(data_path) - examples = [ - nemo_gym_data_processor(dataset.dataset[idx], None, None, None, idx) - for idx in range(len(dataset.dataset)) - ] - input_batch: BatchedDataDict[DatumSpec] = rl_collate_fn(examples) - - # Use only the first prompt - single_prompt = { - "message_log": input_batch["message_log"][0], - "extra_env_info": input_batch["extra_env_info"][0], - "task_name": "nemo_gym", - "idx": 0, - "loss_multiplier": float(input_batch["loss_multiplier"][0]), - } - num_generations = 2 - - manager = RolloutManager( - use_nemo_gym=True, - tokenizer=nemo_gym_tokenizer, - task_to_env={"nemo_gym": nemo_gym}, - num_generations_per_prompt=num_generations, - max_seq_len=nemo_gym_vllm_generation.cfg["vllm_cfg"]["max_model_len"], - generation_config=nemo_gym_vllm_generation.cfg, - ) - record = asyncio.run(manager.run_rollout(single_prompt)) - - assert isinstance(record, PromptGroupRecord) - assert len(record.completions) == num_generations, ( - f"Expected {num_generations} completions, got {len(record.completions)}" - ) - assert record.prompt_idx == 0 - - for i, completion in enumerate(record.completions): - assert isinstance(completion, Completion) - - # 1. message_log length - assert len(completion.message_log) == 2, ( - f"Completion {i}: expected 2 messages, got {len(completion.message_log)}" - ) - - # 2. last assistant token_ids - last_assistant = next( - (m for m in reversed(completion.message_log) if m["role"] == "assistant"), - None, - ) - assert last_assistant is not None, f"Completion {i}: no assistant message found" - assert torch.equal( - last_assistant["token_ids"], - torch.tensor([151667, 198, 32313, 11, 1077]), - ), ( - f"Completion {i}: last assistant token_ids {last_assistant['token_ids'].tolist()} " - f"!= [151667, 198, 32313, 11, 1077]" - ) - - # 3. reward - assert completion.reward == 0.0, ( - f"Completion {i}: reward {completion.reward} != 0.0" - ) - - # completions must be independent objects - assert record.completions[0].message_log is not record.completions[1].message_log - - -@pytest.mark.nemo_gym -def test_async_nemo_gym_rollout_manager_matches_original( - nemo_gym, # noqa: F811 - nemo_gym_vllm_generation, # noqa: F811 - nemo_gym_sanity_test_data, # noqa: F811 - nemo_gym_tokenizer, # noqa: F811 -): - """Comparison test: AsyncNemoGymRolloutManager output is structurally equivalent to the original. - - Calls run_async_nemo_gym_rollout with a batch of N identical rows, - then calls AsyncNemoGymRolloutManager with 1 prompt, N generations. - Asserts that both produce N results and rewards are in the same numeric domain. - - TODO: remove this test together with run_async_nemo_gym_rollout when the legacy path is deleted. - """ - with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: - for data in nemo_gym_sanity_test_data["input"]: - f.write(json.dumps(data) + "\n") - data_path = f.name - - dataset = NemoGymDataset(data_path) - examples = [ - nemo_gym_data_processor(dataset.dataset[idx], None, None, None, idx) - for idx in range(len(dataset.dataset)) - ] - input_batch: BatchedDataDict[DatumSpec] = rl_collate_fn(examples) - - num_generations = 2 - single_prompt = { - "message_log": input_batch["message_log"][0], - "extra_env_info": input_batch["extra_env_info"][0], - "task_name": "nemo_gym", - "idx": 0, - "loss_multiplier": float(input_batch["loss_multiplier"][0]), - } - - # Build a batch of N identical rows for the original function - repeated_batch = BatchedDataDict( - { - "message_log": [ - deepcopy(input_batch["message_log"][0]) for _ in range(num_generations) - ], - "extra_env_info": [ - deepcopy(input_batch["extra_env_info"][0]) - for _ in range(num_generations) - ], - "loss_multiplier": input_batch["loss_multiplier"][0:1].repeat( - num_generations - ), - "idx": list(range(num_generations)), - "task_name": ["nemo_gym"] * num_generations, - } - ) - - original_result = run_async_nemo_gym_rollout( - policy_generation=nemo_gym_vllm_generation, - input_batch=repeated_batch, - tokenizer=nemo_gym_tokenizer, - task_to_env={"nemo_gym": nemo_gym}, - generation_config=nemo_gym_vllm_generation.cfg, - max_seq_len=nemo_gym_vllm_generation.cfg["vllm_cfg"]["max_model_len"], - max_rollout_turns=None, - ) - - manager = RolloutManager( - use_nemo_gym=True, - tokenizer=nemo_gym_tokenizer, - task_to_env={"nemo_gym": nemo_gym}, - num_generations_per_prompt=num_generations, - max_seq_len=nemo_gym_vllm_generation.cfg["vllm_cfg"]["max_model_len"], - generation_config=nemo_gym_vllm_generation.cfg, - ) - record = asyncio.run(manager.run_rollout(single_prompt)) - - # Both should produce N completions - assert len(original_result.final_batch["message_log"]) == num_generations - assert len(record.completions) == num_generations - - for i in range(num_generations): - orig_msg_log = original_result.final_batch["message_log"][i] - new_msg_log = record.completions[i].message_log - - # 1. message_log length matches - assert len(orig_msg_log) == len(new_msg_log), ( - f"Completion {i}: message_log length {len(new_msg_log)} != original {len(orig_msg_log)}" - ) - - # 2. last assistant token_ids match - def _last_assistant_token_ids(msg_log): - for m in reversed(msg_log): - if m["role"] == "assistant": - return m.get("token_ids") - return None - - orig_token_ids = _last_assistant_token_ids(orig_msg_log) - new_token_ids = _last_assistant_token_ids(new_msg_log) - assert orig_token_ids is not None, ( - f"Completion {i}: no assistant message in original" - ) - assert new_token_ids is not None, ( - f"Completion {i}: no assistant message in manager" - ) - assert torch.equal(orig_token_ids, new_token_ids), ( - f"Completion {i}: last assistant token_ids mismatch\n" - f" original: {orig_token_ids.tolist()}\n" - f" manager: {new_token_ids.tolist()}" - ) - - # 3. reward matches - orig_reward = original_result.final_batch["total_reward"][i].item() - new_reward = record.completions[i].reward - assert orig_reward == new_reward, ( - f"Completion {i}: reward mismatch — original {orig_reward}, manager {new_reward}" - ) - - # 4. rollout_metrics numeric values match (timing and Table fields are excluded) - orig_metrics = original_result.rollout_metrics - new_metrics = record.rollout_metrics - for key in orig_metrics.keys(): - # Skip timing and full_result fields - if key.startswith("timing/") or key.endswith("/full_result"): - continue - - # Check that the key is present in the new metrics - assert key in new_metrics, f"rollout_metrics[{key!r}] missing from manager" - - orig_val = orig_metrics[key] - new_val = new_metrics[key] - - # Skip non-numeric fields - assert type(orig_val) == type(new_val), ( - f"rollout_metrics[{key!r}] type mismatch: {type(orig_val)} != {type(new_val)}" - ) - if not isinstance(orig_val, (bool, int, float)): - continue - - # Check equal - assert orig_val == pytest.approx(new_val), ( - f"rollout_metrics[{key!r}] mismatch — original {orig_val}, manager {new_val}" - ) diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py new file mode 100644 index 00000000000..65111211205 --- /dev/null +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -0,0 +1,312 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""S2 worker hosting: install_capture wiring, fan-outs, version stamping. + +Marked nemo_gym (run with ``--nemo-gym-only``): the hosting seam imports +Gym's capture core. No engine or GPU is needed — the worker methods are +driven unbound against light fakes, and the VllmGeneration fan-outs against +a mock worker group. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +nemo_gym = pytest.importorskip("nemo_gym.token_id_capture.staging") + +from nemo_gym.token_id_capture.staging.capture import ( # noqa: E402 + RolloutTokenCapture, +) +from nemo_gym.token_id_capture.staging.records import ( # noqa: E402 + StagedCallRecord, + StageResult, +) + +from nemo_rl.models.generation.vllm.vllm_generation import VllmGeneration # noqa: E402 +from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 + VllmAsyncGenerationWorkerImpl, +) + +pytestmark = pytest.mark.nemo_gym + + +class _MemorySink: + def __init__(self) -> None: + self.records: list[StagedCallRecord] = [] + + def stage(self, record: StagedCallRecord) -> StageResult: + self.records.append(record) + return StageResult(ok=True, staging_key=record.staging_key) + + +def _fake_worker(*, is_model_owner: bool = True) -> SimpleNamespace: + """The attribute surface setup_token_capture touches, minus the engine.""" + worker = SimpleNamespace( + is_model_owner=is_model_owner, + token_capture=None, + _rollout_weight_version=0, + ) + worker.install_token_capture = lambda capture: setattr( + worker, "token_capture", capture + ) + return worker + + +def test_setup_token_capture_installs_capture_with_vllm_adapter(monkeypatch): + sink = _MemorySink() + monkeypatch.setattr( + "nemo_rl.data_plane.build_data_plane_client", + lambda dp_cfg, bootstrap: MagicMock(name="dp_client"), + ) + monkeypatch.setattr( + "nemo_rl.data_plane.tq_token_sink.TQTokenSink", + lambda dp_client, *, staging_partition: sink, + ) + worker = _fake_worker() + + installed = asyncio.run( + VllmAsyncGenerationWorkerImpl.setup_token_capture( + worker, dp_cfg={"backend": "simple"}, staging_partition="rollout_staging" + ) + ) + + assert installed is True + assert isinstance(worker.token_capture, RolloutTokenCapture) + assert worker.token_capture.adapter is not None + # The adapter is the vLLM one (prefix ids enter via the worker's field). + payload = worker.token_capture.adapter.enter_prefix({}, [1, 2]) + assert payload["required_prefix_token_ids"] == [1, 2] + + +def test_setup_token_capture_skips_non_model_owners(monkeypatch): + worker = _fake_worker(is_model_owner=False) + installed = asyncio.run( + VllmAsyncGenerationWorkerImpl.setup_token_capture( + worker, dp_cfg={}, staging_partition="rollout_staging" + ) + ) + assert installed is False + assert worker.token_capture is None + + +def test_weight_version_is_stamped_from_worker_state(monkeypatch): + """The install closure reads _rollout_weight_version live: a + set_rollout_weight_version between calls changes the stamp.""" + sink = _MemorySink() + monkeypatch.setattr( + "nemo_rl.data_plane.build_data_plane_client", + lambda dp_cfg, bootstrap: MagicMock(), + ) + monkeypatch.setattr( + "nemo_rl.data_plane.tq_token_sink.TQTokenSink", + lambda dp_client, *, staging_partition: sink, + ) + worker = _fake_worker() + asyncio.run( + VllmAsyncGenerationWorkerImpl.setup_token_capture( + worker, dp_cfg={}, staging_partition="rollout_staging" + ) + ) + + asyncio.run(VllmAsyncGenerationWorkerImpl.set_rollout_weight_version(worker, 4)) + first = worker.token_capture.begin_call(rollout_id="r", call_id="c1", mode="text") + asyncio.run(VllmAsyncGenerationWorkerImpl.set_rollout_weight_version(worker, 5)) + second = worker.token_capture.begin_call(rollout_id="r", call_id="c2", mode="text") + + assert (first.weight_version, second.weight_version) == (4, 5) + + coords = worker.token_capture.complete_call( + first, prompt_token_ids=[1], generated_token_ids=[2], generated_logprobs=[-0.1] + ) + assert coords.weight_version == 4 + assert sink.records[0].weight_version == 4 + + +def _generation_with_mock_group(*, async_engine: bool = True) -> VllmGeneration: + gen = object.__new__(VllmGeneration) + gen.cfg = {"vllm_cfg": {"async_engine": async_engine}} + gen.worker_group = MagicMock() + gen.worker_group.run_all_workers_single_data.return_value = [] + return gen + + +def test_generation_setup_token_capture_fans_out(monkeypatch): + gen = _generation_with_mock_group() + gen.setup_token_capture({"backend": "simple"}, "rollout_staging") + gen.worker_group.run_all_workers_single_data.assert_called_once_with( + "setup_token_capture", + dp_cfg={"backend": "simple"}, + staging_partition="rollout_staging", + run_rank_0_only_axes=["tensor_parallel", "pipeline_parallel"], + ) + + +def test_generation_setup_token_capture_requires_async_engine(): + gen = _generation_with_mock_group(async_engine=False) + with pytest.raises(AssertionError, match="async vLLM engine"): + gen.setup_token_capture({}, "rollout_staging") + + +def test_generation_set_rollout_weight_version_fans_out(): + gen = _generation_with_mock_group() + gen.set_rollout_weight_version(7) + gen.worker_group.run_all_workers_single_data.assert_called_once_with( + "set_rollout_weight_version", + version=7, + run_rank_0_only_axes=["tensor_parallel", "pipeline_parallel"], + ) + + +# --------------------------------------------------------------------------- +# S4: the request-path hookup (begin -> finish/abort around a served call) +# --------------------------------------------------------------------------- + + +class _FakeRequest(SimpleNamespace): + pass + + +def _worker_with_capture(sink: _MemorySink): + from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter + + worker = _fake_worker() + worker._capture_calls = {} + worker.token_capture = RolloutTokenCapture( + sink=sink, + weight_version_fn=lambda: worker._rollout_weight_version, + adapter=VLLMCaptureAdapter(), + ) + return worker + + +def _served_content(gen_ids, logprobs): + return { + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "x"}, + "logprobs": { + "content": [ + {"token": f"token_id:{t}", "logprob": lp} + for t, lp in zip(gen_ids, logprobs) + ] + }, + } + ] + } + + +def test_request_capture_round_trip_stages_and_rides_coords(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + request = _FakeRequest( + ng_capture={ + "rollout_id": "r0", + "call_id": "c1", + "parent_call_id": None, + "prev_len": 0, + "mode": "text", + }, + stream=False, + ) + VllmAsyncGenerationWorkerImpl._begin_request_capture(worker, request, [10, 11, 12]) + content = _served_content([13, 14], [-0.1, -0.2]) + content = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, content + ) + # Bytes were staged before the coords existed (fail-closed ordering). + assert len(sink.records) == 1 + assert sink.records[0].token_ids_delta == [10, 11, 12, 13, 14] + coords = content["ng_commit_coords"] + assert coords["disposition"] == "staged" + assert (coords["delta_len"], coords["cum_len"]) == (5, 5) + # Logprobs never transit worker -> gate; state map is drained. + assert ( + "logprobs" not in content["choices"][0] + or content["choices"][0]["logprobs"] is None + ) + assert worker._capture_calls == {} + + +def test_request_capture_token_in_prev_len_chains(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + request = _FakeRequest( + ng_capture={ + "rollout_id": "r0", + "call_id": "c2", + "parent_call_id": "c1", + "prev_len": 3, + "mode": "token_in", + }, + stream=False, + ) + spliced_prompt = [10, 11, 12, 20, 21] # exact prefix + fresh suffix + VllmAsyncGenerationWorkerImpl._begin_request_capture( + worker, request, spliced_prompt + ) + content = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([22], [-0.5]) + ) + coords = content["ng_commit_coords"] + assert coords["parent_call_id"] == "c1" + assert (coords["delta_len"], coords["cum_len"]) == (3, 6) + assert sink.records[0].token_ids_delta == [20, 21, 22] + + +def test_request_capture_is_a_noop_without_context_or_capture(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + plain = _FakeRequest(stream=False) # no ng_capture attribute + VllmAsyncGenerationWorkerImpl._begin_request_capture(worker, plain, [1, 2]) + content = { + "choices": [{"message": {"role": "assistant"}, "logprobs": {"content": []}}] + } + out = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, plain, dict(content) + ) + assert "ng_commit_coords" not in out + assert out["choices"][0]["logprobs"] is not None # untouched off the capture path + assert sink.records == [] + + +def test_request_capture_abort_fails_the_call_and_drains_state(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + request = _FakeRequest( + ng_capture={ + "rollout_id": "r0", + "call_id": "c1", + "parent_call_id": None, + "prev_len": 0, + "mode": "text", + }, + stream=False, + ) + VllmAsyncGenerationWorkerImpl._begin_request_capture(worker, request, [1, 2]) + VllmAsyncGenerationWorkerImpl._abort_request_capture( + worker, request, reason="engine_error" + ) + assert worker._capture_calls == {} + assert sink.records == [] + # A late finish after abort is a no-op (state already drained). + out = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([3], [-0.1]) + ) + assert "ng_commit_coords" not in out diff --git a/tests/unit/models/policy/test_megatron_split_state.py b/tests/unit/models/policy/test_megatron_split_state.py new file mode 100644 index 00000000000..fb50c839ef7 --- /dev/null +++ b/tests/unit/models/policy/test_megatron_split_state.py @@ -0,0 +1,638 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CPU state-machine tests for MegatronPolicyWorkerImpl's split-API. + +These tests cover the lifecycle and call-order invariants — they do NOT +exercise real distributed comms, the mcore scheduler, or the optimizer. +Numerical equivalence vs sync ``train()`` lives in the GPU parity tests. + +The bugs these catch: + - silent gradient over-counting if ``model.no_sync()`` is not wrapped + around ``megatron_forward_backward`` (the mcore DDP hooks would + dispatch a per-call reduce, ADDING to an already-reduced bucket). + - PP>1 pipeline-schedule bypass if ``model.config.grad_sync_func`` is + not nulled for the step's duration. + - ``trainer_version`` advancing on abort. + - ``zero_grad_buffer`` not called at begin (mcore's contiguous grad + buffer leaks stale grads otherwise). + - off-by-one in ``total_num_microbatches`` (used to scale MoE aux-loss). +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +import torch + +# megatron.bridge is only available with the mcore extras. Without it the +# eager import of megatron_policy_worker (transitively imports megatron.bridge) +# fails at COLLECTION time on non-mcore shards, which then breaks every other +# test in that shard. importorskip stops collection cleanly here. +pytest.importorskip("megatron.bridge") + +# Eagerly import the worker module so ``unittest.mock.patch`` can resolve +# attributes on it via ``getattr``. Without this the patch path +# ``nemo_rl.models.policy.workers.megatron_policy_worker.`` fails +# at ``getattr(workers, "megatron_policy_worker")``. +import nemo_rl.models.policy.workers.megatron_policy_worker # noqa: E402,F401 + +pytestmark = pytest.mark.mcore + +# Module path of the worker under test +WORKER_MOD = "nemo_rl.models.policy.workers.megatron_policy_worker" + + +# ── Mock fabric ────────────────────────────────────────────────────────── + + +def _make_mock_model(): + """A mcore-DDP-shaped mock: exposes the methods + attributes the + split-API touches, plus an ``inference_params`` attribute and a + ``modules()`` that yields nothing (so the inference-cache reset loop + is a no-op).""" + model = MagicMock() + model.config = MagicMock() + model.config.grad_sync_func = "ORIGINAL_GRAD_SYNC_FUNC" # sentinel + model.config.num_moe_experts = None # disable MoE branch + # no_sync() is a context manager — return a MagicMock that supports + # __enter__/__exit__ so the `with self.model.no_sync():` block works. + model.no_sync = MagicMock( + return_value=MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + ) + model.modules = MagicMock(return_value=iter([])) + model.inference_params = None + model.parameters = MagicMock( + return_value=iter([]) + ) # no params for the rescale loop + return model + + +def _make_worker(loss_type): + """Construct a MegatronPolicyWorkerImpl instance with all heavy + attributes mocked. Bypasses __init__ via ``object.__new__``.""" + # Lazy import so the module-level mcore imports happen inside the + # mcore-marked test process. + from nemo_rl.models.policy.workers.megatron_policy_worker import ( + MegatronPolicyWorkerImpl, + ) + + w = object.__new__(MegatronPolicyWorkerImpl) + w.model = _make_mock_model() + w.optimizer = MagicMock() + # MegatronOptimizer.step returns (success, grad_norm, num_zeros) + w.optimizer.step.return_value = (True, 0.5, 0) + w.optimizer.param_groups = [{"lr": 1e-4, "weight_decay": 0.01}] + w.scheduler = MagicMock() + w.scheduler.get_lr.return_value = 1e-4 + w.scheduler.get_wd.return_value = 0.01 + w.mcore_state = MagicMock() + w.mcore_state.straggler_timer = None + w.cfg = { + "train_global_batch_size": 32, + "train_micro_batch_size": 4, + "megatron_cfg": { + "empty_unused_memory_level": 0, + "moe_per_layer_logging": False, + "use_linear_ce_fusion_loss": False, + }, + } + w.dp_size = 2 + w.cp_size = 1 + w.sampling_params = None + w.draft_model = None + w.defer_fp32_logits = False + w.dtype = torch.float32 + w._is_reward_model = False + + # Stash a loss_fn with the requested loss_type for tests that need one. + w._test_loss_fn = MagicMock(loss_type=loss_type) + return w + + +@pytest.fixture +def mock_module_symbols(): + """Patch every module-level symbol that the split-API methods call + into. Yields a dict of name → mock for assertions.""" + # Make `aggregate_training_statistics` return ({}, scalar) — what the + # finish path expects. + agg_ret = ({"loss": [0.0]}, torch.tensor(0.5)) + + patches = { + "megatron_forward_backward": [ + {"loss": 0.5, "global_valid_seqs": 8.0, "global_valid_toks": 256.0} + ], + "get_microbatch_iterator": (iter([]), 2, 4, 16, 16), # 2 pipeline mbs per call + "LossPostProcessor": MagicMock(), + "broadcast_loss_metrics_from_last_stage": lambda m: m, + "get_pg_collection": MagicMock(mp=MagicMock()), + "logical_and_across_model_parallel_group": lambda v, mp_group: v, + "reduce_max_stat_across_model_parallel_group": lambda v, mp_group: v, + "aggregate_training_statistics": agg_ret, + "get_moe_metrics": MagicMock(return_value={}), + } + + with ( + patch( + f"{WORKER_MOD}.megatron_forward_backward", + return_value=patches["megatron_forward_backward"], + ) as mfb, + patch( + f"{WORKER_MOD}.get_microbatch_iterator", + return_value=patches["get_microbatch_iterator"], + ) as gmi, + patch( + f"{WORKER_MOD}.LossPostProcessor", return_value=patches["LossPostProcessor"] + ) as lpp, + patch( + f"{WORKER_MOD}.broadcast_loss_metrics_from_last_stage", + side_effect=patches["broadcast_loss_metrics_from_last_stage"], + ) as bcast, + patch( + f"{WORKER_MOD}.get_pg_collection", return_value=patches["get_pg_collection"] + ) as gpgc, + patch( + f"{WORKER_MOD}.logical_and_across_model_parallel_group", + side_effect=patches["logical_and_across_model_parallel_group"], + ) as land, + patch( + f"{WORKER_MOD}.reduce_max_stat_across_model_parallel_group", + side_effect=patches["reduce_max_stat_across_model_parallel_group"], + ) as rmax, + patch( + f"{WORKER_MOD}.aggregate_training_statistics", + return_value=patches["aggregate_training_statistics"], + ) as agg, + patch(f"{WORKER_MOD}.get_moe_metrics", return_value={}) as moe, + patch(f"{WORKER_MOD}.get_rerun_state_machine") as grsm, + patch(f"{WORKER_MOD}.parallel_state") as pstate, + patch("torch.distributed.all_reduce") as ar, + patch("torch.cuda.empty_cache") as cec, + patch("torch.cuda.get_device_name", return_value="H100"), + patch("torch.distributed.get_rank", return_value=0), + ): + # rerun state machine: fire forward+backward once per train_microbatch + rsm = MagicMock() + rsm.should_run_forward_backward.side_effect = [True, False] * 100 + grsm.return_value = rsm + + # parallel_state mocks + pstate.is_pipeline_last_stage.return_value = True + pstate.get_data_parallel_group.return_value = MagicMock() + + yield { + "mfb": mfb, + "gmi": gmi, + "lpp": lpp, + "bcast": bcast, + "gpgc": gpgc, + "land": land, + "rmax": rmax, + "agg": agg, + "moe": moe, + "grsm": grsm, + "pstate": pstate, + "all_reduce": ar, + "empty_cache": cec, + } + + +def _fake_batch(): + """A minimal BatchedDataDict-ish object the mask-sum block can read. + train_microbatch reads ``data["sample_mask"]``, ``data["token_mask"]``, + and (only as a fallback for the no-token-mask path) ``data["input_ids"]``.""" + # 8 samples, all valid (mask=1); 256 valid tokens each + sample_mask = torch.ones(8, dtype=torch.float32) + token_mask = torch.ones(8, 257, dtype=torch.float32) # token_mask[:, 1:] → 256 toks + input_ids = torch.zeros(8, 257, dtype=torch.long) + return { + "sample_mask": sample_mask, + "token_mask": token_mask, + "input_ids": input_ids, + } + + +# ── BEGIN ──────────────────────────────────────────────────────────────── + + +class TestBegin: + def test_opens_state(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("step-0", loss_fn=w._test_loss_fn, gbs=16, mbs=4) + assert w._train_step_state is not None + assert w._train_step_state["step_id"] == "step-0" + assert w._train_step_state["loss_type"] == LossType.TOKEN_LEVEL + assert w._train_step_state["gbs"] == 16 + assert w._train_step_state["mbs"] == 4 + assert w._train_step_state["total_num_microbatches"] == 0 + + def test_calls_zero_grad_and_zero_grad_buffer(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("step-0", loss_fn=w._test_loss_fn) + w.model.zero_grad_buffer.assert_called_once() + w.optimizer.zero_grad.assert_called_once() + w.model.train.assert_called_once() + + def test_saves_and_nulls_grad_sync_func(self, mock_module_symbols): + """The PP scheduler's direct reduce dispatch must be suppressed + for the duration of the step. Otherwise PP>1 silently corrupts + grads even when ``no_sync`` is set on the bucket groups.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + assert w.model.config.grad_sync_func == "ORIGINAL_GRAD_SYNC_FUNC" + w.begin_train_step("step-0", loss_fn=w._test_loss_fn) + assert w.model.config.grad_sync_func is None + assert w._train_step_state["saved_grad_sync_func"] == "ORIGINAL_GRAD_SYNC_FUNC" + + def test_double_begin_raises(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("step-0", loss_fn=w._test_loss_fn) + with pytest.raises(RuntimeError, match="already open"): + w.begin_train_step("step-1", loss_fn=w._test_loss_fn) + + def test_uses_cfg_defaults_when_gbs_mbs_omitted(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("step-0", loss_fn=w._test_loss_fn) + assert w._train_step_state["gbs"] == w.cfg["train_global_batch_size"] + assert w._train_step_state["mbs"] == w.cfg["train_micro_batch_size"] + + +# ── _assert_step_open ──────────────────────────────────────────────────── + + +class TestAssertStepOpen: + def test_raises_when_no_step_open(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + with pytest.raises(RuntimeError, match="no train step open"): + w._assert_step_open("step-0") + + def test_raises_on_step_id_mismatch(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("step-correct", loss_fn=w._test_loss_fn) + with pytest.raises(RuntimeError, match="step_id mismatch"): + w._assert_step_open("step-WRONG") + + def test_train_microbatch_without_begin_raises(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + with pytest.raises(RuntimeError, match="no train step open"): + w.train_microbatch("step-0", _fake_batch()) + + def test_finish_without_begin_raises(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + with pytest.raises(RuntimeError, match="no train step open"): + w.finish_train_step("step-0") + + +# ── train_microbatch ───────────────────────────────────────────────────── + + +class TestTrainMicrobatch: + def test_wraps_forward_backward_in_no_sync(self, mock_module_symbols): + """The single most important assertion in this file. Without the + no_sync wrap, mcore DDP dispatches a per-call cross-DP reduce on + the partially-accumulated buffer — silently corrupting grads.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + # no_sync() must have been ENTERED (called as a context manager). + # MagicMock with __enter__/__exit__ records the __enter__ call. + ctx = w.model.no_sync.return_value + ctx.__enter__.assert_called() + ctx.__exit__.assert_called() + + def test_invokes_megatron_forward_backward_once(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + assert mock_module_symbols["mfb"].call_count == 1 + + def test_passes_placeholder_n_one_to_loss(self, mock_module_symbols): + """The N=1 trick: loss must be called with global_valid_*=1 so it + returns un-normalized sums; finish does the 1/N rescale.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + kwargs = mock_module_symbols["mfb"].call_args.kwargs + # placeholder_n is a tensor(1.0) + assert "global_valid_seqs" in kwargs + assert "global_valid_toks" in kwargs + assert float(kwargs["global_valid_seqs"].item()) == pytest.approx(1.0) + assert float(kwargs["global_valid_toks"].item()) == pytest.approx(1.0) + + def test_accumulates_mask_sums_across_calls(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + # _fake_batch has sample_mask sum = 8, token_mask*sample_mask sum = 8*256 = 2048 + w.train_microbatch("s0", _fake_batch()) + assert float(w._train_step_state["local_valid_seqs"].item()) == pytest.approx( + 8.0 + ) + assert float(w._train_step_state["local_valid_toks"].item()) == pytest.approx( + 2048.0 + ) + w.train_microbatch("s0", _fake_batch()) + assert float(w._train_step_state["local_valid_seqs"].item()) == pytest.approx( + 16.0 + ) + assert float(w._train_step_state["local_valid_toks"].item()) == pytest.approx( + 4096.0 + ) + + def test_total_num_microbatches_accumulates(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + # get_microbatch_iterator mock returns num_microbatches=2 per call + w.train_microbatch("s0", _fake_batch()) + w.train_microbatch("s0", _fake_batch()) + w.train_microbatch("s0", _fake_batch()) + assert w._train_step_state["total_num_microbatches"] == 6 + + def test_does_not_call_optimizer_step(self, mock_module_symbols): + """trainer_version semantics: optimizer.step() must NOT fire + per train_microbatch — only at finish.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + w.train_microbatch("s0", _fake_batch()) + w.optimizer.step.assert_not_called() + + +# ── finish_train_step ──────────────────────────────────────────────────── + + +class TestFinish: + def _setup_open_step(self, mock_module_symbols, loss_type): + w = _make_worker(loss_type) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + return w + + def test_rescales_grads_with_inv_n(self, mock_module_symbols): + """The 1/N rescale must happen ON the local main_grad BEFORE the + cross-DP reduce — otherwise the reduce sees un-rescaled sums.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + w.finish_train_step("s0") + # scale_gradients should have been called with some 1/N scalar < 1 + w.model.scale_gradients.assert_called_once() + arg = w.model.scale_gradients.call_args.args[0] + assert 0 < arg <= 1.0 + + def test_start_then_finish_grad_sync_called_after_rescale( + self, mock_module_symbols + ): + """Call order matters: scale_gradients -> start_grad_sync -> + finish_grad_sync -> optimizer.step.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + # Record call order via a shared list + order: list[str] = [] + w.model.scale_gradients.side_effect = lambda s: order.append("scale") + w.model.start_grad_sync.side_effect = lambda: order.append("start_sync") + w.model.finish_grad_sync.side_effect = lambda: order.append("finish_sync") + w.optimizer.step.side_effect = lambda: ( + order.append("opt_step") or (True, 0.5, 0) + ) + w.finish_train_step("s0") + assert order == ["scale", "start_sync", "finish_sync", "opt_step"] + + def test_picks_global_valid_toks_for_token_level_loss(self, mock_module_symbols): + """N selection: TOKEN_LEVEL → N = global_valid_toks (not seqs).""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + w.finish_train_step("s0") + # local_valid_toks accumulated = 2048; with mocked all_reduce as no-op, + # global_valid_toks == 2048 → inv_n = 1/2048 + arg = w.model.scale_gradients.call_args.args[0] + assert arg == pytest.approx(1.0 / 2048.0, rel=1e-4) + + def test_picks_global_valid_seqs_for_sequence_level_loss(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.SEQUENCE_LEVEL) + w.finish_train_step("s0") + # local_valid_seqs = 8 → inv_n = 1/8 + arg = w.model.scale_gradients.call_args.args[0] + assert arg == pytest.approx(1.0 / 8.0, rel=1e-4) + + def test_restores_grad_sync_func(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + w.finish_train_step("s0") + assert w.model.config.grad_sync_func == "ORIGINAL_GRAD_SYNC_FUNC" + + def test_clears_train_step_state(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + w.finish_train_step("s0") + assert w._train_step_state is None + + def test_calls_scheduler_step_with_increment_gbs(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + w._train_step_state["gbs"] = 64 + w.finish_train_step("s0") + w.scheduler.step.assert_called_once_with(increment=64) + + def test_returns_metrics_dict(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + metrics = w.finish_train_step("s0") + for key in ( + "global_loss", + "rank", + "gpu_name", + "model_dtype", + "all_mb_metrics", + "grad_norm", + ): + assert key in metrics, f"missing {key!r}" + + def test_moe_branch_skipped_when_num_experts_is_none(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = self._setup_open_step(mock_module_symbols, LossType.TOKEN_LEVEL) + w.model.config.num_moe_experts = None + metrics = w.finish_train_step("s0") + assert "moe_metrics" not in metrics + + def test_moe_branch_uses_total_num_microbatches_for_scale( + self, mock_module_symbols + ): + """MoE aux-loss scale must use the accumulated total, not the + per-call num_microbatches.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.model.config.num_moe_experts = 4 + # Have get_moe_metrics return non-empty so the branch fires + mock_module_symbols["moe"].return_value = {"aux_loss": 0.1} + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + # 3 train_microbatch calls × 2 pipeline mbs each = 6 + for _ in range(3): + w.train_microbatch("s0", _fake_batch()) + w.finish_train_step("s0") + # get_moe_metrics receives loss_scale=1/6 + kwargs = mock_module_symbols["moe"].call_args.kwargs + assert kwargs["loss_scale"] == pytest.approx(1.0 / 6.0, rel=1e-6) + + +# ── abort_train_step ───────────────────────────────────────────────────── + + +class TestAbort: + def test_restores_grad_sync_func(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.abort_train_step("s0") + assert w.model.config.grad_sync_func == "ORIGINAL_GRAD_SYNC_FUNC" + + def test_zero_grad_buffer_and_zero_grad_called(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.model.zero_grad_buffer.reset_mock() + w.optimizer.zero_grad.reset_mock() + w.abort_train_step("s0") + w.model.zero_grad_buffer.assert_called_once() + w.optimizer.zero_grad.assert_called_once() + + def test_does_not_call_optimizer_step(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + w.abort_train_step("s0") + w.optimizer.step.assert_not_called() + + def test_clears_train_step_state(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.abort_train_step("s0") + assert w._train_step_state is None + + def test_idempotent_with_no_open_step(self, mock_module_symbols): + """abort is a no-op when nothing is open.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + # Should not raise + w.abort_train_step("s0") + assert getattr(w, "_train_step_state", None) is None + + def test_mismatched_step_id_raises(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + with pytest.raises(RuntimeError, match="does not match open step"): + w.abort_train_step("s-WRONG") + + def test_can_begin_new_step_after_abort(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + w.train_microbatch("s0", _fake_batch()) + w.abort_train_step("s0") + # New step opens cleanly + w.begin_train_step("s1", loss_fn=w._test_loss_fn) + assert w._train_step_state["step_id"] == "s1" + assert float(w._train_step_state["local_valid_seqs"].item()) == 0.0 + + +# ── grad_sync_func full lifecycle (integration of begin → finish/abort) ─ + + +class TestGradSyncFuncLifecycle: + def test_begin_finish_round_trip(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + sentinel = "MY_CUSTOM_GRAD_SYNC" + w.model.config.grad_sync_func = sentinel + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + assert w.model.config.grad_sync_func is None + w.train_microbatch("s0", _fake_batch()) + w.finish_train_step("s0") + assert w.model.config.grad_sync_func == sentinel + + def test_begin_abort_round_trip(self, mock_module_symbols): + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + sentinel = "MY_CUSTOM_GRAD_SYNC" + w.model.config.grad_sync_func = sentinel + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + assert w.model.config.grad_sync_func is None + w.abort_train_step("s0") + assert w.model.config.grad_sync_func == sentinel + + def test_handles_originally_none_grad_sync_func(self, mock_module_symbols): + """When PP=1 (or align_grad_reduce=False), grad_sync_func is None + to begin with. begin → finish must leave it as None.""" + from nemo_rl.algorithms.loss.interfaces import LossType + + w = _make_worker(LossType.TOKEN_LEVEL) + w.model.config.grad_sync_func = None + w.begin_train_step("s0", loss_fn=w._test_loss_fn) + assert w.model.config.grad_sync_func is None + w.train_microbatch("s0", _fake_batch()) + w.finish_train_step("s0") + assert w.model.config.grad_sync_func is None diff --git a/tests/unit/single_controller/__init__.py b/tests/unit/single_controller/__init__.py new file mode 100644 index 00000000000..4fc25d0d3c9 --- /dev/null +++ b/tests/unit/single_controller/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py new file mode 100644 index 00000000000..b4d5f291b3d --- /dev/null +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -0,0 +1,317 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end test: SC._rollout_pump writes the expected rows to TQ.""" + +from __future__ import annotations + +import time +from typing import Any + +import ray +import torch +from tensordict import TensorDict + +from nemo_rl.algorithms.async_utils.replay_buffer import TQReplayBuffer +from nemo_rl.algorithms.single_controller import SingleControllerActor +from nemo_rl.algorithms.single_controller_utils import ( + AsyncRLConfig, + MasterConfig, + SingleControllerBundle, +) +from nemo_rl.data_plane.adapters.noop import NoOpDataPlaneClient +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.experience.rollout_manager import RolloutManager + +# Reuse fixtures from the experience tests; same shape as test_async_rollout_manager. +from tests.unit.experience.test_rollout_manager import ( + single_multi_step_calculator_input_sample, # noqa: F401 +) +from tests.unit.experience.test_rollouts import ( + initial_multi_step_calculator_batch, # noqa: F401 + multi_step_calculator_environment, # noqa: F401 + multi_step_setup_vllm_async, # noqa: F401 + rollout_cluster, # noqa: F401 + rollout_tokenizer, # noqa: F401 +) + +_PARTITION_ID = "rollout_data" +# TQReplayBuffer.add tensorizes each PromptGroupRecord and writes +# ``generations_per_prompt`` training rows directly to TQ. +_BULK_FIELDS = [ + "input_ids", + "input_lengths", + "generation_logprobs", + "token_mask", + "sample_mask", + "prompt_ids_for_adv", + "total_reward", +] + + +@ray.remote(num_cpus=0) +class _TQActor: + """Ray-wrapped NoOpDataPlaneClient for cross-process TQ inspection.""" + + def __init__( + self, + partition_id: str, + fields: list[str], + num_samples: int, + consumer_tasks: list[str], + ) -> None: + self._client = NoOpDataPlaneClient() + self._client.register_partition( + partition_id=partition_id, + fields=list(fields), + num_samples=int(num_samples), + consumer_tasks=list(consumer_tasks), + ) + + def put_samples( + self, + sample_ids: list[str], + partition_id: str, + fields: TensorDict | None = None, + tags: list[dict[str, Any]] | None = None, + ) -> Any: + return self._client.put_samples( + sample_ids=sample_ids, + partition_id=partition_id, + fields=fields, + tags=tags, + ) + + def claim_meta(self, **kwargs: Any) -> Any: + return self._client.claim_meta(**kwargs) + + def get_samples( + self, + sample_ids: list[str], + partition_id: str, + select_fields: list[str], + ) -> TensorDict: + return self._client.get_samples( + sample_ids=sample_ids, + partition_id=partition_id, + select_fields=list(select_fields), + ) + + def get_tags( + self, partition_id: str, sample_ids: list[str] + ) -> list[dict[str, Any]]: + rec = self._client._partitions[partition_id] + return [dict(rec.tags.get(sid, {})) for sid in sample_ids] + + def peek_count(self, partition_id: str) -> int: + return len(self._client._partitions[partition_id].rows) + + +class _SyncDPAdapter: + """Sync DataPlaneClient over a Ray actor handle. Pads nested tensors before transport.""" + + def __init__(self, handle: Any) -> None: + self._handle = handle + + def put_samples( + self, + sample_ids: list[str], + partition_id: str, + fields: TensorDict | None = None, + tags: list[dict[str, Any]] | None = None, + ) -> Any: + if fields is not None: + fields = self._padded(fields) + return ray.get( + self._handle.put_samples.remote( + sample_ids=sample_ids, + partition_id=partition_id, + fields=fields, + tags=tags, + ) + ) + + @staticmethod + def _padded(td: TensorDict) -> TensorDict: + out: dict[str, torch.Tensor] = {} + for k in td.keys(): + v = td.get(k) + if isinstance(v, torch.Tensor) and v.is_nested: + v = torch.nested.to_padded_tensor(v, padding=0) + out[k] = v + return TensorDict(out, batch_size=td.batch_size) + + +def test_rollout_pump_writes_expected_tq_data( + multi_step_setup_vllm_async, # noqa: F811 + single_multi_step_calculator_input_sample, # noqa: F811 +): + """SC._rollout_pump writes max_rollout_prompts * num_generations rows to TQ with the expected fields and tags.""" + vllm_generation, tokenizer, env_handles, _, _ = multi_step_setup_vllm_async + input_sample = single_multi_step_calculator_input_sample + + num_generations = 2 + max_rollout_prompts = 2 + # TQReplayBuffer.add writes ``num_generations`` training rows per prompt. + expected_samples = max_rollout_prompts * num_generations + max_seq_len = 1024 + max_rollout_turns = input_sample["extra_env_info"]["max_steps"] + 1 + + tq_actor = _TQActor.remote( + partition_id=_PARTITION_ID, + fields=_BULK_FIELDS, + num_samples=expected_samples * 4, + consumer_tasks=["train"], + ) + dp_adapter = _SyncDPAdapter(tq_actor) + + mc = MasterConfig.model_construct( + grpo={ + "max_num_steps": 1, + "max_num_epochs": None, + "num_generations_per_prompt": num_generations, + }, + async_rl=AsyncRLConfig( + batch_selection_strategy="strict_on_policy", + max_weight_staleness_versions=0, + min_prompt_groups_per_batch=1, + max_inflight_prompts=max_rollout_prompts, + max_buffered_rollouts=max_rollout_prompts, + ), + ) + # Wrap each value in a single-element list so size==1 and v[0] returns the original field. + batched_sample = BatchedDataDict({k: [v] for k, v in input_sample.items()}) + dataloader = [batched_sample] * max_rollout_prompts + + tq_buffer = TQReplayBuffer( + dp_adapter, + partition_id=_PARTITION_ID, + pad_value_dict={"token_ids": int(tokenizer.pad_token_id or 0)}, + ) + rollout_manager = RolloutManager( + tokenizer=tokenizer, + env_handles=env_handles, + num_generations_per_prompt=num_generations, + max_seq_len=max_seq_len, + max_rollout_turns=max_rollout_turns, + policy_generation=vllm_generation, + use_nemo_gym=False, + tq_buffer=tq_buffer, + ) + bundle = SingleControllerBundle( + gen_handle=vllm_generation, + trainer_handle=object(), + env_handles=env_handles, + train_cluster=None, + inference_cluster=None, + dp_client=dp_adapter, + dataloader=dataloader, + weight_synchronizer=object(), + advantage_estimator=None, + loss_fn=None, + rollout_manager=rollout_manager, + tq_buffer=tq_buffer, + partition_id=_PARTITION_ID, + ) + ctrl = SingleControllerActor.remote(master_config=mc, bundle=bundle) + + vllm_generation.prepare_for_generation() + + # _rollout_pump runs until cancelled, so poll TQ then cancel. + pump_ref = ctrl._rollout_pump.remote() + deadline = time.monotonic() + 120.0 + while time.monotonic() < deadline: + if ray.get(tq_actor.peek_count.remote(_PARTITION_ID)) >= expected_samples: + break + time.sleep(0.5) + assert ray.get(tq_actor.peek_count.remote(_PARTITION_ID)) >= expected_samples, ( + "rollout_pump did not push expected_samples within timeout" + ) + ray.cancel(pump_ref) + try: + ray.get(pump_ref) + except (ray.exceptions.RayTaskError, ray.exceptions.TaskCancelledError): + pass + + vllm_generation.finish_generation() + + meta = ray.get( + tq_actor.claim_meta.remote( + partition_id=_PARTITION_ID, + task_name="train", + required_fields=_BULK_FIELDS, + batch_size=expected_samples * 4, + blocking=False, + timeout_s=0.0, + ) + ) + assert meta.size == expected_samples + + # pack_payload stamps sample_ids as ``{group_uuid}_g{i}``. + group_ids: set[str] = set() + for sid in meta.sample_ids: + head, _, tail = sid.rpartition("_g") + assert head and tail.isdigit(), f"unexpected sample_id: {sid}" + group_ids.add(head) + assert len(group_ids) == max_rollout_prompts + + bulk = ray.get( + tq_actor.get_samples.remote( + sample_ids=meta.sample_ids, + partition_id=_PARTITION_ID, + select_fields=_BULK_FIELDS, + ) + ) + assert set(bulk.keys()) >= set(_BULK_FIELDS), ( + f"missing bulk fields: {set(_BULK_FIELDS) - set(bulk.keys())}" + ) + + input_lengths = bulk["input_lengths"].long() + assert input_lengths.shape[0] == expected_samples + assert torch.all(input_lengths > 0) + assert torch.allclose( + bulk["sample_mask"].float(), + torch.ones(expected_samples, dtype=torch.float32), + ) + + # Same deterministic prompt as test_async_rollout_manager: the model + # solves the calculator task every time -> reward == 1.0 and decoded + # tail contains " 16". + rewards = bulk["total_reward"].float().flatten() + assert rewards.shape == (expected_samples,) + assert torch.allclose(rewards, torch.ones(expected_samples)), ( + f"expected all rewards == 1.0, got {rewards.tolist()}" + ) + + input_ids = bulk["input_ids"] + token_mask = bulk["token_mask"] + for i in range(expected_samples): + length = int(input_lengths[i]) + decoded = tokenizer.decode( + input_ids[i, :length].tolist(), skip_special_tokens=False + ) + assert " 16" in decoded[-64:], ( + f"sample {i}: decoded tail {decoded[-64:]!r} missing ' 16'" + ) + assert int(token_mask[i, :length].sum().item()) > 0, ( + f"sample {i}: token_mask has no assistant tokens" + ) + + tags = ray.get( + tq_actor.get_tags.remote(partition_id=_PARTITION_ID, sample_ids=meta.sample_ids) + ) + for tag in tags: + assert tag["weight_version"] == 0 + # Slim tag schema: weight_version is the only field producers stamp. + assert set(tag) == {"weight_version"} diff --git a/tests/unit/single_controller/test_single_controller_setup.py b/tests/unit/single_controller/test_single_controller_setup.py new file mode 100644 index 00000000000..1f526bd0558 --- /dev/null +++ b/tests/unit/single_controller/test_single_controller_setup.py @@ -0,0 +1,294 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for setup_single_controller (factories monkey-patched).""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +import nemo_rl.algorithms.single_controller_utils.setup as sc_setup_mod +from nemo_rl.algorithms.loss import ClippedPGLossConfig +from nemo_rl.algorithms.single_controller_utils import ( + MasterConfig, + SingleControllerBundle, + setup_single_controller, +) + + +def _make_master_config( + *, + dp_enabled: bool = True, + use_multiple_dataloader: bool = False, + colocated: bool = True, + backend: str = "vllm", + megatron_enabled: bool = False, + env: dict | None = None, + max_num_steps: int = 100, + max_num_epochs: int = 1, + num_prompts_per_step: int = 4, +) -> MasterConfig: + """Build a partially-populated MasterConfig for unit tests. + + Cross-cutting components (cluster/checkpointing/...) are required by pydantic for + normal load but unused here — model_construct skips validation, and we hand-fill + only the dict-shaped fields setup reads. + """ + return MasterConfig.model_construct( + data_plane={"enabled": dp_enabled, "impl": "transfer_queue"}, + data={ + "use_multiple_dataloader": use_multiple_dataloader, + "shuffle": False, + "num_workers": 0, + "train": [{"env_name": "math"}], + }, + grpo={ + "max_num_steps": max_num_steps, + "max_num_epochs": max_num_epochs, + "num_prompts_per_step": num_prompts_per_step, + "num_generations_per_prompt": 2, + "max_rollout_turns": 1, + }, + policy={ + "max_total_sequence_length": 32, + "megatron_cfg": {"enabled": megatron_enabled}, + "generation": { + "backend": backend, + "colocated": {"enabled": colocated, "resources": {}}, + }, + }, + loss_fn=ClippedPGLossConfig(), + env=env if env is not None else {}, + ) + + +@pytest.fixture +def patched_factories(): + """Patch every external factory setup calls. + + Returns a dict of mocks keyed by name so individual tests can assert on call args + without re-importing the patch handles. + """ + fake_dataset = list(range(8)) + fake_dataloader = MagicMock(name="dataloader") + # len(dataloader) used by the Megatron train_iters injection. + fake_dataloader.__len__ = MagicMock(return_value=4) + fake_env_handles = {"math": MagicMock(name="math_env")} + + with ( + patch.object( + sc_setup_mod, + "setup_response_data", + return_value=(fake_dataset, None, fake_env_handles, {}), + ) as mock_setup_response, + patch.object( + sc_setup_mod, + "StatefulDataLoader", + return_value=fake_dataloader, + ) as mock_dataloader, + patch.object( + sc_setup_mod, + "_build_clusters", + return_value=( + MagicMock(name="train_cluster"), + MagicMock(name="inference_cluster"), + ), + ) as mock_clusters, + patch.object( + sc_setup_mod, "_build_generation", return_value=MagicMock(name="gen") + ) as mock_gen, + patch.object( + sc_setup_mod, "_build_trainer", return_value=MagicMock(name="policy") + ) as mock_trainer, + patch.object( + sc_setup_mod, + "build_data_plane_client", + return_value=MagicMock(name="dp_client"), + ) as mock_dp_client, + patch.object( + sc_setup_mod, + "create_weight_synchronizer", + return_value=MagicMock(name="weight_sync"), + ) as mock_weight_sync, + patch.object( + sc_setup_mod, + "_create_advantage_estimator", + return_value=MagicMock(name="adv"), + ) as mock_adv, + patch.object( + sc_setup_mod, "ClippedPGLossFn", return_value=MagicMock(name="loss_fn") + ) as mock_loss, + patch.object( + sc_setup_mod, + "_generation_max_seq_len", + return_value=32, + ), + ): + yield { + "setup_response_data": mock_setup_response, + "StatefulDataLoader": mock_dataloader, + "_build_clusters": mock_clusters, + "_build_generation": mock_gen, + "_build_trainer": mock_trainer, + "build_data_plane_client": mock_dp_client, + "create_weight_synchronizer": mock_weight_sync, + "_create_advantage_estimator": mock_adv, + "ClippedPGLossFn": mock_loss, + "dataloader": fake_dataloader, + "env_handles": fake_env_handles, + } + + +class TestSetup: + """setup arg validation + bundle assembly.""" + + def test_raises_when_data_plane_disabled(self): + mc = _make_master_config(dp_enabled=False) + with pytest.raises(ValueError, match="data_plane.enabled=True"): + setup_single_controller(mc, MagicMock()) + + def test_multiple_dataloader_not_supported(self): + mc = _make_master_config(use_multiple_dataloader=True) + with pytest.raises(NotImplementedError, match="use_multiple_dataloader"): + setup_single_controller(mc, MagicMock(pad_token_id=0)) + + def test_returns_bundle(self, patched_factories): + mc = _make_master_config(colocated=True) + tokenizer = MagicMock(pad_token_id=0) + + bundle = setup_single_controller(mc, tokenizer) + + assert isinstance(bundle, SingleControllerBundle) + assert bundle.gen_handle is patched_factories["_build_generation"].return_value + assert bundle.trainer_handle is patched_factories["_build_trainer"].return_value + assert bundle.env_handles is patched_factories["env_handles"] + assert ( + bundle.dp_client + is patched_factories["build_data_plane_client"].return_value + ) + assert bundle.dataloader is patched_factories["dataloader"] + assert bundle.weight_synchronizer is ( + patched_factories["create_weight_synchronizer"].return_value + ) + assert bundle.advantage_estimator is ( + patched_factories["_create_advantage_estimator"].return_value + ) + assert bundle.loss_fn is patched_factories["ClippedPGLossFn"].return_value + # tq_buffer + rollout_manager are constructed inline (not mocked). + assert bundle.tq_buffer is not None + assert bundle.rollout_manager is not None + # rollout_manager binds the same tq_buffer for the writer + sampler. + assert bundle.rollout_manager._tq_buffer is bundle.tq_buffer + # tq_buffer wires the dp_client + default partition. + assert bundle.tq_buffer._dp_client is bundle.dp_client + assert bundle.partition_id == "rollout_data" + assert bundle.tq_buffer._partition_id == "rollout_data" + + def test_env_handles_sourced_from_setup_response_data(self, patched_factories): + """setup_response_data receives master_config.env and supplies env handles.""" + math_env_cfg = {"some": "value"} + mc = _make_master_config(env={"math": math_env_cfg}) + + bundle = setup_single_controller(mc, MagicMock(pad_token_id=0)) + + _, call_kwargs = patched_factories["setup_response_data"].call_args + assert call_kwargs["env_configs"] == {"math": math_env_cfg} + assert bundle.env_handles is patched_factories["env_handles"] + + def test_weight_sync_factory_args(self, patched_factories): + """create_weight_synchronizer receives policy / generation / topology.""" + mc = _make_master_config(colocated=False, backend="vllm") + tokenizer = MagicMock(pad_token_id=0) + + setup_single_controller(mc, tokenizer) + + _, factory_kwargs = patched_factories["create_weight_synchronizer"].call_args + assert ( + factory_kwargs["policy"] is patched_factories["_build_trainer"].return_value + ) + assert ( + factory_kwargs["generation"] + is patched_factories["_build_generation"].return_value + ) + assert factory_kwargs["generation_backend"] == "vllm" + assert factory_kwargs["colocated"] is False + + def test_custom_partition_id(self, patched_factories): + mc = _make_master_config() + tokenizer = MagicMock(pad_token_id=7) + + bundle = setup_single_controller(mc, tokenizer, partition_id="custom_partition") + + assert bundle.partition_id == "custom_partition" + assert bundle.tq_buffer._partition_id == "custom_partition" + assert bundle.tq_buffer._pad_value_dict == { + "token_ids": 7, + "input_ids": 7, + } + + def test_max_num_steps_capped_by_self(self, patched_factories): + """grpo.max_num_steps stays put when smaller than max_num_epochs * len(dl).""" + mc = _make_master_config( + megatron_enabled=False, + max_num_steps=2, + max_num_epochs=1, + ) + # patched dataloader has len() == 4, so the min picks max_num_steps. + setup_single_controller(mc, MagicMock(pad_token_id=0)) + + assert mc.grpo["max_num_steps"] == 2 + + def test_max_num_steps_capped_by_dataloader_epochs(self, patched_factories): + """grpo.max_num_steps drops to max_num_epochs * len(dataloader) when smaller.""" + mc = _make_master_config( + megatron_enabled=False, + max_num_steps=1000, + max_num_epochs=2, + ) + # patched dataloader has len() == 4 → 2 * 4 = 8 < 1000. + setup_single_controller(mc, MagicMock(pad_token_id=0)) + + assert mc.grpo["max_num_steps"] == 8 + + def test_megatron_train_iters_capped_by_max_num_steps(self, patched_factories): + """train_iters = min(max_num_steps, max_num_epochs * len(dataloader)).""" + mc = _make_master_config( + megatron_enabled=True, + max_num_steps=2, + max_num_epochs=1, + ) + # patched dataloader has len() == 4, so the min picks max_num_steps. + setup_single_controller(mc, MagicMock(pad_token_id=0)) + + assert mc.policy["megatron_cfg"]["train_iters"] == 2 + + def test_megatron_train_iters_capped_by_dataloader_epochs(self, patched_factories): + """train_iters drops to max_num_epochs * len(dataloader) when smaller.""" + mc = _make_master_config( + megatron_enabled=True, + max_num_steps=1000, + max_num_epochs=2, + ) + # patched dataloader has len() == 4 → 2 * 4 = 8 < 1000. + setup_single_controller(mc, MagicMock(pad_token_id=0)) + + assert mc.policy["megatron_cfg"]["train_iters"] == 8 + + def test_megatron_train_iters_not_set_when_disabled(self, patched_factories): + mc = _make_master_config(megatron_enabled=False) + setup_single_controller(mc, MagicMock(pad_token_id=0)) + + assert "train_iters" not in mc.policy.get("megatron_cfg", {}) diff --git a/tests/unit/single_controller/test_staleness_sampler.py b/tests/unit/single_controller/test_staleness_sampler.py new file mode 100644 index 00000000000..ce15f34acf8 --- /dev/null +++ b/tests/unit/single_controller/test_staleness_sampler.py @@ -0,0 +1,466 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for StalenessSampler (pure filter over TQReplayBuffer state).""" + +from __future__ import annotations + +import asyncio + +import pytest + +from nemo_rl.algorithms.async_utils.staleness_sampler import StalenessSampler +from nemo_rl.data_plane import KVBatchMeta + + +class FakeBuffer: + """Minimal TQReplayBuffer surface used by StalenessSampler tests.""" + + def __init__(self, partition_id: str = "rollout_data") -> None: + self._partition_id = partition_id + self.meta_list: list[KVBatchMeta | None] = [] + self.start_weight_list: list[int] = [] + self.end_weight_list: list[int] = [] + self.ready_list: list[bool] = [] + self.remove_calls: list[tuple[list[int], bool]] = [] + + def add( + self, + group_id: str, + weight: int, + group_size: int = 1, + ready: bool = True, + end_weight: int | None = None, + ) -> KVBatchMeta: + sample_ids = [f"{group_id}_g{i}" for i in range(group_size)] + meta = KVBatchMeta( + partition_id=self._partition_id, + task_name=None, + sample_ids=sample_ids, + tags=[{"weight_version": weight, "group_id": group_id}] * group_size, + ) + self.meta_list.append(meta if ready else None) + self.start_weight_list.append(weight) + self.end_weight_list.append(weight if end_weight is None else end_weight) + self.ready_list.append(ready) + return meta + + async def remove(self, idxs: list[int], remove_in_dp: bool) -> int: + self.remove_calls.append((list(idxs), remove_in_dp)) + for i in sorted(idxs, reverse=True): + del self.meta_list[i] + del self.start_weight_list[i] + del self.end_weight_list[i] + del self.ready_list[i] + return len(idxs) + + +def _run(coro): + return asyncio.run(coro) + + +class TestStalenessSamplerSelect: + def test_select_returns_none_when_insufficient(self): + buf = FakeBuffer() + buf.add("g0", weight=5) + sampler = StalenessSampler(buf, max_staleness_versions=2) + + result = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert result == (None, 0) + + def test_select_returns_none_on_empty_buffer(self): + buf = FakeBuffer() + sampler = StalenessSampler(buf, max_staleness_versions=2) + + result = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=1, max_prompt_groups=1 + ) + ) + assert result == (None, 0) + + def test_select_filters_by_staleness_window(self): + buf = FakeBuffer() + # Weights 3, 4, 5, 2, 6 against trainer=5, max_staleness=2: + # lags = 2, 1, 0, 3 (stale), -1 (future) + for i, w in enumerate([3, 4, 5, 2, 6]): + buf.add(f"g{i}", weight=w) + sampler = StalenessSampler( + buf, max_staleness_versions=2, sample_freshest_first=True + ) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + + assert selected is not None + # Freshest first → g2 (lag 0), g1 (lag 1) + assert selected.sample_ids == ["g2_g0", "g1_g0"] + assert num_groups == 2 + + def test_select_freshest_first_orders_by_lag(self): + buf = FakeBuffer() + for w in [3, 4, 5]: + buf.add(f"v{w}", weight=w) + sampler = StalenessSampler( + buf, max_staleness_versions=5, sample_freshest_first=True + ) + + selected, num_groups = _run( + sampler.select( + current_train_weight=6, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert selected is not None + assert selected.sample_ids == ["v5_g0", "v4_g0"] + assert num_groups == 2 + + def test_select_fifo_orders_by_insertion(self): + buf = FakeBuffer() + for w in [3, 4, 5]: + buf.add(f"v{w}", weight=w) + sampler = StalenessSampler( + buf, max_staleness_versions=5, sample_freshest_first=False + ) + + selected, num_groups = _run( + sampler.select( + current_train_weight=6, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert selected is not None + assert selected.sample_ids == ["v3_g0", "v4_g0"] + assert num_groups == 2 + + def test_select_skips_future_weight(self): + buf = FakeBuffer() + buf.add("now", weight=5) + buf.add("future", weight=7) + sampler = StalenessSampler(buf, max_staleness_versions=10) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=1, max_prompt_groups=1 + ) + ) + + assert selected is not None + assert selected.sample_ids == ["now_g0"] + assert num_groups == 1 + + def test_select_concats_groups(self): + buf = FakeBuffer() + buf.add("g0", weight=5, group_size=2) + buf.add("g1", weight=5, group_size=2) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + + assert selected is not None + assert selected.sample_ids == [ + "g0_g0", + "g0_g1", + "g1_g0", + "g1_g1", + ] + # Two groups concatenated, each of size 2 → 4 sample_ids total. + assert num_groups == 2 + + def test_select_strict_on_policy_requires_exact_version(self): + buf = FakeBuffer() + for i, w in enumerate([4, 5, 5, 6]): + buf.add(f"g{i}", weight=w) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + # 3 eligible (need weight=5), only have 2 + result = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=3, max_prompt_groups=3 + ) + ) + assert result == (None, 0) + + # Buffer still intact: select with min=3 returned None without dropping anything. + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert selected is not None + assert selected.sample_ids == ["g1_g0", "g2_g0"] + assert num_groups == 2 + + def test_select_drops_returned_entries_from_buffer(self): + buf = FakeBuffer() + for i, w in enumerate([5, 5, 5]): + buf.add(f"g{i}", weight=w) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + first_meta, first_num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=1, max_prompt_groups=1 + ) + ) + assert first_meta is not None + assert first_meta.sample_ids == ["g0_g0"] + assert first_num_groups == 1 + assert buf.start_weight_list == [5, 5] + # remove_in_dp=False; DP rows kept for trainer. + assert buf.remove_calls[-1][1] is False + + second_meta, second_num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=1, max_prompt_groups=1 + ) + ) + assert second_meta is not None + assert second_meta.sample_ids == ["g1_g0"] + assert second_num_groups == 1 + + def test_select_rejects_zero_min_prompt_groups(self): + buf = FakeBuffer() + sampler = StalenessSampler(buf, max_staleness_versions=0) + with pytest.raises(ValueError): + _run( + sampler.select( + current_train_weight=0, min_prompt_groups=0, max_prompt_groups=0 + ) + ) + + def test_select_rejects_max_less_than_min(self): + buf = FakeBuffer() + for i in range(3): + buf.add(f"g{i}", weight=5) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + with pytest.raises(ValueError): + _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=1 + ) + ) + + def test_select_caps_at_max_prompt_groups(self): + buf = FakeBuffer() + for i in range(5): + buf.add(f"g{i}", weight=5) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=3 + ) + ) + + assert selected is not None + # FIFO order; capped at max=3 even though 5 are eligible. + assert selected.sample_ids == ["g0_g0", "g1_g0", "g2_g0"] + assert num_groups == 3 + # The remaining two stay in the buffer. + assert buf.start_weight_list == [5, 5] + + def test_select_takes_all_available_when_between_min_and_max(self): + buf = FakeBuffer() + for i in range(3): + buf.add(f"g{i}", weight=5) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=8 + ) + ) + + assert selected is not None + assert selected.sample_ids == ["g0_g0", "g1_g0", "g2_g0"] + assert num_groups == 3 + assert buf.start_weight_list == [] + + +class TestStalenessSamplerEvict: + def test_evict_removes_stale_groups(self): + buf = FakeBuffer() + # trainer=5, max_staleness=1 → lag >1 means stale (weights 0, 1, 2 stale; 4, 5 fresh) + for i, w in enumerate([0, 1, 4, 5, 2]): + buf.add(f"g{i}", weight=w) + sampler = StalenessSampler(buf, max_staleness_versions=1) + + dropped = _run(sampler.evict(current_train_weight=5)) + + assert dropped == 3 + assert buf.start_weight_list == [4, 5] + # Survivors' sample_ids + assert [m.sample_ids[0] for m in buf.meta_list] == ["g2_g0", "g3_g0"] + + def test_evict_returns_zero_when_nothing_stale(self): + buf = FakeBuffer() + for w in [4, 5]: + buf.add(f"v{w}", weight=w) + sampler = StalenessSampler(buf, max_staleness_versions=1) + + assert _run(sampler.evict(current_train_weight=5)) == 0 + assert buf.remove_calls == [] + + def test_evict_keeps_future_groups(self): + buf = FakeBuffer() + buf.add("future", weight=7) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + assert _run(sampler.evict(current_train_weight=5)) == 0 + assert buf.start_weight_list == [7] + + def test_evict_drops_whole_group(self): + buf = FakeBuffer() + buf.add("stale", weight=1, group_size=4) + buf.add("fresh", weight=5, group_size=4) + sampler = StalenessSampler(buf, max_staleness_versions=1) + + dropped = _run(sampler.evict(current_train_weight=5)) + + assert dropped == 1 + assert buf.remove_calls == [([0], True)] + assert buf.start_weight_list == [5] + assert [m.sample_ids[0] for m in buf.meta_list] == ["fresh_g0"] + + +class TestStalenessSamplerInit: + def test_rejects_negative_max_staleness(self): + buf = FakeBuffer() + with pytest.raises(ValueError): + StalenessSampler(buf, max_staleness_versions=-1) + + def test_rejects_require_order_with_freshest_first(self): + buf = FakeBuffer() + with pytest.raises(ValueError): + StalenessSampler( + buf, + max_staleness_versions=0, + sample_freshest_first=True, + require_order=True, + ) + + +class TestStalenessSamplerReady: + def test_default_mode_skips_unready_slots(self): + buf = FakeBuffer() + buf.add("g0", weight=5, ready=False) + buf.add("g1", weight=5, ready=True) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=1, max_prompt_groups=1 + ) + ) + + assert selected is not None + assert selected.sample_ids == ["g1_g0"] + assert num_groups == 1 + + def test_default_mode_waits_when_too_few_ready(self): + buf = FakeBuffer() + buf.add("g0", weight=5, ready=False) + buf.add("g1", weight=5, ready=True) + sampler = StalenessSampler(buf, max_staleness_versions=0) + + result = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert result == (None, 0) + + +class TestStalenessSamplerRequireOrder: + def test_consumes_oldest_batch_first(self): + buf = FakeBuffer() + # Two complete batches: v=4 then v=5; require_order must take v=4 first. + for i, w in enumerate((4, 4, 5, 5)): + buf.add(f"v{w}_{i}", weight=w) + sampler = StalenessSampler(buf, max_staleness_versions=1, require_order=True) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + + assert selected is not None + # Insertion-order FIFO inside the oldest batch. + assert selected.sample_ids == ["v4_0_g0", "v4_1_g0"] + assert num_groups == 2 + assert buf.start_weight_list == [5, 5] + + def test_waits_when_oldest_batch_partially_ready(self): + buf = FakeBuffer() + # Oldest batch v=4 has 1 ready + 1 unready; v=5 batch is fully ready. + # require_order must NOT skip ahead to v=5. + buf.add("v4_a", weight=4, ready=True) + buf.add("v4_b", weight=4, ready=False) + buf.add("v5_a", weight=5, ready=True) + buf.add("v5_b", weight=5, ready=True) + sampler = StalenessSampler(buf, max_staleness_versions=1, require_order=True) + + result = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert result == (None, 0) + # Buffer untouched: nothing removed. + assert buf.start_weight_list == [4, 4, 5, 5] + assert buf.ready_list == [True, False, True, True] + + def test_returns_none_when_oldest_batch_not_filled(self): + buf = FakeBuffer() + buf.add("v4_a", weight=4, ready=True) + # Only 1 ready in oldest batch; need 2. + sampler = StalenessSampler(buf, max_staleness_versions=1, require_order=True) + + result = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + assert result == (None, 0) + + def test_ignores_future_versions_when_picking_target(self): + buf = FakeBuffer() + # Trainer at 5, staleness 1: window is [4, 5]; v=7 (future) must not + # become the oldest target. + buf.add("v7", weight=7, ready=True) + buf.add("v5_a", weight=5, ready=True) + buf.add("v5_b", weight=5, ready=True) + sampler = StalenessSampler(buf, max_staleness_versions=1, require_order=True) + + selected, num_groups = _run( + sampler.select( + current_train_weight=5, min_prompt_groups=2, max_prompt_groups=2 + ) + ) + + assert selected is not None + assert selected.sample_ids == ["v5_a_g0", "v5_b_g0"] + assert num_groups == 2 + assert buf.start_weight_list == [7] diff --git a/tests/unit/single_controller/test_tq_replay_buffer.py b/tests/unit/single_controller/test_tq_replay_buffer.py new file mode 100644 index 00000000000..6c586978478 --- /dev/null +++ b/tests/unit/single_controller/test_tq_replay_buffer.py @@ -0,0 +1,530 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for TQReplayBuffer (plain SC-process buffer + TQ proxy).""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +import torch + +import nemo_rl.algorithms.async_utils.replay_buffer as _replay_buffer_module +from nemo_rl.algorithms.async_utils.replay_buffer import TQReplayBuffer +from nemo_rl.data_plane import KVBatchMeta +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.experience.interfaces import PromptGroupRecord + +# Each record yields _N_GENS training rows. +_N_GENS = 2 + + +def _stub_record_to_train_batch( + record: PromptGroupRecord, *, pad_value_dict: Any +) -> BatchedDataDict[Any]: + del record, pad_value_dict + return BatchedDataDict[Any]( + { + "input_ids": torch.ones((_N_GENS, 3), dtype=torch.long), + "input_lengths": torch.full((_N_GENS,), 3, dtype=torch.long), + "total_reward": torch.zeros(_N_GENS, dtype=torch.float32), + } + ) + + +@pytest.fixture(autouse=True) +def _patch_converter(monkeypatch): + """Bypass the real ``record_to_train_batch`` so tests can use empty records.""" + monkeypatch.setattr( + _replay_buffer_module, + "record_to_train_batch", + _stub_record_to_train_batch, + ) + + +class FakeDataPlaneClient: + """Sync in-memory DataPlaneClient stub used by TQReplayBuffer tests.""" + + def __init__(self, partition_id: str = "rollout_data") -> None: + self._partition_id = partition_id + self._rows: dict[str, dict[str, Any]] = {} + self.put_calls: list[dict[str, Any]] = [] + self.clear_calls: list[list[str]] = [] + + def put_samples( + self, + sample_ids: list[str], + partition_id: str, + fields: Any = None, + tags: list[dict[str, Any]] | None = None, + ) -> KVBatchMeta: + assert partition_id == self._partition_id + self.put_calls.append( + { + "sample_ids": list(sample_ids), + "fields": fields, + "tags": [dict(t) for t in tags] if tags is not None else None, + } + ) + for i, sid in enumerate(sample_ids): + self._rows[sid] = { + "tag": dict(tags[i]) if tags is not None else {}, + } + return KVBatchMeta( + partition_id=partition_id, + task_name=None, + sample_ids=list(sample_ids), + fields=None, + tags=[dict(t) for t in tags] if tags is not None else None, + ) + + def clear_samples(self, sample_ids: list[str] | None, partition_id: str) -> None: + assert partition_id == self._partition_id + ids = list(sample_ids) if sample_ids is not None else list(self._rows) + self.clear_calls.append(list(ids)) + for sid in ids: + self._rows.pop(sid, None) + + def depth(self) -> int: + return len(self._rows) + + +def _run(coro): + return asyncio.run(coro) + + +def _make_record() -> PromptGroupRecord: + """Opaque PromptGroupRecord — converter is stubbed, so contents are unused.""" + return PromptGroupRecord( + prompt_idx=0, + prompt=[], + extra_env_info=None, + metadata={}, + completions=[], + rollout_metrics={}, + ) + + +def _make_buffer(dp: FakeDataPlaneClient) -> TQReplayBuffer: + return TQReplayBuffer( + dp, partition_id="rollout_data", pad_value_dict={"token_ids": 0} + ) + + +def _add_group( + buf: TQReplayBuffer, weight: int, end_weight: int | None = None +) -> KVBatchMeta: + if end_weight is None: + end_weight = weight + group_id = buf.reserve(weight_version=weight) + return _run( + buf.commit( + group_id, + _make_record(), + start_weight_version=weight, + end_weight_version=end_weight, + ) + ) + + +class TestTQReplayBufferReserveCommit: + def test_reserve_appends_placeholder_unready(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + + group_id = buf.reserve(weight_version=3) + + assert isinstance(group_id, str) and group_id + assert buf.size() == 1 + assert buf.start_weight_list == [3] + assert buf.end_weight_list == [-1] + assert buf.ready_list == [False] + assert buf.meta_list == [None] + assert dp.depth() == 0 + assert dp.put_calls == [] + + def test_commit_writes_tq_then_fills_meta(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + + group_id = buf.reserve(weight_version=3) + meta = _run( + buf.commit( + group_id, + _make_record(), + start_weight_version=3, + end_weight_version=4, + ) + ) + + # pack_payload stamps sample_ids as ``{group_uuid}_g{i}``. + assert len(meta.sample_ids) == _N_GENS + head, _, idx = meta.sample_ids[0].rpartition("_g") + assert head == group_id and idx == "0" + assert all(sid.startswith(group_id + "_g") for sid in meta.sample_ids) + assert dp.depth() == _N_GENS + assert buf.size() == 1 + assert buf.start_weight_list == [3] + assert buf.end_weight_list == [4] + assert buf.ready_list == [True] + assert buf.meta_list[0].sample_ids == meta.sample_ids + # TQ tag uses start_weight_version (dispatch time). + assert meta.tags == [{"weight_version": 3}] * _N_GENS + assert len(dp.put_calls) == 1 + + def test_commit_raises_for_unknown_group_id(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + buf.reserve(weight_version=3) + + with pytest.raises(ValueError): + _run( + buf.commit( + "not-a-real-id", + _make_record(), + start_weight_version=3, + end_weight_version=3, + ) + ) + + def test_reserve_then_commit_preserves_dispatch_order(self): + """Reserve in dispatch order, commit out of order; insertion order holds.""" + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + + weights = (1, 2, 3) + gids = [buf.reserve(weight_version=w) for w in weights] + # Commit out of order: 2, 0, 1 — buffer order must still match reserve order. + for i in (2, 0, 1): + _run( + buf.commit( + gids[i], + _make_record(), + start_weight_version=weights[i], + end_weight_version=weights[i], + ) + ) + + assert buf.size() == 3 + assert buf.start_weight_list == [1, 2, 3] + assert buf.end_weight_list == [1, 2, 3] + assert buf.ready_list == [True, True, True] + # sample_id head equals reserved group_id at each slot. + for i, gid in enumerate(gids): + assert buf.meta_list[i] is not None + assert buf.meta_list[i].sample_ids[0].startswith(gid + "_g") + + def test_commit_appends_multiple_records_in_order(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + + metas = [_add_group(buf, weight=w) for w in (1, 2, 3)] + + assert buf.size() == 3 + assert buf.start_weight_list == [1, 2, 3] + assert buf.end_weight_list == [1, 2, 3] + assert [m.sample_ids for m in buf.meta_list] == [ + list(metas[0].sample_ids), + list(metas[1].sample_ids), + list(metas[2].sample_ids), + ] + + +class TestTQReplayBufferRemove: + def test_remove_drops_indices_and_clears_dp_when_requested(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + metas = [_add_group(buf, weight=g) for g in range(3)] + + n = _run(buf.remove([0, 2], remove_in_dp=True)) + + assert n == 2 + assert buf.size() == 1 + assert buf.start_weight_list == [1] + assert buf.end_weight_list == [1] + assert buf.meta_list[0].sample_ids == list(metas[1].sample_ids) + assert dp.depth() == _N_GENS + assert set(dp._rows) == set(metas[1].sample_ids) + + def test_remove_without_dp_keeps_rows(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + metas = [_add_group(buf, weight=g) for g in range(2)] + + n = _run(buf.remove([0], remove_in_dp=False)) + + assert n == 1 + assert buf.size() == 1 + assert buf.start_weight_list == [1] + assert buf.end_weight_list == [1] + assert buf.meta_list[0].sample_ids == list(metas[1].sample_ids) + assert dp.clear_calls == [] + assert dp.depth() == 2 * _N_GENS + + def test_remove_rejects_out_of_range_before_mutating(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + metas = [_add_group(buf, weight=g) for g in range(2)] + + with pytest.raises(IndexError, match=r"out of range: 5; size=2"): + _run(buf.remove([0, 5], remove_in_dp=True)) + + assert buf.size() == 2 + assert [m.sample_ids for m in buf.meta_list] == [ + list(metas[0].sample_ids), + list(metas[1].sample_ids), + ] + assert dp.depth() == 2 * _N_GENS + assert dp.clear_calls == [] + + def test_remove_empty_is_noop(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + _add_group(buf, weight=0) + _add_group(buf, weight=0) + + n = _run(buf.remove([], remove_in_dp=True)) + + assert n == 0 + assert buf.size() == 2 + assert dp.depth() == 2 * _N_GENS + assert dp.clear_calls == [] + + +class TestTQReplayBufferSize: + def test_size_and_len(self): + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + assert buf.size() == 0 + assert len(buf) == 0 + + _add_group(buf, weight=0) + assert buf.size() == 1 + assert len(buf) == 1 + + _add_group(buf, weight=0) + assert buf.size() == 2 + assert len(buf) == 2 + + _run(buf.remove([0], remove_in_dp=True)) + assert buf.size() == 1 + assert len(buf) == 1 + + +class MultiPartitionFakeDataPlaneClient(FakeDataPlaneClient): + """Fake DP client that tracks rows per partition (token-capture mode).""" + + def __init__(self) -> None: + super().__init__(partition_id="rollout_data") + self.rows_by_partition: dict[str, dict[str, Any]] = {} + self.clear_calls_by_partition: list[tuple[str, list[str]]] = [] + + def put_samples(self, sample_ids, partition_id, fields=None, tags=None): + bucket = self.rows_by_partition.setdefault(partition_id, {}) + for i, sid in enumerate(sample_ids): + bucket[sid] = {"tag": dict(tags[i]) if tags is not None else {}} + return KVBatchMeta( + partition_id=partition_id, + task_name=None, + sample_ids=list(sample_ids), + fields=None, + tags=[dict(t) for t in tags] if tags is not None else None, + ) + + def clear_samples(self, sample_ids, partition_id): + ids = list(sample_ids) if sample_ids is not None else [] + self.clear_calls_by_partition.append((partition_id, ids)) + bucket = self.rows_by_partition.setdefault(partition_id, {}) + for sid in ids: + bucket.pop(sid, None) + + +class TestTQReplayBufferTokenCaptureMode: + """commit_finalized / abort / rollout_ids / staging-aware remove. + + All of these are uncalled on the legacy (token_capture.enabled=false) + path; the existing test classes above are the legacy-invariance guard. + """ + + def _make_capture_buffer(self, dp) -> TQReplayBuffer: + return TQReplayBuffer( + dp, + partition_id="rollout_data", + pad_value_dict={"token_ids": 0}, + staging_partition_id="rollout_staging", + ) + + def test_reserve_records_rollout_ids(self): + buf = self._make_capture_buffer(MultiPartitionFakeDataPlaneClient()) + buf.reserve(weight_version=1, rollout_ids=["g0_g0", "g0_g1"]) + assert buf._rollout_ids_list == [["g0_g0", "g0_g1"]] + # Legacy reserve records None. + buf.reserve(weight_version=1) + assert buf._rollout_ids_list[1] is None + + def test_commit_finalized_fills_slot_with_group_min_wv(self): + dp = MultiPartitionFakeDataPlaneClient() + buf = self._make_capture_buffer(dp) + group_id = buf.reserve(weight_version=4, rollout_ids=["r0", "r1"]) + # The finalizer published its own rows; commit_finalized only fills the slot. + meta = KVBatchMeta( + partition_id="rollout_data", + task_name=None, + sample_ids=[f"{group_id}_g0", f"{group_id}_g1"], + fields=None, + ) + _run( + buf.commit_finalized( + group_id, + meta, + group_min_wv=3, + group_max_wv=5, + staging_keys=["r0/c1", "r0/c2", "r1/c1"], + ) + ) + assert buf.ready_list == [True] + assert buf.start_weight_list == [3] # oldest call version, not reserve-time 4 + assert buf.end_weight_list == [5] + assert buf.meta_list[0] is meta + assert buf._staging_keys_list == [["r0/c1", "r0/c2", "r1/c1"]] + # No tensorize/put happened here. + assert dp.rows_by_partition.get("rollout_data") is None + + def test_commit_finalized_raises_for_evicted_slot(self): + buf = self._make_capture_buffer(MultiPartitionFakeDataPlaneClient()) + meta = KVBatchMeta( + partition_id="rollout_data", task_name=None, sample_ids=[], fields=None + ) + with pytest.raises(ValueError, match="no live slot"): + _run(buf.commit_finalized("ghost", meta, group_min_wv=0, group_max_wv=0)) + + def test_abort_drops_unready_slot_only(self): + dp = MultiPartitionFakeDataPlaneClient() + buf = self._make_capture_buffer(dp) + gid_unready = buf.reserve(weight_version=1) + gid_ready = buf.reserve(weight_version=1) + _run( + buf.commit( + gid_ready, + _make_record(), + start_weight_version=1, + end_weight_version=1, + ) + ) + assert buf.abort(gid_unready) is True + assert buf.size() == 1 + # Ready slots and unknown ids are not abortable. + assert buf.abort(gid_ready) is False + assert buf.abort("ghost") is False + assert buf.size() == 1 + + def test_remove_clears_staging_rows_alongside_canonical(self): + dp = MultiPartitionFakeDataPlaneClient() + buf = self._make_capture_buffer(dp) + group_id = buf.reserve(weight_version=1, rollout_ids=["r0"]) + meta = KVBatchMeta( + partition_id="rollout_data", + task_name=None, + sample_ids=[f"{group_id}_g0"], + fields=None, + ) + _run( + buf.commit_finalized( + group_id, + meta, + group_min_wv=1, + group_max_wv=1, + staging_keys=["r0/c1", "r0/c2"], + ) + ) + n = _run(buf.remove([0], remove_in_dp=True)) + assert n == 1 + assert ("rollout_data", [f"{group_id}_g0"]) in dp.clear_calls_by_partition + assert ("rollout_staging", ["r0/c1", "r0/c2"]) in dp.clear_calls_by_partition + + def test_remove_without_staging_partition_skips_staging_clear(self): + dp = MultiPartitionFakeDataPlaneClient() + buf = TQReplayBuffer( + dp, partition_id="rollout_data", pad_value_dict={"token_ids": 0} + ) + group_id = buf.reserve(weight_version=1) + meta = KVBatchMeta( + partition_id="rollout_data", + task_name=None, + sample_ids=[f"{group_id}_g0"], + fields=None, + ) + _run(buf.commit_finalized(group_id, meta, group_min_wv=1, group_max_wv=1)) + _run(buf.remove([0], remove_in_dp=True)) + partitions_cleared = {p for p, _ in dp.clear_calls_by_partition} + assert partitions_cleared == {"rollout_data"} + + +class TestTQReplayBufferEvictedCommit: + def test_commit_on_evicted_slot_writes_nothing(self): + """The pre-write check: an evicted group must not orphan rows.""" + dp = FakeDataPlaneClient() + buf = _make_buffer(dp) + group_id = buf.reserve(weight_version=1) + _run(buf.remove([0], remove_in_dp=False)) + + with pytest.raises(ValueError, match="no live slot"): + _run( + buf.commit( + group_id, + _make_record(), + start_weight_version=1, + end_weight_version=1, + ) + ) + assert dp.put_calls == [] + assert dp.depth() == 0 + + def test_commit_evicted_during_write_unwrites_rows(self): + """Eviction interleaving with the awaited put must clear the rows.""" + + class EvictDuringPut(FakeDataPlaneClient): + def __init__(self): + super().__init__() + self.buf: TQReplayBuffer | None = None + + async def put_samples( + self, sample_ids, partition_id, fields=None, tags=None + ): + result = FakeDataPlaneClient.put_samples( + self, sample_ids, partition_id, fields=fields, tags=tags + ) + # Simulate the sampler evicting the slot mid-write. + await self.buf.remove([0], remove_in_dp=False) + return result + + dp = EvictDuringPut() + buf = _make_buffer(dp) + dp.buf = buf + group_id = buf.reserve(weight_version=1) + + with pytest.raises(ValueError, match="evicted during"): + _run( + buf.commit( + group_id, + _make_record(), + start_weight_version=1, + end_weight_version=1, + ) + ) + # The written rows were un-written. + assert dp.depth() == 0 + assert len(dp.clear_calls) == 1 diff --git a/uv.lock b/uv.lock index 2fd51395111..873b14452ef 100644 --- a/uv.lock +++ b/uv.lock @@ -207,7 +207,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -218,12 +218,15 @@ dependencies = [ { name = "propcache", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "yarl", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/86/b7c870053e36a94e8951b803cb5b909bfbc9b90ca941527f5fcafbf6b0fa/aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090", size = 1732476, upload-time = "2026-03-31T21:58:18.925Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d2/e2f77eef1acb7111405433c707dc735e63f67a56e176e72e9e7a2cd3f493/aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665", size = 1754113, upload-time = "2026-03-31T21:58:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/ec/a6/9b3e91eb8ae791cce4ee736da02211c85c6f835f1bdfac0594a8a3b7018c/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb", size = 1693205, upload-time = "2026-03-31T21:58:32.214Z" }, - { url = "https://files.pythonhosted.org/packages/01/a4/62f05a0a98d88af59d93b7fcac564e5f18f513cb7471696ac286db970d6a/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c", size = 1730356, upload-time = "2026-03-31T21:58:44.049Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, ] [[package]] @@ -238,6 +241,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/3b/40a68de458904bcc143622015fff2352b6461cd92fd66d3527bf1c6f5716/aiohttp_cors-0.8.1-py3-none-any.whl", hash = "sha256:3180cf304c5c712d626b9162b195b1db7ddf976a2a25172b35bb2448b890a80d", size = 25231, upload-time = "2025-03-31T14:16:18.478Z" }, ] +[[package]] +name = "aiohttp-retry" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/61/ebda4d8e3d8cfa1fd3db0fb428db2dd7461d5742cea35178277ad180b033/aiohttp_retry-2.9.1.tar.gz", hash = "sha256:8eb75e904ed4ee5c2ec242fefe85bf04240f685391c4879d8f541d6028ff01f1", size = 13608, upload-time = "2024-11-06T10:44:54.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl", hash = "sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54", size = 9981, upload-time = "2024-11-06T10:44:52.917Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -451,6 +466,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "bidict" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" }, +] + [[package]] name = "blake3" version = "1.0.8" @@ -491,6 +515,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/4d/1392562369b1139e741b30d624f09fe7091d17dd5579fae5732f044b12bb/blobfile-3.0.0-py3-none-any.whl", hash = "sha256:48ecc3307e622804bd8fe13bf6f40e6463c4439eba7a1f9ad49fd78aa63cc658", size = 75413, upload-time = "2024-08-27T00:02:51.518Z" }, ] +[[package]] +name = "boto3" +version = "1.43.57" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "jmespath", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "s3transfer", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/30/324319a914752e4021358ccf8252c04364eb8358f9d41d9c3f021959b363/boto3-1.43.57.tar.gz", hash = "sha256:549c95e45f9b04cf0c69727632dbdda23b84dcd3d0ed7981c6aaff72ef9cb5af", size = 112690, upload-time = "2026-07-27T19:31:09.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/a4/2c4ee25556a997a81ea01073962cf8351da3707412c584d053b3861d09e8/boto3-1.43.57-py3-none-any.whl", hash = "sha256:115ed9cac409d9b57e2437b52fdb4286660d12a2bd44a0f48d530e68623d09a7", size = 140028, upload-time = "2026-07-27T19:31:07.226Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.57" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "urllib3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/92/1f8a454cf90bcb0c0e32a25817441cb7f3702fdd76710d7018abad12a2fc/botocore-1.43.57.tar.gz", hash = "sha256:001a5653bebc03b862bde2da63bad4adb0b30072a3f247aba4daae5aa097b546", size = 15739550, upload-time = "2026-07-27T19:30:57.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/7a/1cfe9c296df0fa6e0143c8622b10f21fa8fb9cce25450e9a8e5cf6523e4b/botocore-1.43.57-py3-none-any.whl", hash = "sha256:319c6d79f66c3f3f2b538f7dcdc90e410a15a7bfced1db06479134947e629482", size = 15424433, upload-time = "2026-07-27T19:30:55.08Z" }, +] + [[package]] name = "build" version = "1.5.0" @@ -915,6 +967,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/99/00f3196036501b53032c4b1ab8337a0b978dee832ed276dae3815df4e8b5/datasets-4.8.5-py3-none-any.whl", hash = "sha256:5079900781719c0e063a8efdd2cd95a31ad0c63209178669cd23cf1b926149ff", size = 528973, upload-time = "2026-04-27T15:43:53.702Z" }, ] +[[package]] +name = "daytona" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiofiles", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "aiohttp", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona-analytics-api-client", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona-analytics-api-client-async", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona-api-client", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona-api-client-async", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona-toolbox-api-client", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona-toolbox-api-client-async", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "deprecated", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "httpx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "httpx-ws", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "obstore", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-api", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-instrumentation-aiohttp-client", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-sdk", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dotenv", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-multipart", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-socketio", extra = ["asyncio-client", "client"], marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "toml", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "urllib3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "wsproto", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/f4/ee8ab4cebd17a3c6033c656d5a7cb00d492865d74f888dc6603b3c9c26e9/daytona-0.201.0.tar.gz", hash = "sha256:62c0e05f7ffbacbc319151111bb60ad53f8882597449c88a73c7e5d4e4e4f569", size = 182380, upload-time = "2026-07-27T08:17:17.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5f/7d59dc8b8a6600bbfcbb588398565e0db1748320fb982f634bf70ba85fe9/daytona-0.201.0-py3-none-any.whl", hash = "sha256:4488cdf19ffdc0f1acdb5cbe6128349d64ec809e4bc12c6a6885802ca2303a90", size = 218674, upload-time = "2026-07-27T08:17:19.26Z" }, +] + +[[package]] +name = "daytona-analytics-api-client" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "urllib3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c0/fb1facfe6be00db733d74ba20e9a94d75442d14b9f5dbf975eae03fb9fad/daytona_analytics_api_client-0.201.0.tar.gz", hash = "sha256:43490fbae09877a8dd8559c3737bfa8cf9d4c83e41c051e53c55492c40fd106c", size = 30052, upload-time = "2026-07-27T08:16:57.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/4e/3b3eee66d9046f7b847bf60b49eba530ab1d3cfd91660986365e62d7e080/daytona_analytics_api_client-0.201.0-py3-none-any.whl", hash = "sha256:57ad08ae442cbe605bd020b42ee9131a9a35a22ab515bf44dc3a661141009459", size = 45012, upload-time = "2026-07-27T08:16:57.918Z" }, +] + +[[package]] +name = "daytona-analytics-api-client-async" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "aiohttp-retry", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/68/e26ed7feade3e0038594a6ec4c15cee7909aa4cee09f58c44659a5d95f9c/daytona_analytics_api_client_async-0.201.0.tar.gz", hash = "sha256:d3abe3ede6bce850c75bb38bfe4fb19e787428d6b25d783fd83c283ce1e7ae6d", size = 30075, upload-time = "2026-07-27T08:16:48.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/1f/589a6ca887acf1ffb3916b7b633efcff0aee98d920cb94857bc581f84ebb/daytona_analytics_api_client_async-0.201.0-py3-none-any.whl", hash = "sha256:2037229523115cc80a48cb82bb862d9a220b613c1189637fd2532d3039c7af6b", size = 45283, upload-time = "2026-07-27T08:16:49.425Z" }, +] + +[[package]] +name = "daytona-api-client" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "urllib3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/61/96162514ed811fa9c187e2698e18203d532ae495f5243006844a1e274cc2/daytona_api_client-0.201.0.tar.gz", hash = "sha256:0f19bc371982b8c82680ad8ae6983c33e2cfbe59babcaea575c7ce121af81623", size = 129148, upload-time = "2026-07-27T08:17:03.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/b1/9d860a122b4e540906b79841f65d1e69532db523cc6bfff850642787fd37/daytona_api_client-0.201.0-py3-none-any.whl", hash = "sha256:5f7b835488b60f873531e4f5b299fc566b2f342161866584f35070f2d2beaf97", size = 327241, upload-time = "2026-07-27T08:17:04.863Z" }, +] + +[[package]] +name = "daytona-api-client-async" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "aiohttp-retry", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/7d/dd3afb430077ef0cbb925f4b15fe757b3ac75dd436f4b7778c2a59e1a1b0/daytona_api_client_async-0.201.0.tar.gz", hash = "sha256:d269b2e26a418f3c922654806165309dbed408d45a6780b430d4e2efec4a1987", size = 129824, upload-time = "2026-07-27T08:16:49.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/3b/1ad3942c305d0946753f1cc51beff29b2f19ed01c075c4d5f2caa44906e1/daytona_api_client_async-0.201.0-py3-none-any.whl", hash = "sha256:376d01d9a3ce872bce085b4d2217519cf470d660e8bc4bd9755036d67f71f2ee", size = 329912, upload-time = "2026-07-27T08:16:50.189Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "urllib3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/a3/f3e4bcaef5201fd75ef45f0fa80ca86fda54c23660fc409935ef3b96af0c/daytona_toolbox_api_client-0.201.0.tar.gz", hash = "sha256:726d7b00dbd9b233dc0d07314bb354748ac90bfedcbec102fc360119cf73a275", size = 88034, upload-time = "2026-07-27T08:16:58.304Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a6/38ac9893f93459d7a3ec80aa0de3decc96922e2b1aca6e224d11deb10094/daytona_toolbox_api_client-0.201.0-py3-none-any.whl", hash = "sha256:02fa2086941abfabb6e096f2a16460ff104064c2749535fead794c40bee0f141", size = 252389, upload-time = "2026-07-27T08:16:59.557Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client-async" +version = "0.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "aiohttp-retry", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-dateutil", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/8d/4fe5bf1e42b2cf82077cb3ae1e4fd57a84e2b428af138538eabba9d3a8e2/daytona_toolbox_api_client_async-0.201.0.tar.gz", hash = "sha256:3fc8acec0cb301f3b041fa67f54d4544b2d23fb51a50ad39660d08d585dfeabc", size = 81939, upload-time = "2026-07-27T08:16:49.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/fc/7850319223ef4425ad069eea86946deaa4f57600fd6168ac991d38323a1b/daytona_toolbox_api_client_async-0.201.0-py3-none-any.whl", hash = "sha256:98459520781ee59d485e3418207cbdbe1220de29f5f49dd06deab18436e2797b", size = 250873, upload-time = "2026-07-27T08:16:50.892Z" }, +] + [[package]] name = "debugpy" version = "1.8.20" @@ -993,6 +1173,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + [[package]] name = "depyf" version = "0.20.0" @@ -1225,8 +1417,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "pydantic", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "0.52.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "starlette", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-inspection", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] @@ -1954,6 +2145,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx-ws" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "httpcore", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "httpx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "wsproto", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/cd/ca91a07ae446451f7476bf3fcc909e98cb942ff032ebfda0e3fe449aca7b/httpx_ws-0.9.0.tar.gz", hash = "sha256:797373326f70eec1ae96f6e43ae9f12002fd7d73aee139a4985eaab964338a08", size = 107105, upload-time = "2026-03-28T14:11:10.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/f8/a6bc80313a9e93c888fa10534dfce2ad76ff86911b6f485777ce6de6a073/httpx_ws-0.9.0-py3-none-any.whl", hash = "sha256:71640d2fb1bf9a225775015b33cd755cfd4c5f7e21c885192fe3adc4c387b248", size = 15759, upload-time = "2026-03-28T14:11:11.887Z" }, +] + [[package]] name = "huey" version = "2.6.0" @@ -2473,8 +2679,7 @@ dependencies = [ { name = "pyjwt", extra = ["crypto"], marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "python-multipart", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "sse-starlette", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "0.52.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "starlette", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-inspection", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "uvicorn", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -2793,8 +2998,7 @@ dependencies = [ { name = "pyyaml", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "requests", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "sqlparse", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "0.52.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "starlette", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "uvicorn", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] @@ -2832,7 +3036,7 @@ dependencies = [ { name = "jmespath", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "setuptools", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "supervisor", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/03/5a/d669bdeb5ba96db42c6ef010835a25119b05f8c35ee5f1c3f715626625fe/model_hosting_container_standards-0.1.15.tar.gz", hash = "sha256:ae8dd74d3250545c14f0a7068186c7b0f0ab6563d31e7137f556b6b660c8a6a9", size = 93994, upload-time = "2026-05-05T18:22:29.357Z" } @@ -3173,6 +3377,21 @@ dependencies = [ ] [package.optional-dependencies] +all = [ + { name = "boto3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "coverage", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "mypy", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opensandbox", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pre-commit", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pytest", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pytest-asyncio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pytest-cov", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pytest-xdist", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "requests-mock", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "ruff", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "tenacity", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] dev = [ { name = "coverage", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "mypy", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -3185,30 +3404,20 @@ dev = [ { name = "ruff", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] sandbox = [ + { name = "boto3", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "daytona", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "opensandbox", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "tenacity", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] -[package.dev-dependencies] -docs = [ - { name = "myst-parser", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "nvidia-sphinx-theme", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinx-autobuild", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinx-autodoc2", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinx-copybutton", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinx-design", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinx-reredirects", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "sphinxcontrib-mermaid", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "swagger-plugin-for-sphinx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, -] - [package.metadata] requires-dist = [ - { name = "aiohttp", specifier = ">=3.13.3" }, + { name = "aiohttp", specifier = ">=3.14.1" }, { name = "anthropic", specifier = "<=0.109.2" }, + { name = "boto3", marker = "extra == 'sandbox'", specifier = ">=1.34" }, { name = "coverage", extras = ["toml"], marker = "extra == 'dev'" }, { name = "datasets" }, + { name = "daytona", marker = "extra == 'sandbox'", specifier = ">=0.179.0" }, { name = "devtools" }, { name = "fastapi" }, { name = "fonttools", specifier = ">=4.60.2" }, @@ -3220,6 +3429,7 @@ requires-dist = [ { name = "mlflow", specifier = ">=3.14.0" }, { name = "mlflow-skinny", specifier = ">=3.14.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8.0" }, + { name = "nemo-gym", extras = ["dev", "sandbox"], marker = "extra == 'all'", editable = "3rdparty/Gym-workspace/Gym" }, { name = "omegaconf" }, { name = "openai", specifier = "<=2.7.2" }, { name = "opensandbox", marker = "extra == 'sandbox'", specifier = ">=0.1.9" }, @@ -3246,21 +3456,7 @@ requires-dist = [ { name = "wandb" }, { name = "yappi" }, ] -provides-extras = ["sandbox", "dev"] - -[package.metadata.requires-dev] -docs = [ - { name = "myst-parser", specifier = ">=4.0.1" }, - { name = "nvidia-sphinx-theme", specifier = ">=0.0.8" }, - { name = "sphinx", specifier = ">=8.2.3" }, - { name = "sphinx-autobuild", specifier = ">=2025.8.25" }, - { name = "sphinx-autodoc2", specifier = ">=0.5.0" }, - { name = "sphinx-copybutton", specifier = ">=0.5.2" }, - { name = "sphinx-design", specifier = ">=0.6.1" }, - { name = "sphinx-reredirects", specifier = ">=0.1.6" }, - { name = "sphinxcontrib-mermaid", specifier = ">=1.0.0" }, - { name = "swagger-plugin-for-sphinx", specifier = ">=6.0.0" }, -] +provides-extras = ["all", "sandbox", "dev"] [[package]] name = "nemo-rl" @@ -3980,6 +4176,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/56/c7e8645061cc2fc23f3a54f33e1e340df59216f07dcfb97d46b8ae7dd26c/nvtx-0.2.15-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3741edac4678b92f03d22a3f0a2dfd469f422f85e63db71b038e02525b2404ad", size = 788639, upload-time = "2026-03-18T10:12:01.69Z" }, ] +[[package]] +name = "obstore" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/2f/f83afaab7945509d72245b2b00af0b4834ce78fdd2d9ae9f0ad1a3036a91/obstore-0.11.0.tar.gz", hash = "sha256:a2f55163bcd348b4a60d12e6893eac50eddc742bad8032a1705d49140b992204", size = 130565, upload-time = "2026-06-25T18:29:49.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/59/3f47822683ee2b6db8685faa25829946d6343a561251ec2704548455d946/obstore-0.11.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2ff6d3ed553298828fb760b4aef6347fbcc7b5c5e3ce3f8381ce805c370021a", size = 5638724, upload-time = "2026-06-25T18:28:43.897Z" }, + { url = "https://files.pythonhosted.org/packages/23/50/1df335fdf9b527b3933f1e94ab6fc720ad314260fab8591cb0b6668ff192/obstore-0.11.0-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:39d04b324fcf984e7050734ebda77b81764025b0c011750201a0d8954087f7aa", size = 5413508, upload-time = "2026-06-25T18:28:45.624Z" }, + { url = "https://files.pythonhosted.org/packages/de/dc/a259aba149b841ca7c91fea177df9972a60a636b54077beed1a35b254994/obstore-0.11.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:37c0d15d775b1370ef5204ee3919a5ddf7e2592d11815213105f8db031f2ab8d", size = 5619995, upload-time = "2026-06-25T18:28:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/57/b7/577a965f440e9ea64243518663f9d16be7df8eafc7123818e8e841fa21ce/obstore-0.11.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9c8fd2a544e2e0b926669c47fcfb8d2314e234abc240ea165dae04ee42e1d7ac", size = 5869187, upload-time = "2026-06-25T18:28:53.166Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -4180,7 +4388,7 @@ name = "opentelemetry-exporter-otlp-proto-common" version = "1.41.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "opentelemetry-proto", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-proto", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/fa/f9e3bd3c4d692b3ce9a2880a167d1f79681a1bea11f00d5bf76adc03e6ea/opentelemetry_exporter_otlp_proto_common-1.41.1.tar.gz", hash = "sha256:0e253156ea9c36b0bd3d2440c5c9ba7dd1f3fb64ba7a08fc85fbac536b56e1fb", size = 20409, upload-time = "2026-04-24T13:15:40.924Z" } wheels = [ @@ -4210,13 +4418,13 @@ name = "opentelemetry-exporter-otlp-proto-http" version = "1.41.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "googleapis-common-protos", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "opentelemetry-api", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "opentelemetry-exporter-otlp-proto-common", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "opentelemetry-proto", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "opentelemetry-sdk", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "requests", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "typing-extensions", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "googleapis-common-protos", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-api", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-exporter-otlp-proto-common", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-proto", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-sdk", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "requests", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/5b/9d3c7f70cca10136ba82a81e738dee626c8e7fc61c6887ea9a58bf34c606/opentelemetry_exporter_otlp_proto_http-1.41.1.tar.gz", hash = "sha256:4747a9604c8550ab38c6fd6180e2fcb80de3267060bef2c306bad3cb443302bc", size = 24139, upload-time = "2026-04-24T13:15:42.977Z" } wheels = [ @@ -4237,6 +4445,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/d2/ee4002b88e20c59fae52bed008a63c6f7eff7d498f302032f6b0434a6de7/opentelemetry_exporter_prometheus-0.62b1-py3-none-any.whl", hash = "sha256:7a0b8a6402e107e1f93e38f074a668797e1103936b189561959531a67ffeba55", size = 13278, upload-time = "2026-04-24T13:15:22.485Z" }, ] +[[package]] +name = "opentelemetry-instrumentation" +version = "0.62b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-semantic-conventions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "packaging", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "wrapt", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/cb/0523b92c112a6cc70be43724343dc45225d3af134419844d7879a07755d4/opentelemetry_instrumentation-0.62b1.tar.gz", hash = "sha256:90e92a905ba4f84db06ac3aec96701df6c079b2d66e9379f8739f0a1bdcc7f45", size = 34043, upload-time = "2026-04-24T13:22:31.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/0f/45adbaea1f81b847cffdcee4f4b5f89297e42facf7fac78c7aaac4c38e75/opentelemetry_instrumentation-0.62b1-py3-none-any.whl", hash = "sha256:976fc6e640f2006599e97429c949e622c108d0c17c2059347d1e6c93c707f257", size = 34163, upload-time = "2026-04-24T13:21:31.722Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-aiohttp-client" +version = "0.62b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-instrumentation", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-semantic-conventions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "opentelemetry-util-http", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "wrapt", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/3b/3feb8c0ff2ce154a1633580269ba89f0f6a247a930761c6ef3419dc3a534/opentelemetry_instrumentation_aiohttp_client-0.62b1.tar.gz", hash = "sha256:602c52358fdf56841ded98025593298f93de5c98fc8ab7799f5282e227bd9487", size = 19313, upload-time = "2026-04-24T13:22:33.996Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/77/4518a02b266c9f97083c93b38865956cc99029e97ecad1e544bbabee9452/opentelemetry_instrumentation_aiohttp_client-0.62b1-py3-none-any.whl", hash = "sha256:e0fbf5489e4c5e08928de25ae9efcceb6f94c2a8e9f5e9529e3434a50aacfd01", size = 14533, upload-time = "2026-04-24T13:21:34.309Z" }, +] + [[package]] name = "opentelemetry-proto" version = "1.41.1" @@ -4289,6 +4528,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/22/41fb05f1dc5fda2c468e05a41814c20859016c85117b66c8a257cae814f6/opentelemetry_semantic_conventions_ai-0.5.1-py3-none-any.whl", hash = "sha256:25aeb22bd261543b4898a73824026d96770e5351209c7d07a0b1314762b1f6e4", size = 11250, upload-time = "2026-03-26T14:20:37.108Z" }, ] +[[package]] +name = "opentelemetry-util-http" +version = "0.62b1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/1b/aa71b63e18d30a8384036b9937f40f7618f8030a7aa213155fb54f6f2b47/opentelemetry_util_http-0.62b1.tar.gz", hash = "sha256:adf6facbb89aef8f8bc566e2f04624942ba08a7b678b3479a91051a8f4dc70a3", size = 11393, upload-time = "2026-04-24T13:23:12.994Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/a9d9d32161c1ced61346267db4c9702da54f81ec5dc88214bc65c23f4e9d/opentelemetry_util_http-0.62b1-py3-none-any.whl", hash = "sha256:c57e8a6c19fc422c288e6074e882f506f85030b69b7376182f74f9257b9261f0", size = 9295, upload-time = "2026-04-24T13:22:28.078Z" }, +] + [[package]] name = "orjson" version = "3.11.9" @@ -4538,7 +4786,7 @@ version = "8.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "prometheus-client", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1b/e9/2065686d1dfa62296fdc158b6e8fd25b0cb3dca09b0632cabeb5ae81fe4d/prometheus_fastapi_instrumentator-8.0.2.tar.gz", hash = "sha256:3c252e748151768a7aefd66824a04a870144f71de48a67aed211749a9ca2a548", size = 21342, upload-time = "2026-06-23T09:39:31.611Z" } wheels = [ @@ -5052,6 +5300,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-engineio" +version = "4.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "simple-websocket", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/a0/f75491f942184d9960b15e763270f765fe9f239745ca5f9e16289011aed4/python_engineio-4.13.3.tar.gz", hash = "sha256:572b7783e341fed21edbc7cea297ccd378dad79265fdde96aa4664420a7c06c9", size = 79734, upload-time = "2026-06-20T22:53:52.197Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/96/82f6328e410515fab21d5602ba35b9377a47b5a141a0c1f9efa00ce21eb4/python_engineio-4.13.3-py3-none-any.whl", hash = "sha256:1f60ecaf1358190f0e26c48c578a60428dc02a8f1295bc3dbf53d1b31116821f", size = 59993, upload-time = "2026-06-20T22:53:50.775Z" }, +] + [[package]] name = "python-json-logger" version = "4.1.0" @@ -5063,11 +5323,33 @@ wheels = [ [[package]] name = "python-multipart" -version = "0.0.28" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "python-socketio" +version = "5.16.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/54/a85eb421fbdd5007bc5af39d0f4ed9fa609e0fedbfdc2adcf0b34526870e/python_multipart-0.0.28.tar.gz", hash = "sha256:8550da197eac0f7ab748961fc9509b999fa2662ea25cef857f05249f6893c0f8", size = 45314, upload-time = "2026-05-10T11:05:16.596Z" } +dependencies = [ + { name = "bidict", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "python-engineio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/2d/ffce71017c106b75099fea569df6518c63fee5d6202ce0cfe7b01e6f22c3/python_socketio-5.16.3.tar.gz", hash = "sha256:89b136f677ae65607a84cecda9b4d6c5377b40a97582c504c25df89af16d520e", size = 128095, upload-time = "2026-06-15T22:07:04.003Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/a2/43bbc5860b5034e2af4ef99a0e04d726ff329c43e192ef3abaa8d7ecfce5/python_multipart-0.0.28-py3-none-any.whl", hash = "sha256:10faac07eb966c3f48dc415f9dee46c04cb10d58d30a35677db8027c825ed9b6", size = 29438, upload-time = "2026-05-10T11:05:15.052Z" }, + { url = "https://files.pythonhosted.org/packages/0a/38/8c5e72d53ff8eb27497c4f268a7f6d9121e727a50b65248288ad79a93053/python_socketio-5.16.3-py3-none-any.whl", hash = "sha256:e7ad14202a5e6448824c7c2f86161d04e13dec05992257df5c709e6a2798c041", size = 82087, upload-time = "2026-06-15T22:07:02.498Z" }, +] + +[package.optional-dependencies] +asyncio-client = [ + { name = "aiohttp", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +client = [ + { name = "requests", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "websocket-client", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] [[package]] @@ -5351,6 +5633,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/45/8e5fd559bea0d2f57c4e12bf197a2fade2fac465aa518284f157dfbca92b/ruff-0.9.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ab90a7944c5a1296f3ecb08d1cbf8c2da34c7e68114b1271a431a3ad30cb660e", size = 11327490, upload-time = "2025-02-28T10:16:27.654Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, +] + [[package]] name = "safetensors" version = "0.8.0" @@ -5586,6 +5880,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "simple-websocket" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wsproto", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, +] + [[package]] name = "simplejson" version = "4.1.1" @@ -5762,8 +6068,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "sphinx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "0.52.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "starlette", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "uvicorn", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "watchfiles", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "websockets", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5810,18 +6115,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/cf/45dd359f6ca0c3762ce0490f681da242f0530c49c81050c035c016bfdd3a/sphinx_design-0.7.0-py3-none-any.whl", hash = "sha256:f82bf179951d58f55dca78ab3706aeafa496b741a91b1911d371441127d64282", size = 2220350, upload-time = "2026-01-19T13:12:51.077Z" }, ] -[[package]] -name = "sphinx-reredirects" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1b/8d/0e39fe2740d7d71417edf9a6424aa80ca2c27c17fc21282cdc39f90d5a40/sphinx_reredirects-1.1.0.tar.gz", hash = "sha256:fb9b195335ab14b43f8273287d0c7eeb637ba6c56c66581c11b47202f6718b29", size = 614624, upload-time = "2025-12-22T08:28:02.792Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/81/b5dd07067f3daac6d23687ec737b2d593740671ebcd145830c8f92d381c5/sphinx_reredirects-1.1.0-py3-none-any.whl", hash = "sha256:4b5692273c72cd2d4d917f4c6f87d5919e4d6114a752d4be033f7f5f6310efd9", size = 6351, upload-time = "2025-12-22T08:27:59.724Z" }, -] - [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" @@ -5926,8 +6219,7 @@ version = "3.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "0.52.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "starlette", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "starlette", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f7/2b/58abc2d1fd397e7dde08e947e05c884d8ef2f78d5e2588c17a12d42d6994/sse_starlette-3.4.4.tar.gz", hash = "sha256:07e0fa0460138baf25cdd5fb28683472c3995dc1642225191b3832d62526bcb0", size = 31819, upload-time = "2026-05-12T17:37:17.019Z" } wheels = [ @@ -5948,32 +6240,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] -[[package]] -name = "starlette" -version = "0.52.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "platform_machine == 'x86_64' and sys_platform == 'linux'", - "platform_machine == 'aarch64' and sys_platform == 'linux'", -] -dependencies = [ - { name = "anyio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/68/79977123bb7be889ad680d79a40f339082c1978b5cfcf62c2d8d196873ac/starlette-0.52.1.tar.gz", hash = "sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933", size = 2653702, upload-time = "2026-01-18T13:34:11.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" }, -] - [[package]] name = "starlette" version = "1.3.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "platform_machine == 'x86_64' and sys_platform == 'linux'", - "platform_machine == 'aarch64' and sys_platform == 'linux'", -] dependencies = [ - { name = "anyio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-automodel') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-7-nemo-rl-mcore' and extra != 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'x86_64' and extra != 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-fsdp' and extra != 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "anyio", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ @@ -6270,6 +6542,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/49/7bae94729bfd7a3f331795251302f0b0c8e54a7ec25b3af5d5bfe133367c/tokenspeed_triton-3.7.10.post20260531-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b90ac41e7f15933797545ff1a9e803a9d8beb4ca9ba70f6d41a9e0fc26484f5c", size = 85888791, upload-time = "2026-05-31T01:29:25.584Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "torch" version = "2.11.0+cu130" @@ -6552,11 +6833,11 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] @@ -7041,6 +7322,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "16.0" @@ -7100,7 +7390,7 @@ name = "wsproto" version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "h11", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-fsdp') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-fsdp' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (sys_platform != 'linux' and extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } wheels = [