diff --git a/README.md b/README.md index 6165fef1..644818c0 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,14 @@ opinionated about supporting workloads that are: - `src/stitch/providers/modal.py`: Modal helpers for Volume commit/reload and Flash container discovery. - `cookbook/`: End-to-end examples. + - `local_disagg/`: minimal in-memory harness that exercises the sync + protocol with a fake engine — start here. - `slime_disagg/`: SLIME plus a stitch-managed Modal Flash/SGLang pool. + - `miles_disagg/`: the miles twin of `slime_disagg` (NVFP4 QAT on Blackwell). - `standalone_rollouts/`: standalone Modal/SGLang rollout provider with a hot-load API shim. The core package has no required dependencies; extras pull in what each -adapter needs (`modal`, `sglang`, `slime`). +adapter needs (`modal`, `sglang`). ## Adding adapters diff --git a/cookbook/bulletin_hooks.py b/cookbook/bulletin_hooks.py index 3493facc..e9ac4942 100644 --- a/cookbook/bulletin_hooks.py +++ b/cookbook/bulletin_hooks.py @@ -9,9 +9,9 @@ staleness weight version so unusable (too-stale) rollouts are never generated. Both hooks read their config off the trainer's ``args`` namespace (the trainer's -``--custom-config-path`` setattr's every key onto ``args``). The only -trainer-specific axis is the env-var fallback for the Flash app / class name; -callers pass those as ``app_name_env`` / ``cls_name_env``. +``--custom-config-path`` setattr's every key onto ``args``), with +``DELTA_APP_NAME`` / ``DELTA_SERVER_CLS_NAME`` env vars as the fallback for the +Flash app / class name. """ from __future__ import annotations @@ -34,14 +34,7 @@ # ── Publish hook ────────────────────────────────────────────────────────────── -def commit_and_wake( - args: Any, - version_dir: str, - rollout_engines: list[Any], - *, - app_name_env: str, - cls_name_env: str, -) -> None: +def commit_and_wake(args: Any, version_dir: str, rollout_engines: list[Any]) -> None: """Trainer ``custom_delta_pre_push_path`` hook (publish-only, bulletin board). The trainer has written ``weight_v{N}/`` to the Modal Volume. Advance the @@ -49,9 +42,6 @@ def commit_and_wake( ``reload`` sees the new version, then best-effort wake the Flash pool. The sidecars self-sync (wake RPC, periodic poll, startup), so a missed wake only costs latency. - - ``app_name_env`` / ``cls_name_env`` are the env-var names the trainer uses - for the Flash app and server class (e.g. ``"SLIME_DELTA_APP_NAME"``). """ del rollout_engines version = parse_weight_identity(Path(version_dir).name) @@ -81,10 +71,10 @@ def commit_and_wake( if version is None or rank not in (None, 0): return - _best_effort_wake(args, version, app_name_env=app_name_env, cls_name_env=cls_name_env) + _best_effort_wake(args, version) -def claim_pool(args: Any, *, app_name_env: str, cls_name_env: str) -> None: +def claim_pool(args: Any) -> None: """Trainer launch hook (rank 0): claim the rollout pool for this run. Write the empty pointer ``/weight_v000000``, commit the Volume, and @@ -100,17 +90,17 @@ def claim_pool(args: Any, *, app_name_env: str, cls_name_env: str) -> None: board = FilesystemBulletinBoard(_transport_root(args), layout="slime") board.claim(_run_id(args)) commit_volume(_volume_name(args)) - _best_effort_wake(args, BASE_VERSION, app_name_env=app_name_env, cls_name_env=cls_name_env) + _best_effort_wake(args, BASE_VERSION) -def _best_effort_wake(args: Any, version: int, *, app_name_env: str, cls_name_env: str) -> None: +def _best_effort_wake(args: Any, version: int) -> None: """Nudge warm Flash containers to reconcile now. Best-effort: a transient Modal control-plane error must not kill the training step — `latest` is already committed and sidecars self-sync on their next poll/startup.""" try: - app_name = getattr(args, "rollout_modal_flash_app_name", None) or os.environ[app_name_env] + app_name = getattr(args, "rollout_modal_flash_app_name", None) or os.environ["DELTA_APP_NAME"] cls_name = getattr(args, "rollout_modal_flash_server_cls_name", None) or os.getenv( - cls_name_env, "Server" + "DELTA_SERVER_CLS_NAME", "Server" ) wake_targets(discover_flash_targets(app_name=app_name, cls_name=cls_name), version) except Exception: # noqa: BLE001 diff --git a/cookbook/miles_disagg/README.md b/cookbook/miles_disagg/README.md index 7b512755..33a89840 100644 --- a/cookbook/miles_disagg/README.md +++ b/cookbook/miles_disagg/README.md @@ -28,13 +28,14 @@ NVFP4 QAT is native Megatron FP4 training (`NVFP4BlockScaling`, TransformerEngin rollout pool run on B200 — unlike the INT4 recipe, which fake-quantizes on H200. There is no simulated/non-Blackwell NVFP4 weight-QAT path. -`glm45_air_bf16_disagg` is the exception: it is a BF16 experiment on H200. It -does not use NVFP4 QAT, does not run `convert_hf_to_nvfp4.py`, and serves the -prepared BF16 Hugging Face checkpoint directly. +`glm45_air_bf16_disagg` is the exception: a BF16 experiment on H200 with no +NVFP4 anywhere (see the variant note below). ## Checkpoint lifecycle (three roles) -`prepare_checkpoints` builds them on a GPU (see `modal_train.py`): +`prepare_checkpoints` builds them on a GPU (see `modal_train.py`). The +`tools/convert_*.py` scripts it invokes live in the pinned miles fork, not in +this repo. 1. **BF16 masters** (`--ref-load`): the trainable parameters. Moonlight ships bf16 (masters = the download); Kimi K2.6 ships INT4, so masters are @@ -49,62 +50,6 @@ The trainer reads the NVFP4 base for both the export quant config and the diff baseline, so applying delta_vN reproduces export_vN byte-for-byte — the served weights become the trainer's NVFP4 export. -## GLM-4.5-Air BF16 on H200 - -Use `glm45_air_bf16_disagg` for `zai-org/GLM-4.5-Air`. - -This config has two prepared checkpoint paths: - -- `/prep/glm45-air-bf16/bf16`: the prepared Hugging Face BF16 checkpoint. This - is both the SGLang served base (`--hf-checkpoint`) and the disk-delta baseline. -- `/prep/glm45-air-bf16/torch_dist`: the Megatron raw-mode checkpoint loaded by - the trainer (`--ref-load`). - -The weight-sync loop is still the same disk-delta loop: miles exports BF16 HF -tensors after each update, XORs the new bytes against the previous bytes, writes -the delta to the Modal Volume bulletin board, and the stitch sidecar applies the -delta onto each rollout container's local BF16 checkpoint copy. - -The GLM path has two Modal-specific details: - -- `prepare_checkpoints` disables Xet and `hf_transfer`; the standard Hugging - Face downloader was the path that finished reliably for this large checkpoint. -- `prepare_torch_dist` uses a small wrapper around miles' - `convert_hf_to_torch_dist.py` so multi-node Modal Volume commits merge all - `iter_0000001` shard files instead of only rank 0's renamed `release` dir. - -Run from the repo root: - -```bash -alias m="uv run --extra modal modal" -export EXPERIMENT_CONFIG=glm45_air_bf16_disagg - -# Long-running one-time prep. Keep the rollout pool down until the served base -# exists, otherwise warm containers crash-loop on the missing model path. If you -# use --detach, wait for each prep job to finish before starting the next one. -POOL_MIN_CONTAINERS=0 m run --detach -m cookbook.miles_disagg.modal_train::prepare_checkpoints -POOL_MIN_CONTAINERS=0 m run --detach -m cookbook.miles_disagg.modal_train::prepare_torch_dist -POOL_MIN_CONTAINERS=0 m run -m cookbook.miles_disagg.modal_train::prepare_dataset - -# Deploy the H200 rollout pool and trainer app. -m deploy --strategy recreate -m cookbook.miles_disagg.modal_train - -# Verify SGLang serves the prepared BF16 base, then launch training. -m run -m cookbook.miles_disagg.modal_train::smoke_flash_pool -m run -m cookbook.miles_disagg.modal_train::launch_train - -# Optional: check a later synced weight version. -m run -m cookbook.miles_disagg.modal_train::smoke_flash_pool --weight-version 1 -``` - -Expected prepared outputs in the `miles-prep-checkpoints` Volume: - -```text -glm45-air-bf16/bf16/ -glm45-air-bf16/torch_dist/latest_checkpointed_iteration.txt -glm45-air-bf16/torch_dist/iter_0000001/ -``` - ## Run it You need a Modal account and a `huggingface-secret` Modal secret. Work from the @@ -133,23 +78,36 @@ m run -m cookbook.miles_disagg.modal_train::launch_train m run -m cookbook.miles_disagg.modal_train::smoke_flash_pool --weight-version 3 ``` -The full `kimi_k2_6_nvfp4_disagg` recipe is a 32×8 B200 trainer footprint that -exceeds the de-risk budget — run the Moonlight de-risk first to validate the -QAT → NVFP4-export → XOR-delta → SGLang-reload loop, then scale. +To start a run against the already-deployed app without a client-tied `m run` +(whose ephemeral app context can stop the deployed serving app), use plain +Python instead: `python -m cookbook.miles_disagg._spawn_into_deployed +`. + +The full `kimi_k2_6_nvfp4_disagg` recipe is a 32×8 B200 trainer footprint — run +the Moonlight de-risk first to validate the QAT → NVFP4-export → XOR-delta → +SGLang-reload loop, then scale. + +### GLM-4.5-Air BF16 variant + +`glm45_air_bf16_disagg` runs `zai-org/GLM-4.5-Air` as BF16 on H200; the +weight-sync loop is the same disk-delta loop, applied to BF16 HF tensors. It +prepares two checkpoints: `/prep/glm45-air-bf16/bf16` (served base + delta +baseline) and `/prep/glm45-air-bf16/torch_dist` (`--ref-load`, built by +`prepare_torch_dist` via the multi-node conversion wrapper in +`convert_hf_to_torch_dist_modal.py`). Run the same flow as above with +`EXPERIMENT_CONFIG=glm45_air_bf16_disagg`, plus +`m run --detach -m cookbook.miles_disagg.modal_train::prepare_torch_dist` +between the prep and deploy steps. Keep `POOL_MIN_CONTAINERS=0` during prep so +warm containers don't crash-loop on the missing model path, and let each prep +job finish before starting the next. ### Fork dependencies -The image pins the miles fork branch `nvfp4-disagg-fixes` (`MILES_REPO_REF`), -which carries the disaggregated-rollout features plus the publish-only / NVFP4 -fixes this cookbook needs: NVFP4 export dispatch -(`megatron_to_hf/processors/__init__.py`), the publish-only rollout semaphore -and HTTP client, the 0-dim NVFP4-scale delta encode, and the `encoding_dsv4` -import guard. Push that branch before deploying. - -The megatron routing-replay (R3) fix lives in `radixark/Megatron-LM` and is -**baked into the trainer image at build time** (a `.run_commands` step in -`modal_train.py`; source diff in `megatron_r3_num_out_tokens.patch`). The bake -is idempotent — it becomes a no-op once the fork itself ships the fix. +The image pins a miles fork commit (`MILES_REPO_REF` in `modal_train.py`) that +carries the disaggregated-rollout features plus the NVFP4/publish-only fixes +this cookbook needs; push the ref to `modal-projects/miles` before deploying. +The megatron routing-replay (R3) fix is baked into the trainer image at build +time (idempotent — a no-op once the fork ships it). Dev iteration: overlay a local miles checkout at deploy time (no rebuild, no push). This only overlays miles; the megatron R3 fix still comes from the bake. @@ -178,14 +136,13 @@ m app logs --since 4h --search "passrate " m app logs --since 4h --search "weight_v" ``` -## Bring-up checklist (flagged, validated by the Moonlight run) +## Bring-up checklist - The miles image's TransformerEngine is ≥ 2.7.0.dev0 and the trainer runs on Blackwell (NVFP4 BlockScaling). -- SGLang serves the prepared NVFP4 base on Blackwell (the `serving.py` fork is - proven for NVFP4) — verify on a warm container. +- SGLang serves the prepared NVFP4 base on Blackwell — verify on a warm + container. - The `convert_hf_to_nvfp4.py` quantization scope (which tensors get NVFP4 + exclude rules) matches the export processor's scope, so the XOR delta aligns. - A scope mismatch fails loud on the first delta apply (checksum/shape) — which - is exactly what the Moonlight run catches cheaply. + A scope mismatch fails loud on the first delta apply (checksum/shape). - `miles.utils.disk_delta` is import-light in the `--no-deps` serving image. diff --git a/cookbook/miles_disagg/configs/base.py b/cookbook/miles_disagg/configs/base.py index d19882f1..e1093895 100644 --- a/cookbook/miles_disagg/configs/base.py +++ b/cookbook/miles_disagg/configs/base.py @@ -58,7 +58,7 @@ class ModalConfig: # Flash autoscaler target: concurrent inputs (requests) per container before it # scales OUT. None = use sglang_server_concurrency (legacy). Set it well below the # SGLang engine concurrency so Flash adds containers instead of packing requests - # onto a few until their KV cache saturates (which 502'd / stalled the rollout). + # onto a few until their KV cache saturates and requests 502/stall. rollout_target_inputs: int | None = None proxy_regions: list[str] = ["us-west"] # Flash gateway proxy regions # Ephemeral disk (MiB) for the rollout Server. The sidecar materializes a diff --git a/cookbook/miles_disagg/configs/kimi_k25_2layer_nvfp4_disagg.py b/cookbook/miles_disagg/configs/kimi_k25_2layer_nvfp4_disagg.py index 62ad400b..4a84a24a 100644 --- a/cookbook/miles_disagg/configs/kimi_k25_2layer_nvfp4_disagg.py +++ b/cookbook/miles_disagg/configs/kimi_k25_2layer_nvfp4_disagg.py @@ -93,7 +93,7 @@ class _Miles(MilesConfig): async_mode = True update_weights_interval = 1 - # NVFP4 QAT — canonical recipe (radixark/miles#1261), same as K2.6. + # NVFP4 QAT — same canonical recipe as K2.6. fp4_format = "e2m1" fp4_recipe = "nvfp4" fp4_param_gather = False diff --git a/cookbook/miles_disagg/configs/kimi_k2_6_nvfp4_disagg.py b/cookbook/miles_disagg/configs/kimi_k2_6_nvfp4_disagg.py index 59ac9cc8..e47cfbeb 100644 --- a/cookbook/miles_disagg/configs/kimi_k2_6_nvfp4_disagg.py +++ b/cookbook/miles_disagg/configs/kimi_k2_6_nvfp4_disagg.py @@ -208,17 +208,17 @@ class _Miles(MilesConfig): async_mode = True update_weights_interval = 1 - # NVFP4 QAT — canonical miles recipe per radixark/miles#1261 (NVFP4 RL). + # NVFP4 QAT — miles' canonical NVFP4 RL recipe. fp4_format = "e2m1" fp4_recipe = "nvfp4" - # fp4_param_gather=False is canonical (#1261 never sets --fp4-param-gather): - # keeps NVFP4 GEMM compute QAT (config.fp4 in raw mode) with bf16 master params. - # With it True, params are TE NVFP4Tensor and Megatron DDP's param-buffer repoint - # (modify_underlying_storage -> TE replace_raw_data) crashes (TE: FP8 yes, NVFP4 no). + # fp4_param_gather=False keeps NVFP4 GEMM compute QAT (config.fp4 in raw + # mode) with bf16 master params. With it True, params are TE NVFP4Tensor and + # Megatron DDP's param-buffer repoint (modify_underlying_storage -> TE + # replace_raw_data) crashes (TE: FP8 yes, NVFP4 no). fp4_param_gather = False - # Per-module TE precision config (#1261's mechanism): NVFP4 ONLY on the routed - # expert GEMMs, everything else bf16 — matches the experts-only served base. - # Materialized to a temp YAML and passed as --te-precision-config-file. + # Per-module TE precision config: NVFP4 ONLY on the routed expert GEMMs, + # everything else bf16 — matches the experts-only served base. Materialized + # to a temp YAML and passed as --te-precision-config-file. te_precision_config_file = { "configs": { "nvfp4": { @@ -346,8 +346,8 @@ class _Miles(MilesConfig): "NCCL_NVLS_ENABLE": "1", "NVSHMEM_DISABLE_NCCL": "1", "NCCL_TIMEOUT_MS": "360000000", - # NVFP4 numerics (radixark/miles#1261 NVFP4 train env). Without these the - # NVFP4 QAT is mis-configured even once the build/DDP/load gaps are cleared. + # NVFP4 numerics: without these the NVFP4 QAT is mis-configured even + # once the build/DDP/load gaps are cleared. "NVTE_NVFP4_DISABLE_2D_QUANTIZATION": "1", "NVTE_NVFP4_DISABLE_RHT": "1", "NVTE_NVFP4_DISABLE_STOCHASTIC_ROUNDING": "1", diff --git a/cookbook/miles_disagg/helpers.py b/cookbook/miles_disagg/helpers.py index 48dfd667..86ba4be2 100644 --- a/cookbook/miles_disagg/helpers.py +++ b/cookbook/miles_disagg/helpers.py @@ -1,17 +1,18 @@ -"""Trainer-specific helpers for the miles_disagg example. - -Thin wrappers over the shared launch spine: Ray-cluster/sidecar/process helpers -come from :mod:`cookbook.ray_cluster` / :mod:`cookbook.sidecar_process`, and -config-prep / train-command / smoke-check / host-RAM monitor come from -:mod:`cookbook.trainer_helpers`. This module only supplies the miles-specific -axes: the sidecar module path, the config-field tuple to materialize, the -model-script attribute, and that the rollout pool scales from zero (wake on -demand). +"""Miles-specific wiring for the shared disagg launch spine. + +The constants below are the axes where miles differs from slime; everything +else is re-exported unchanged from the shared cookbook modules. This module +also owns the two helpers only miles needs: the node-local YAML materializer +(for ``te_precision_config_file``) and the host-RAM monitor. """ from __future__ import annotations +import os +import socket import subprocess +import threading +import time from typing import Any from cookbook.miles_disagg.configs.base import YAML_CONFIG_FIELDS @@ -26,46 +27,26 @@ training_nodes, ) from cookbook.sidecar_process import ( # noqa: F401 + start_sglang_sidecar as _start_sidecar, terminate_process, wait_http, ) from cookbook.trainer_helpers import ( # noqa: F401 VersionAheadError, build_train_cmd as _build_train_cmd, - materialize_node_local_yaml, prepare_config, smoke_flash_pool as _smoke_flash_pool, - start_host_mem_monitor, ) -SIDECAR_MODULE = "cookbook.miles_disagg.sidecar" - +SIDECAR_MODULE = "cookbook.miles_disagg.sidecar" # `python3 -m` entry on each rollout replica +MODEL_SCRIPT_ATTR = "miles_model_script" # config attr naming the sourced MODEL_ARGS script +WAKE_ON_DEMAND = True # scale-from-zero rollout pool (the smoke completion wakes it) -def start_sglang_sidecar( - *, - sidecar_port: int, - sglang_port: int, - bulletin_root: str, - local_checkpoint_dir: str, - base_checkpoint_dir: str, - volume_name: str, - commit_mode: str, - debug_requests: bool = False, -) -> subprocess.Popen: - from cookbook.sidecar_process import start_sglang_sidecar as _start - return _start( - sidecar_module=SIDECAR_MODULE, - sidecar_port=sidecar_port, - sglang_port=sglang_port, - bulletin_root=bulletin_root, - local_checkpoint_dir=local_checkpoint_dir, - base_checkpoint_dir=base_checkpoint_dir, - volume_name=volume_name, - commit_mode=commit_mode, - debug_requests=debug_requests, - ) +def start_sglang_sidecar(**kwargs: Any) -> subprocess.Popen: + """Launch the sidecar (`python3 -m` on this recipe's sidecar module).""" + return _start_sidecar(sidecar_module=SIDECAR_MODULE, **kwargs) def prepare_miles_config(miles_cfg: Any, tmpdir: str) -> None: @@ -80,26 +61,85 @@ def prepare_miles_config(miles_cfg: Any, tmpdir: str) -> None: def build_train_cmd(miles_cfg: Any, miles_root: str) -> str: """Build the training command, sourcing miles' model arch args if needed.""" - return _build_train_cmd(miles_cfg, miles_root, model_script_attr="miles_model_script") - - -def smoke_flash_pool( - *, - app_name: str, - cls_name: str, - model_name: str, - weight_version: int, - expect_min_containers: int, - timeout_seconds: int, -) -> None: - """Smoke the scale-from-zero miles rollout pool (min_containers=0): the - completion wakes it, then each warmed container is confirmed at the version.""" - _smoke_flash_pool( - app_name=app_name, - cls_name=cls_name, - model_name=model_name, - weight_version=weight_version, - expect_min_containers=expect_min_containers, - timeout_seconds=timeout_seconds, - wake_on_demand=True, - ) + return _build_train_cmd(miles_cfg, miles_root, model_script_attr=MODEL_SCRIPT_ATTR) + + +def smoke_flash_pool(**kwargs: Any) -> None: + """Smoke the scale-from-zero miles rollout pool: the completion wakes it, + then each warmed container is confirmed at the version.""" + _smoke_flash_pool(wake_on_demand=WAKE_ON_DEMAND, **kwargs) + + +def materialize_node_local_yaml(cfg: Any, field: str, dest_dir: str = "/root/.miles_node_yaml") -> None: + """Materialize a per-actor-read YAML config to a deterministic node-local path. + + Some config files (notably ``te_precision_config_file``, which + ``load_quantization_recipe`` re-reads on every Ray actor during model build) + are read independently on each trainer node — not just parsed once on the head. + ``prepare_config`` writes them under ``tempfile.mkdtemp()`` on the head only, + so on a multi-node cluster the other containers can't see that path. + + Call this on EVERY node (SPMD train()), before the rank-0 gate: each node + writes identical content (from the shared payload) to the same fixed path, so + the path the head embeds in the args resolves locally on all actors. No volume + commit/reload race — Ray actors are long-lived and wouldn't see post-start + volume writes anyway. + """ + import yaml + + if isinstance(val := getattr(cfg, field, None), dict): + os.makedirs(dest_dir, exist_ok=True) + path = os.path.join(dest_dir, f"{field}.yaml") + with open(path, "w") as f: + yaml.dump(val, f) + setattr(cfg, field, path) + + +def start_host_mem_monitor(interval_s: int = 20) -> None: + """Log this node's host-RAM trajectory to stdout from a daemon thread. + + The trainer can OOM-kill on host-RAM exhaustion (the publish/update_weights + full-model gather is the peak consumer), but Megatron only reports GPU memory + and the kill leaves no durable peak behind. This logs MemTotal/MemAvailable + + the container cgroup usage every ``interval_s`` so a live ``modal app logs -f`` + shows exactly which phase blows a big node and how high it peaks. Runs on EVERY + node (called from the SPMD enter()), so whichever rank OOMs has its own trace. + Best-effort: never raises.""" + host = socket.gethostname() + + def _meminfo() -> tuple[float, float]: + total = avail = 0.0 + try: + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemTotal:"): + total = int(line.split()[1]) / 1024 / 1024 # GiB + elif line.startswith("MemAvailable:"): + avail = int(line.split()[1]) / 1024 / 1024 + except Exception: # noqa: BLE001 + pass + return total, avail + + def _cgroup_used_gib() -> float: + for path in ("/sys/fs/cgroup/memory.current", # cgroup v2 + "/sys/fs/cgroup/memory/memory.usage_in_bytes"): # v1 + try: + with open(path) as f: + return int(f.read().strip()) / 1024**3 + except Exception: # noqa: BLE001 + continue + return -1.0 + + def _loop() -> None: + while True: + total, avail = _meminfo() + used = total - avail + cg = _cgroup_used_gib() + print( + f"[hostmem] {host} used={used:.0f}GiB avail={avail:.0f}GiB " + f"total={total:.0f}GiB cgroup_used={cg:.0f}GiB", + flush=True, + ) + time.sleep(interval_s) + + threading.Thread(target=_loop, daemon=True, name="host-mem-monitor").start() diff --git a/cookbook/miles_disagg/hooks.py b/cookbook/miles_disagg/hooks.py index 9f75160c..a586c6ab 100644 --- a/cookbook/miles_disagg/hooks.py +++ b/cookbook/miles_disagg/hooks.py @@ -1,39 +1,13 @@ -"""Modal publish + rollout-gating hooks for the miles_disagg example. +"""Hook entry points for the miles_disagg example. -Thin wrappers around :mod:`cookbook.bulletin_hooks` with the miles-specific -env-var fallbacks for the Flash app / server class name. +This module exists because the experiment configs reference these symbols by +dotted string (``custom_delta_pre_push_path = "cookbook.miles_disagg.hooks. +commit_and_wake"``, ``custom_rollout_request_hook_path = "...gated_rollout_ +request_hook"``), resolved inside the trainer process. The implementations +live in :mod:`cookbook.bulletin_hooks`. """ -from __future__ import annotations +from cookbook.bulletin_hooks import claim_pool, commit_and_wake, gated_rollout_request_hook -from typing import Any -from cookbook.bulletin_hooks import ( - claim_pool as _claim_pool, - commit_and_wake as _commit_and_wake, - gated_rollout_request_hook, -) - - -_APP_NAME_ENV = "MILES_DELTA_APP_NAME" -_CLS_NAME_ENV = "MILES_DELTA_SERVER_CLS_NAME" - - -def claim_pool(args: Any) -> None: - """Claim the rollout pool for this run at launch (resets every replica to base).""" - _claim_pool(args, app_name_env=_APP_NAME_ENV, cls_name_env=_CLS_NAME_ENV) - - -def commit_and_wake(args: Any, version_dir: str, rollout_engines: list[Any]) -> None: - """miles ``custom_delta_pre_push_path`` hook (publish-only, bulletin board).""" - _commit_and_wake( - args, - version_dir, - rollout_engines, - app_name_env=_APP_NAME_ENV, - cls_name_env=_CLS_NAME_ENV, - ) - - -# Re-export for the trainer's custom_rollout_request_hook_path. __all__ = ["claim_pool", "commit_and_wake", "gated_rollout_request_hook"] diff --git a/cookbook/miles_disagg/megatron_r3_num_out_tokens.patch b/cookbook/miles_disagg/megatron_r3_num_out_tokens.patch deleted file mode 100644 index 407134f6..00000000 --- a/cookbook/miles_disagg/megatron_r3_num_out_tokens.patch +++ /dev/null @@ -1,43 +0,0 @@ -megatron/core/transformer/moe/token_dispatcher.py --- Python -521 self.num_out_tokens = n 521 self.num_out_tokens = n -... um_local_tokens_per_expert.sum() ... um_local_tokens_per_expert.sum() -522 self._maybe_update_cuda 522 self._maybe_update_cuda -... _sync_point("before_permutation_1") ... _sync_point("before_permutation_1") -523 else: 523 else: -524 # For dropless training 524 # For dropless training -... , output size is static (num_tokens ... , output size is normally num_token -... * topk) ... s * topk. -525 # No explicit sync need 525 # Under routing replay -... ed ... (R3), the replayed routing_map can -... ... have fewer -... 526 # than topk experts per -... ... token (duplicate / -1 entries coll -... ... apse in the -... 527 # boolean map), so num_ -... ... tokens*topk overcounts and disagree -... ... s with -... 528 # input_splits (= routi -... ... ng_map True-count), causing "Split -... ... sizes doesn't -... 529 # match total dim 0 siz -... ... e" in the EP all-to-all. Derive it -... ... from the -... 530 # actual routing_map to -... ... stay consistent (equals num_tokens -... ... *topk in the -... 531 # normal dense case; ne -... ... eds the GPU->CPU sync like the drop -... ... ping path). -526 self.num_out_tokens = r 532 self.num_out_tokens = n -... outing_map.size(0) * self.config.mo ... um_local_tokens_per_expert.sum() -... e_router_topk ... -... 533 self._maybe_update_cuda -... ... _sync_point("before_permutation_1") -527 if self.ep_size > 1 or self 534 if self.ep_size > 1 or self -... .tp_size > 1: ... .tp_size > 1: -528 # ===================== 535 # ===================== -... ============================== ... ============================== -529 # Calculate input_split 536 # Calculate input_split -... s, output_splits for alltoall/allga ... s, output_splits for alltoall/allga -... ther in variable size. ... ther in variable size. - diff --git a/cookbook/miles_disagg/modal_train.py b/cookbook/miles_disagg/modal_train.py index 76857f47..ac492187 100644 --- a/cookbook/miles_disagg/modal_train.py +++ b/cookbook/miles_disagg/modal_train.py @@ -71,15 +71,11 @@ # only in this source tree; miles' own launcher puts it on PYTHONPATH, and so # must we (we run train_async.py directly rather than via execute_train). MEGATRON_PATH = "/root/Megatron-LM" -# Fork branch with the disaggregated-rollout features (opaque HTTP endpoint, -# publish-only disk-delta, request hook) AND the NVFP4 / publish-only fixes -# (NVFP4 export dispatch, publish-only semaphore + http client, 0-dim delta -# encode, encoding_dsv4 import guard). See branch nvfp4-disagg-fixes. +# Fork commit with the disaggregated-rollout features (opaque HTTP endpoint, +# publish-only disk-delta, request hook) plus the NVFP4 fixes this cookbook +# needs. Pin to an exact commit, not the branch tip (cached image layer); push +# the ref to modal-projects/miles before deploying. MILES_REPO_URL = "https://github.com/modal-projects/miles.git" -# nvfp4-disagg-v2 @ 9e98062af = merge(feat/disaggregated-rollout @121035147, -# radixark/miles#1261 NVFP4 RL @f95c2c495) + our publish-only/0-dim/encoding_dsv4 -# fixes + KimiK25 mbridge import converter (miles_plugins/mbridge/kimi.py). PUSH -# this branch to modal-projects/miles before deploying. (Built in /tmp/miles-merge.) MILES_REPO_REF = "e9ad52dbbe09b6113b4fa4dccfb5ace35341540e" # Build-time bake of the megatron R3 dispatch fix (see the .run_commands call @@ -155,7 +151,6 @@ # crash with "Split sizes doesn't match total dim 0 size". Derive it from the # actual per-expert counts instead (identical to size*topk in the dense # non-replay case). Idempotent: a no-op once the fork itself ships the fix. - # Source diff: cookbook/miles_disagg/megatron_r3_num_out_tokens.patch. .run_commands(f"python3 -c {shlex.quote(_R3_DISPATCH_BAKE_PY)}") .pip_install( "fastapi", # stitch sidecar (rollout pool reuses this image only if no serving image) @@ -218,9 +213,9 @@ def _select_server_image() -> modal.Image: if builder is None: return image return builder( - miles_repo_url=MILES_REPO_URL, - miles_repo_ref=MILES_REPO_REF, - miles_root=MILES_ROOT, + trainer_repo_url=MILES_REPO_URL, + trainer_repo_ref=MILES_REPO_REF, + trainer_root=MILES_ROOT, hf_cache_path=str(HF_CACHE_PATH), experiment=EXPERIMENT, ) @@ -582,7 +577,7 @@ def _build_bf16(out: str) -> None: print(f"Prepared masters={bf16_dir} served_base={bf16_dir}") return - # 2. served NVFP4 base (TE-direct quantizer per radixark/miles#1261). bf16 + # 2. served NVFP4 base (miles' TE-direct quantizer). bf16 # carve-outs for the dense first / last layers must match the trainer's # --num-layers-at-start/end-in-bf16 so the served base == the export layout. _nvfp4_carveouts = [] diff --git a/cookbook/miles_disagg/serving.py b/cookbook/miles_disagg/serving.py index 3e7a43c2..98c8b27e 100644 --- a/cookbook/miles_disagg/serving.py +++ b/cookbook/miles_disagg/serving.py @@ -4,18 +4,14 @@ twin of cookbook/slime_disagg/serving.py. The image itself is trainer-agnostic (see that module): NVFP4 vs INT4 is driven by the served checkpoint's own quant config, not by this builder. This wrapper pins miles as the ``--no-deps`` decoder -package, does a full clone, and clears the SGLang kernel cache as the final -filesystem step (modal_train mounts a kernel-cache volume at /root/.cache/sglang, -which can't mount over a non-empty path). - -NOTE: this assumes ``miles.utils.disk_delta`` is import-light (no heavy package -__init__ chain); verify on a warm container during bring-up. +package, does a full clone (the miles ref is not a branch tip), and clears the +SGLang kernel cache as the final filesystem step (modal_train mounts a +kernel-cache volume at /root/.cache/sglang, which can't mount over a non-empty +path). """ from __future__ import annotations -from pathlib import Path - import modal from cookbook.serving import build_b200_serving_image @@ -23,17 +19,16 @@ def build_nvfp4_b200_serving_image( *, - miles_repo_url: str, - miles_repo_ref: str, - miles_root: str, + trainer_repo_url: str, + trainer_repo_ref: str, + trainer_root: str, hf_cache_path: str, experiment: str, ) -> modal.Image: return build_b200_serving_image( - trainer_repo_url=miles_repo_url, - trainer_repo_ref=miles_repo_ref, - trainer_root=miles_root, - cookbook_dir=Path(__file__).parent, + trainer_repo_url=trainer_repo_url, + trainer_repo_ref=trainer_repo_ref, + trainer_root=trainer_root, hf_cache_path=hf_cache_path, experiment=experiment, shallow_clone=False, diff --git a/cookbook/serving.py b/cookbook/serving.py index 42b622c3..c0b147bc 100644 --- a/cookbook/serving.py +++ b/cookbook/serving.py @@ -49,7 +49,6 @@ def build_b200_serving_image( trainer_repo_url: str, trainer_repo_ref: str, trainer_root: str, - cookbook_dir: Path, hf_cache_path: str, experiment: str, shallow_clone: bool = True, @@ -59,13 +58,13 @@ def build_b200_serving_image( ``trainer_repo_url`` / ``trainer_repo_ref`` / ``trainer_root`` pin the ``--no-deps`` trainer checkout (so the pool's ``disk_delta`` matches the - trainer's encoder). ``cookbook_dir`` is the per-trainer cookbook package dir; - its parent (the whole ``cookbook`` package) is mounted at ``/root/cookbook``, - so the sidecar subprocess can import both ``cookbook..sidecar`` and the - shared ``cookbook.sidecar`` spine it delegates to. Mounting the package (not - just the subdir) is required because the sidecar is launched as - ``python3 -m cookbook..sidecar`` and is never imported at deploy time, - so Modal's import-time automounting never sees the shared module. + trainer's encoder). The whole ``cookbook`` package is mounted at + ``/root/cookbook``, so the sidecar subprocess can import both + ``cookbook..sidecar`` and the shared ``cookbook.sidecar`` spine it + delegates to. Mounting the package (not just the per-trainer subdir) is + required because the sidecar is launched as ``python3 -m + cookbook..sidecar`` and is never imported at deploy time, so Modal's + import-time automounting never sees the shared module. ``shallow_clone`` does a ``--depth 1`` clone+fetch (fine when the ref is a branch tip or recent commit). ``clear_sglang_cache_at_end`` removes @@ -134,7 +133,7 @@ def build_b200_serving_image( # shared cookbook.sidecar spine it is a thin adapter over. .add_local_python_source("stitch") .add_local_dir( - cookbook_dir.parent, + Path(__file__).parent, remote_path="/root/cookbook", ignore=["**/__pycache__"], ) diff --git a/cookbook/sidecar.py b/cookbook/sidecar.py index 72cda04f..702e4fcc 100644 --- a/cookbook/sidecar.py +++ b/cookbook/sidecar.py @@ -49,7 +49,7 @@ def parallel_init_local_checkpoint(disk_delta_module: str, workers: int = 32) -> def _base_fingerprint(base_dir: str) -> str: """Identity of the base's bytes: (filename, size, mtime_ns) over every shard. Re-prep rewrites the shards (new mtime/size), so this changes even when the tensor index/shapes - don't — which is exactly the case that bit K2.6 (NVFP4 values changed, names didn't).""" + don't (e.g. requantized values under unchanged names).""" h = hashlib.sha256() for e in sorted(os.scandir(base_dir), key=lambda e: e.name): if e.is_file(): diff --git a/cookbook/slime_disagg/README.md b/cookbook/slime_disagg/README.md index fbd6a4ed..0e217c1b 100644 --- a/cookbook/slime_disagg/README.md +++ b/cookbook/slime_disagg/README.md @@ -51,12 +51,19 @@ m run -m cookbook.slime_disagg.modal_train::launch_train m run -m cookbook.slime_disagg.modal_train::smoke_flash_pool --weight-version 3 ``` -To train again, just run `launch_train` again. Each launch gets a fresh run id -and writes its delta chain under its own `/` partition, so sequential -runs never collide — no bulletin-board reset between runs. The warm `Trainer` -cluster (Ray started once per container in `@modal.enter()`) goes straight to -training, and the rollout pool re-materializes to the new run's base on its own -when it sees the pointer move to the new ``. +To train again, just run `launch_train` again. Each launch gets a fresh +`run_id`: the trainer writes that run's delta chain under +`/delta-bulletin//weight_v{N}/` and a single `latest` pointer names the +active snapshot (`/weight_v{N}`), so sequential runs never collide — no +bulletin-board reset between runs. Sidecars apply versions in order; when the +pointer moves to a new run they re-materialize the base and replay that run's +chain. The warm `Trainer` cluster (Ray started once per container in +`@modal.enter()`) goes straight to training. + +Sidecars default to `quiesce` commit mode, which drains in-flight requests +before applying a delta. The `in_place` mode (set `SIDECAR_COMMIT_MODE` in the +config module) applies without draining and relies on version-namespaced KV +keys. ## Configuration @@ -86,18 +93,3 @@ searches (the app name is the config's `APP_NAME`): m app logs --since 4h --search "passrate " m app logs --since 4h --search "Published sparse delta" ``` - -## Protocol notes - -Each launch gets a fresh `run_id`. The trainer writes that run's delta chain -under `/delta-bulletin//weight_v{N}/`, and a single `/delta-bulletin/latest` -pointer names the active snapshot (`/weight_v{N}`). Sidecars apply -versions in order from their current version; when the pointer moves to a new -run they re-materialize the base and replay that run's chain from the start. -Each run is isolated under its own `run_id`, so sequential runs never collide. -Bounding the per-run replay with periodic recovery anchors is left for later. - -Sidecars default to `quiesce` commit mode, which drains in-flight requests -before applying a delta. The `in_place` mode (set `SIDECAR_COMMIT_MODE` in -the config module) applies without draining and relies on version-namespaced -KV keys. diff --git a/cookbook/slime_disagg/configs/kimi_k2_6_int4_disagg.py b/cookbook/slime_disagg/configs/kimi_k2_6_int4_disagg.py index e2ea7a4a..c9ce1c65 100644 --- a/cookbook/slime_disagg/configs/kimi_k2_6_int4_disagg.py +++ b/cookbook/slime_disagg/configs/kimi_k2_6_int4_disagg.py @@ -35,19 +35,15 @@ Deploy as its own app: EXPERIMENT_CONFIG=kimi_k2_6_int4_disagg m deploy --strategy recreate -m cookbook.slime_disagg.modal_train -Prerequisites the bring-up depends on (flagged, not yet automated): - 1. hf_checkpoint must resolve to a native compressed-tensors INT4 (W4A16) Kimi - K2.6 checkpoint whose group_size equals INT4_GROUP_SIZE below. Verify the - repo id and its config.json (quant_method == "compressed-tensors"). If no - native-INT4 K2.6 exists yet, fall back by setting hf_checkpoint to - moonshotai/Kimi-K2-Thinking: it is a drop-in swap here — native W4A16 at - group_size 32 (matches INT4_GROUP_SIZE) with the same kimi-k2-thinking.sh - arch and a published slime int4 recipe. - 2. kimi-k2-thinking.sh must describe K2.6's arch (rope scaling, norm eps). If - K2.6 diverges, add scripts/models/kimi-k2.6.sh to the slime fork and point - slime_model_script at it. - 3. The serving image's SGLang fork must serve native-INT4 MLA MoE on Blackwell - (proven for NVFP4; verify on a warm container — see serving.py). +Before running: + 1. hf_checkpoint must resolve to a native compressed-tensors INT4 (W4A16) + checkpoint whose group_size equals INT4_GROUP_SIZE below + (moonshotai/Kimi-K2-Thinking is a drop-in swap if no native-INT4 K2.6 is + published: same arch script, W4A16 at group_size 32). + 2. kimi-k2-thinking.sh must describe K2.6's arch (rope scaling, norm eps); if + K2.6 diverges, add a kimi-k2.6.sh model script to the slime fork. + 3. The serving image's SGLang must serve native-INT4 MLA MoE on Blackwell + (verify on a warm container — see serving.py). The smaller `moonlight_int4_disagg` config is the same machinery at a size that fits a couple of GPUs — run it first to de-risk the INT4-QAT/disk-delta loop. diff --git a/cookbook/slime_disagg/configs/moonlight_disagg.py b/cookbook/slime_disagg/configs/moonlight_disagg.py index fa863ab7..e1a65232 100644 --- a/cookbook/slime_disagg/configs/moonlight_disagg.py +++ b/cookbook/slime_disagg/configs/moonlight_disagg.py @@ -1,10 +1,10 @@ -"""Moonlight-16B-A3B GRPO on Modal Flash, disaggregated (M1: synchronous bring-up). +"""Moonlight-16B-A3B GRPO on Modal Flash, disaggregated. Moonlight-16B-A3B is Moonshot's small DeepSeek-V3-architecture MoE -- MLA plus DeepSeek-MoE (sigmoid router, shared experts, grouped top-k) -- i.e. the Kimi -K2.6 family at a size that fits a single H200 rollout container. This config is -the synchronous bring-up rung; routing replay and async-first layer on top (see -the M2/M3 notes at the bottom). +K2.6 family at a size that fits a single H200 rollout container. This is the +cheap rung of the K2.6 ladder: async-first with routing replay, staleness +gating, and in_place commits, at bring-up scale. Reshape provenance rule (colocated `run-moonlight-16B-A3B.sh` -> disagg config): ARCHITECTURE -> sourced from scripts/models/moonlight.sh via `slime_model_script` @@ -15,15 +15,6 @@ Deploy as its own app: EXPERIMENT_CONFIG=moonlight_disagg m deploy --strategy recreate -m cookbook.slime_disagg.modal_train - -Prerequisites the bring-up depends on (flagged, not yet automated): - 1. The training image's slime fork ref (modal_train.SLIME_REPO_REF) must include - scripts/models/moonlight.sh, the deepseekv3 megatron_to_hf export, and the - routing-replay code. The base image's SGLang (used by the rollout pool) is - independent. - 2. Bridge-mode HF load is UNVERIFIED for Moonlight/DeepSeek-V3 (it is proven for - Qwen here). If load fails, the fallback is the proven torch_dist path - (tools/convert_hf_to_torch_dist.py + ref_load pointed at the converted dir). """ from __future__ import annotations @@ -35,7 +26,7 @@ DELTA_VOLUME_NAME = "slime-delta-bulletin-moonlight" DELTA_BULLETIN_ROOT = "/delta-bulletin" -# M3: in_place commit applies weights without draining in-flight rollouts. Stale +# in_place commit applies weights without draining in-flight rollouts. Stale # KV is isolated per weight version by the sidecar's extra_key stamping (so old # requests keep decoding on their version's KV and it drains as they finish); # min-version pins cross commits freely, only exact pins are quiesced. @@ -49,10 +40,10 @@ SGLANG_SERVER_ARGS = { "--context-length": "8192", "--mem-fraction-static": "0.85", - # M2 routing replay: the pool must emit per-token routed experts. slime + # Routing replay: the pool must emit per-token routed experts. slime # launches no engine in publish-only mode, so this is set here (not by # sglang_engine.py). num_layers/moe_router_topk come from moonlight.sh, so - # the rollout's [tokens, 27, 6] reshape matches the served model (M0-verified). + # the rollout's [tokens, 27, 6] reshape matches the served model. "--enable-return-routed-experts": "", } @@ -64,8 +55,7 @@ class _Slime(SlimeConfig): # (the script carries MLA + the full DeepSeek-MoE arg set). slime_model_script = "scripts/models/moonlight.sh" - # Model + checkpoint. Bridge-mode HF load, same as the dense disagg example - # (see prerequisite 2 in the module docstring -- this is the main M1 unknown). + # Model + checkpoint. Bridge-mode HF load, same as the dense disagg example. hf_checkpoint = "moonshotai/Moonlight-16B-A3B-Instruct" ref_load = hf_checkpoint megatron_to_hf_mode = "bridge" @@ -78,7 +68,7 @@ class _Slime(SlimeConfig): rollout_num_gpus = 0 rollout_num_gpus_per_engine = 1 # 1xH200 per rollout container (MLA -> cheap KV) rollout_endpoint_url = None - # M3 staleness gate: each rollout request is pinned to + # Staleness gate: each rollout request is pinned to # min_required_version = latest_published - lag (derived out-of-band from the # bulletin `latest`, since the per-request hook gets no rollout_id). A replica # more than `lag` versions behind 409s -> retried, so no too-stale rollouts. @@ -89,7 +79,7 @@ class _Slime(SlimeConfig): rollout_request_retry_sleep = 1.0 rollout_session_affinity_header = "Modal-Session-ID" - # M3 async-first: one-step off-policy (train_async pipelines generate(N+1) with + # Async-first: one-step off-policy (train_async pipelines generate(N+1) with # train(N)); publish weights every step. async_mode = True update_weights_interval = 1 @@ -165,7 +155,7 @@ class _Slime(SlimeConfig): kl_loss_type = "low_var_kl" entropy_coef = 0.0 - # Routing replay (M2): replay the rollout engine's per-token expert routing + # Routing replay: replay the rollout engine's per-token expert routing # during the training forward/backward to cut train-inference divergence # (R3, arxiv 2510.11370). Auto-implies use_routing_replay; needs the pool's # --enable-return-routed-experts (in SGLANG_SERVER_ARGS above). @@ -187,12 +177,3 @@ def prepare_data(self) -> None: slime = _Slime() - -# Milestone progression (kept here so it's legible): -# M1 (sync MoE bring-up): DONE. -# M2 (routing replay): DONE — use_rollout_routing_replay=True + -# --enable-return-routed-experts; num_layers(27)/moe_router_topk(6) from moonlight.sh. -# M3 (async-first): DONE above — async_mode=True (train_async one-step off-policy), -# SIDECAR_COMMIT_MODE="in_place" (in-flight updates + extra_key KV namespacing), -# min-version gate at latest-lag via gated_rollout_request_hook. Tune -# rollout_request_weight_version_lag for the staleness/throughput trade-off. diff --git a/cookbook/slime_disagg/helpers.py b/cookbook/slime_disagg/helpers.py index 031afe41..df9ad802 100644 --- a/cookbook/slime_disagg/helpers.py +++ b/cookbook/slime_disagg/helpers.py @@ -1,11 +1,7 @@ -"""Trainer-specific helpers for the slime_disagg example. +"""Slime-specific wiring for the shared disagg launch spine. -Thin wrappers over the shared launch spine: Ray-cluster/sidecar/process helpers -come from :mod:`cookbook.ray_cluster` / :mod:`cookbook.sidecar_process`, and -config-prep / train-command / smoke-check come from :mod:`cookbook.trainer_helpers`. -This module only supplies the slime-specific axes: the sidecar module path, the -config-field tuple to materialize, the model-script attribute, and that the -rollout pool runs with a warm floor (not scale-from-zero). +The constants below are the axes where slime differs from miles; everything +else is re-exported unchanged from the shared cookbook modules. """ from __future__ import annotations @@ -25,6 +21,7 @@ training_nodes, ) from cookbook.sidecar_process import ( # noqa: F401 + start_sglang_sidecar as _start_sidecar, terminate_process, wait_http, ) @@ -36,33 +33,14 @@ ) -SIDECAR_MODULE = "cookbook.slime_disagg.sidecar" +SIDECAR_MODULE = "cookbook.slime_disagg.sidecar" # `python3 -m` entry on each rollout replica +MODEL_SCRIPT_ATTR = "slime_model_script" # config attr naming the sourced MODEL_ARGS script +WAKE_ON_DEMAND = False # warm-floor rollout pool (min_containers > 0) -def start_sglang_sidecar( - *, - sidecar_port: int, - sglang_port: int, - bulletin_root: str, - local_checkpoint_dir: str, - base_checkpoint_dir: str, - volume_name: str, - commit_mode: str, - debug_requests: bool = False, -) -> subprocess.Popen: - from cookbook.sidecar_process import start_sglang_sidecar as _start - - return _start( - sidecar_module=SIDECAR_MODULE, - sidecar_port=sidecar_port, - sglang_port=sglang_port, - bulletin_root=bulletin_root, - local_checkpoint_dir=local_checkpoint_dir, - base_checkpoint_dir=base_checkpoint_dir, - volume_name=volume_name, - commit_mode=commit_mode, - debug_requests=debug_requests, - ) +def start_sglang_sidecar(**kwargs: Any) -> subprocess.Popen: + """Launch the sidecar (`python3 -m` on this recipe's sidecar module).""" + return _start_sidecar(sidecar_module=SIDECAR_MODULE, **kwargs) def prepare_slime_config(slime_cfg: Any, tmpdir: str) -> None: @@ -72,25 +50,9 @@ def prepare_slime_config(slime_cfg: Any, tmpdir: str) -> None: def build_train_cmd(slime_cfg: Any, slime_root: str) -> str: """Build the training command, sourcing slime's model arch args if needed.""" - return _build_train_cmd(slime_cfg, slime_root, model_script_attr="slime_model_script") + return _build_train_cmd(slime_cfg, slime_root, model_script_attr=MODEL_SCRIPT_ATTR) -def smoke_flash_pool( - *, - app_name: str, - cls_name: str, - model_name: str, - weight_version: int, - expect_min_containers: int, - timeout_seconds: int, -) -> None: - """Smoke the warm-floor slime rollout pool (min_containers > 0).""" - _smoke_flash_pool( - app_name=app_name, - cls_name=cls_name, - model_name=model_name, - weight_version=weight_version, - expect_min_containers=expect_min_containers, - timeout_seconds=timeout_seconds, - wake_on_demand=False, - ) +def smoke_flash_pool(**kwargs: Any) -> None: + """Smoke the warm-floor slime rollout pool.""" + _smoke_flash_pool(wake_on_demand=WAKE_ON_DEMAND, **kwargs) diff --git a/cookbook/slime_disagg/hooks.py b/cookbook/slime_disagg/hooks.py index d4b1814d..96b84394 100644 --- a/cookbook/slime_disagg/hooks.py +++ b/cookbook/slime_disagg/hooks.py @@ -1,39 +1,13 @@ -"""Modal publish + rollout-gating hooks for the slime_disagg example. +"""Hook entry points for the slime_disagg example. -Thin wrappers around :mod:`cookbook.bulletin_hooks` with the slime-specific -env-var fallbacks for the Flash app / server class name. +This module exists because the experiment configs reference these symbols by +dotted string (``custom_delta_pre_push_path = "cookbook.slime_disagg.hooks. +commit_and_wake"``, ``custom_rollout_request_hook_path = "...gated_rollout_ +request_hook"``), resolved inside the trainer process. The implementations +live in :mod:`cookbook.bulletin_hooks`. """ -from __future__ import annotations +from cookbook.bulletin_hooks import claim_pool, commit_and_wake, gated_rollout_request_hook -from typing import Any -from cookbook.bulletin_hooks import ( - claim_pool as _claim_pool, - commit_and_wake as _commit_and_wake, - gated_rollout_request_hook, -) - - -_APP_NAME_ENV = "SLIME_DELTA_APP_NAME" -_CLS_NAME_ENV = "SLIME_DELTA_SERVER_CLS_NAME" - - -def claim_pool(args: Any) -> None: - """Claim the rollout pool for this run at launch (resets every replica to base).""" - _claim_pool(args, app_name_env=_APP_NAME_ENV, cls_name_env=_CLS_NAME_ENV) - - -def commit_and_wake(args: Any, version_dir: str, rollout_engines: list[Any]) -> None: - """SLIME ``custom_delta_pre_push_path`` hook (publish-only, bulletin board).""" - _commit_and_wake( - args, - version_dir, - rollout_engines, - app_name_env=_APP_NAME_ENV, - cls_name_env=_CLS_NAME_ENV, - ) - - -# Re-export for the trainer's custom_rollout_request_hook_path. __all__ = ["claim_pool", "commit_and_wake", "gated_rollout_request_hook"] diff --git a/cookbook/slime_disagg/modal_train.py b/cookbook/slime_disagg/modal_train.py index da5f95ca..e98579cf 100644 --- a/cookbook/slime_disagg/modal_train.py +++ b/cookbook/slime_disagg/modal_train.py @@ -62,10 +62,8 @@ SLIME_REPO_URL = "https://github.com/modal-projects/slime.git" # Pin to an exact commit, not the branch tip: the build's `git fetch ... && # checkout` is a cached image layer, so a moving branch tip silently leaves the -# container on a stale slime. This is PR #5 head (disaggregated-rollout): disk- -# delta publish-only + rollout_endpoint_url + custom_rollout_request_hook_path. -# Bump this SHA to roll slime forward. -SLIME_REPO_REF = "ebfe153949b1a69c39e92f947ed5d475166dd724" # incl. deepseekv3 router-dtype export fix + per-request rollout hook +# container on a stale slime. Bump this SHA to roll slime forward. +SLIME_REPO_REF = "ebfe153949b1a69c39e92f947ed5d475166dd724" image = ( modal.Image.from_registry(SLIME_IMAGE_TAG) @@ -149,9 +147,9 @@ def _select_server_image() -> modal.Image: if builder is None: return image return builder( - slime_repo_url=SLIME_REPO_URL, - slime_repo_ref=SLIME_REPO_REF, - slime_root=SLIME_ROOT, + trainer_repo_url=SLIME_REPO_URL, + trainer_repo_ref=SLIME_REPO_REF, + trainer_root=SLIME_ROOT, hf_cache_path=str(HF_CACHE_PATH), experiment=EXPERIMENT, ) diff --git a/cookbook/slime_disagg/serving.py b/cookbook/slime_disagg/serving.py index d1956e46..d3d03b29 100644 --- a/cookbook/slime_disagg/serving.py +++ b/cookbook/slime_disagg/serving.py @@ -9,8 +9,6 @@ from __future__ import annotations -from pathlib import Path - import modal from cookbook.serving import build_b200_serving_image @@ -18,17 +16,16 @@ def build_int4_b200_serving_image( *, - slime_repo_url: str, - slime_repo_ref: str, - slime_root: str, + trainer_repo_url: str, + trainer_repo_ref: str, + trainer_root: str, hf_cache_path: str, experiment: str, ) -> modal.Image: return build_b200_serving_image( - trainer_repo_url=slime_repo_url, - trainer_repo_ref=slime_repo_ref, - trainer_root=slime_root, - cookbook_dir=Path(__file__).parent, + trainer_repo_url=trainer_repo_url, + trainer_repo_ref=trainer_repo_ref, + trainer_root=trainer_root, hf_cache_path=hf_cache_path, experiment=experiment, shallow_clone=True, diff --git a/cookbook/standalone_rollouts/README.md b/cookbook/standalone_rollouts/README.md index c805a6b3..3dc25366 100644 --- a/cookbook/standalone_rollouts/README.md +++ b/cookbook/standalone_rollouts/README.md @@ -35,29 +35,21 @@ transport is the source of truth, and the elastic pool reconciles to it by pull. ## Compatibility Notes -The provider targets slime's `disk-delta-weight-sync` branch + PR #5. Each -version is a canonical HF/SafeTensors directory `weight_v{N}/` with a -`model.safetensors.index.json`; the engine applies deltas **host-side** (slime -`disk_delta`) onto a local full checkpoint and then reloads through the ordinary -`update_weights_from_disk` path — there is no engine-side `load_format="delta"` -receiver. The delta format is XOR (or `overwrite`) encoding, zstd compression, -and xxh3-128 (or blake3/adler32) per-tensor checksums; the version's -`index.json` carries `delta_encoding`/`compression_format`/`checksum_format`. - -A customer-produced delta (XOR + adler32 + zstd) is directly applicable by this -applier. The only adapter work is metadata *location*: the customer sends -`compression_format`/`checksum_format`/`previous_snapshot_identity` in the POST -body and ships a weight-map-only `index.json`, whereas the applier reads those -from the index's `metadata` block — so a customer-facing front door normalizes -the POST metadata into the index before advancing `latest`. - -Session affinity is delegated to Modal's Flash gateway. External clients send -the neutral `x-session-affinity` header to the **front door** (the advertised -provider URL, an `App.server` in front of the pool); the front door rewrites it to -`Modal-Session-ID` *before* the gateway, which then consistently routes related -requests to the same replica. The rewrite must happen pre-gateway, so it lives -in the front door rather than the per-container sidecar. Requests without the -header are routed normally. +The provider targets the pinned modal-projects/slime fork commit (see +`modal_serve.py`). Each version is a canonical HF/SafeTensors directory +`weight_v{N}/` with a `model.safetensors.index.json`. Deltas are applied +**host-side** (slime `disk_delta`: XOR or overwrite encoding, zstd compression, +per-tensor checksums — all declared in the index's `metadata` block) onto a +local full checkpoint, then reloaded through the ordinary +`update_weights_from_disk` path; there is no engine-side delta receiver. The +front door normalizes hot-load POST metadata into the index before advancing +`latest`, so a customer-produced delta is directly applicable. + +Session affinity is delegated to Modal's Flash gateway: external clients send +the neutral `x-session-affinity` header to the front door, which rewrites it to +`Modal-Session-ID` *before* the gateway so related requests route to the same +replica. The rewrite must happen pre-gateway, so it lives in the front door +rather than the per-container sidecar. ## Deploy the Provider @@ -200,19 +192,7 @@ SLIME publish-only mode: it launches no rollout engines, routes `/generate` to the provider front door, and writes each `weight_v{N}/` to a local disk dir. Staleness is gated by a **publish-hook readiness barrier**, not a per-request -pin (this is the key difference from `slime_disagg`, which uses a min-version -request gate). The `custom_delta_pre_push_path` hook (`announce_and_wait`) copies -the new version to the S3 transport, POSTs the customer hot-load API, and blocks -until the front door reports **every live replica** ready on the new version -(`readiness_threshold` 1.0) — so the next rollouts always run on current weights. -The request hook therefore leaves the version gate off -(`api_shim_rollout_request_weight_version_mode="none"`) and only carries auth -headers, retries, and session affinity. - -Routing replay requires the pool to return per-token routed experts: the provider -config sets `--enable-return-routed-experts` and the trainer sets -`use_rollout_routing_replay=True` (the layer count / router top-k come from the -sourced `scripts/models/moonlight.sh`). - -The first weight update only seeds SLIME's baseline snapshot and publishes -nothing; subsequent updates publish `weight_v000001`, `weight_v000002`, … . +pin (the key difference from `slime_disagg`, which uses a min-version request +gate): the `announce_and_wait` publish hook copies each new version to the S3 +transport, POSTs the hot-load API, and blocks until the front door reports every +live replica ready — so the next rollouts always run on current weights. diff --git a/cookbook/standalone_rollouts/modal_serve.py b/cookbook/standalone_rollouts/modal_serve.py index f06a963a..f0f19e3b 100644 --- a/cookbook/standalone_rollouts/modal_serve.py +++ b/cookbook/standalone_rollouts/modal_serve.py @@ -57,11 +57,11 @@ SLIME_IMAGE_TAG = "slimerl/slime:nightly-dev-20260527a" SLIME_ROOT = "/root/slime" SLIME_REPO_URL = "https://github.com/modal-projects/slime.git" -# PR #5 head (disaggregated-rollout, stacked on disk-delta-weight-sync). The -# provider sidecar applies disk deltas host-side via slime.utils.disk_delta, so -# the image must carry that branch's slime plus its checksum/compression deps. -# Pin a SHA, not the branch tip: the clone is a cached image layer. -SLIME_REPO_REF = "ebfe153949b1a69c39e92f947ed5d475166dd724" # incl. deepseekv3 router-dtype export fix + per-request rollout hook +# The provider sidecar applies disk deltas host-side via slime.utils.disk_delta, +# so the image must carry the fork's slime plus its checksum/compression deps. +# Pin a SHA, not the branch tip: the clone is a cached image layer (see +# cookbook/slime_disagg/modal_train.py). +SLIME_REPO_REF = "ebfe153949b1a69c39e92f947ed5d475166dd724" image = ( modal.Image.from_registry(SLIME_IMAGE_TAG) diff --git a/cookbook/standalone_rollouts/slime/configs/moonlight_slime_trainer.py b/cookbook/standalone_rollouts/slime/configs/moonlight_slime_trainer.py index 1148d6e8..f9dfeec1 100644 --- a/cookbook/standalone_rollouts/slime/configs/moonlight_slime_trainer.py +++ b/cookbook/standalone_rollouts/slime/configs/moonlight_slime_trainer.py @@ -62,12 +62,10 @@ class _Slime(SlimeConfig): use_rollout_routing_replay = True # Staleness is gated by the announce_and_wait readiness BARRIER (the publish - # hook below), so the request hook leaves the per-request version pin OFF and - # only carries auth headers, retries, and affinity. + # hook below); the request hook carries auth headers, retries, and affinity. custom_rollout_request_hook_path = ( "cookbook.standalone_rollouts.slime.hooks.rollout_request_weight_version_hook" ) - api_shim_rollout_request_weight_version_mode = "none" api_shim_rollout_request_retry_attempts = 240 api_shim_rollout_request_retry_sleep = 1.0 diff --git a/cookbook/standalone_rollouts/slime/hooks.py b/cookbook/standalone_rollouts/slime/hooks.py index f6b484a8..d36df7ab 100644 --- a/cookbook/standalone_rollouts/slime/hooks.py +++ b/cookbook/standalone_rollouts/slime/hooks.py @@ -15,7 +15,6 @@ import json import logging -import os import shutil import time import urllib.error @@ -235,43 +234,13 @@ def announce_claim(args: Any, *, run_id: str) -> None: def rollout_request_weight_version_hook( args: Any, sample: Any, request: dict[str, Any] ) -> None: - """SLIME ``custom_rollout_request_hook_path`` hook. + """SLIME ``custom_rollout_request_hook_path`` hook: retry budget, provider + auth headers, and session affinity for one rollout request. - The publish hook has already decided which versions are usable by polling the - provider pool. This request hook turns that trainer decision into provider - admission control, so requests routed to lagging replicas fail with a - retryable 409 instead of producing unusable samples. + Version admission needs no per-request pin here: announce_and_wait blocks + the next rollout until the pool serves the target, and a lagging replica + rejects with a retryable 409. """ - - mode = str( - getattr( - args, - "api_shim_rollout_request_weight_version_mode", - os.environ.get("STITCH_SHIM_ROLLOUT_REQUEST_WEIGHT_VERSION_MODE", "exact"), - ) - ) - # PR #5's apply_rollout_request_hook builds request={url,payload,headers, - # max_retries,retry_sleep} with NO rollout_id — that is per-rollout context, - # not per-request (sample.index is the batch position, not the trainer step). - # So the step-based version pin only runs if a caller supplies a rollout_id. - # announce_and_wait already blocks the next rollout until the pool serves the - # target, so skipping the pin is safe; it is belt-and-suspenders admission - # control for lagging replicas. TODO: re-derive a per-request target under - # the PR #5 contract (e.g. the latest published version) if pinning is wanted. - rollout_id = request.get("rollout_id") - if mode != "none" and rollout_id is not None: - target_version = _rollout_request_target_version( - args, int(rollout_id), bool(request.get("evaluation", False)) - ) - if mode == "exact": - request["payload"]["weight_version"] = {"exact_version": target_version} - elif mode == "min": - request["payload"]["weight_version"] = {"min_required_version": target_version} - else: - raise ValueError( - f"Unsupported api_shim_rollout_request_weight_version_mode: {mode!r}" - ) - # Generous retries on every rollout request so a cold/scaling/lagging replica # (a transient 503, or a 409 weight-version reject) is retried rather than # failing the rollout — the point of the elastic-pool readiness model. This @@ -316,20 +285,6 @@ def rollout_request_weight_version_hook( apply_session_affinity(request, getattr(sample, "session_id", None), affinity_header) -def _rollout_request_target_version(args: Any, rollout_id: int, evaluation: bool) -> int: - lag = int( - _setting( - args, - "api_shim_rollout_request_version_lag", - "STITCH_SHIM_ROLLOUT_REQUEST_VERSION_LAG", - default="0", - ) - ) - if evaluation: - lag = 0 - return max(0, int(rollout_id) - lag) - - def wait_until_ready(cfg: ShimConfig, identity: str) -> RolloutPoolState: # The pool reports current_snapshot_identity run-scoped (`/weight_vN`), # so match against the same composite — otherwise a replica still serving a diff --git a/cookbook/standalone_rollouts/slime/hooks_test.py b/cookbook/standalone_rollouts/slime/hooks_test.py index e15841ba..801e3741 100644 --- a/cookbook/standalone_rollouts/slime/hooks_test.py +++ b/cookbook/standalone_rollouts/slime/hooks_test.py @@ -130,29 +130,19 @@ def boom(*a, **k) -> None: class RolloutRequestHookTest(unittest.TestCase): - def test_skips_pin_without_rollout_id_but_sets_affinity(self) -> None: - # PR #5's request carries no rollout_id: the hook must not crash, must - # skip the version pin, and still apply session affinity. - args = Namespace( - api_shim_rollout_request_weight_version_mode="exact", - rollout_endpoint_url="http://provider", - ) + def test_sets_retries_and_affinity(self) -> None: + args = Namespace(rollout_endpoint_url="http://provider") sample = Namespace(session_id="grp-1") request = {"url": "u", "payload": {}, "headers": None, "max_retries": 60, "retry_sleep": 1.0} with mock.patch.dict("os.environ", {}, clear=True): hooks.rollout_request_weight_version_hook(args, sample, request) - self.assertNotIn("weight_version", request["payload"]) self.assertEqual(request["headers"]["x-session-affinity"], "grp-1") - # Retries are applied even without a version pin (cold/scaling pool). self.assertEqual(request["max_retries"], 60) def test_attaches_auth_headers(self) -> None: # The front door enforces auth on inference too, so every rollout request # must carry the provider auth headers. - args = Namespace( - api_shim_rollout_request_weight_version_mode="none", - rollout_endpoint_url="http://provider", - ) + args = Namespace(rollout_endpoint_url="http://provider") sample = Namespace(session_id=None) request = {"payload": {}, "headers": None} env = { @@ -166,18 +156,6 @@ def test_attaches_auth_headers(self) -> None: self.assertEqual(request["headers"]["Provider-Model"], "moonlight") self.assertEqual(request["headers"]["Provider-Deployment"], "prod") - def test_pins_exact_when_rollout_id_supplied(self) -> None: - args = Namespace( - api_shim_rollout_request_weight_version_mode="exact", - api_shim_rollout_request_version_lag=0, - rollout_endpoint_url="http://provider", - ) - sample = Namespace(session_id=None) - request = {"payload": {}, "headers": None, "rollout_id": 3} - with mock.patch.dict("os.environ", {}, clear=True): - hooks.rollout_request_weight_version_hook(args, sample, request) - self.assertEqual(request["payload"]["weight_version"], {"exact_version": 3}) - if __name__ == "__main__": unittest.main() diff --git a/cookbook/standalone_rollouts/slime/modal_train.py b/cookbook/standalone_rollouts/slime/modal_train.py index 7e977444..f231125d 100644 --- a/cookbook/standalone_rollouts/slime/modal_train.py +++ b/cookbook/standalone_rollouts/slime/modal_train.py @@ -49,10 +49,8 @@ SLIME_ROOT = "/root/slime" SLIME_REPO_URL = "https://github.com/modal-projects/slime.git" # Pin to an exact commit (see cookbook/slime_disagg/modal_train.py): the cached -# clone layer otherwise leaves the container on a stale slime. This is PR #5 -# head (disaggregated-rollout): disk-delta publish-only + rollout_endpoint_url + -# custom_rollout_request_hook_path. -SLIME_REPO_REF = "ebfe153949b1a69c39e92f947ed5d475166dd724" # incl. deepseekv3 router-dtype export fix + per-request rollout hook +# clone layer otherwise leaves the container on a stale slime. +SLIME_REPO_REF = "ebfe153949b1a69c39e92f947ed5d475166dd724" trainer_image = ( modal.Image.from_registry(SLIME_IMAGE_TAG) @@ -202,8 +200,8 @@ def train(self, experiment: str, payload: dict) -> None: ) # Claim the pool for this fresh run before any delta: reset `latest` to # base via the front door so replicas reconcile to base up front instead - # of inferring the reset from the first publish (the explicit-claim model - # PR #5 established for the bulletin path, now mirrored here). + # of inferring the reset from the first publish (mirrors the bulletin + # path's explicit-claim model). hooks.announce_claim(cfg, run_id=run_id) helpers.prepare_slime_config(cfg, tempfile.mkdtemp()) cmd = helpers.build_train_cmd(cfg, SLIME_ROOT) diff --git a/cookbook/trainer_helpers.py b/cookbook/trainer_helpers.py index 68468f18..ff772f04 100644 --- a/cookbook/trainer_helpers.py +++ b/cookbook/trainer_helpers.py @@ -2,10 +2,10 @@ slime and miles drive the same launch spine: resolve HF repo ids + materialize inline YAML configs, build the ``train.py`` command (optionally sourcing a model -arch script), monitor host RAM, and smoke the deployed Flash rollout pool. The -per-trainer ``helpers.py`` modules are thin wrappers that supply the only axes -that actually differ: which config-field tuple to materialize, the model-script -attribute name, and whether the rollout pool has a warm floor or scales from zero. +arch script), and smoke the deployed Flash rollout pool. The per-trainer +``helpers.py`` modules are thin wrappers that supply the only axes that actually +differ: which config-field tuple to materialize, the model-script attribute +name, and whether the rollout pool has a warm floor or scales from zero. """ from __future__ import annotations @@ -13,8 +13,6 @@ import json import os import shlex -import socket -import threading import time import urllib.error import urllib.request @@ -49,31 +47,6 @@ def prepare_config(cfg: Any, tmpdir: str, yaml_config_fields: Iterable[str]) -> setattr(cfg, field, path) -def materialize_node_local_yaml(cfg: Any, field: str, dest_dir: str = "/root/.miles_node_yaml") -> None: - """Materialize a per-actor-read YAML config to a deterministic node-local path. - - Some config files (notably miles' ``te_precision_config_file``, which - ``load_quantization_recipe`` re-reads on every Ray actor during model build) - are read independently on each trainer node — not just parsed once on the head. - ``prepare_config`` writes them under ``tempfile.mkdtemp()`` on the head only, - so on a multi-node cluster the other containers can't see that path. - - Call this on EVERY node (SPMD train()), before the rank-0 gate: each node - writes identical content (from the shared payload) to the same fixed path, so - the path the head embeds in the args resolves locally on all actors. No volume - commit/reload race — Ray actors are long-lived and wouldn't see post-start - volume writes anyway. - """ - import yaml - - if isinstance(val := getattr(cfg, field, None), dict): - os.makedirs(dest_dir, exist_ok=True) - path = os.path.join(dest_dir, f"{field}.yaml") - with open(path, "w") as f: - yaml.dump(val, f) - setattr(cfg, field, path) - - def build_train_cmd(cfg: Any, trainer_root: str, *, model_script_attr: str) -> str: """Build the training command, sourcing model arch args if needed. @@ -216,56 +189,3 @@ def _post_json(url: str, payload: dict, *, timeout: float) -> dict: ) with urllib.request.urlopen(request, timeout=timeout) as resp: return json.load(resp) - - -# ── Host-RAM monitor ────────────────────────────────────────────────────────── - - -def start_host_mem_monitor(interval_s: int = 20) -> None: - """Log this node's host-RAM trajectory to stdout from a daemon thread. - - The trainer can OOM-kill on host-RAM exhaustion (the publish/update_weights - full-model gather is the peak consumer), but Megatron only reports GPU memory - and the kill leaves no durable peak behind. This logs MemTotal/MemAvailable + - the container cgroup usage every ``interval_s`` so a live ``modal app logs -f`` - shows exactly which phase blows a big node and how high it peaks. Runs on EVERY - node (called from the SPMD enter()), so whichever rank OOMs has its own trace. - Best-effort: never raises.""" - host = socket.gethostname() - - def _meminfo() -> tuple[float, float]: - total = avail = 0.0 - try: - with open("/proc/meminfo") as f: - for line in f: - if line.startswith("MemTotal:"): - total = int(line.split()[1]) / 1024 / 1024 # GiB - elif line.startswith("MemAvailable:"): - avail = int(line.split()[1]) / 1024 / 1024 - except Exception: # noqa: BLE001 - pass - return total, avail - - def _cgroup_used_gib() -> float: - for path in ("/sys/fs/cgroup/memory.current", # cgroup v2 - "/sys/fs/cgroup/memory/memory.usage_in_bytes"): # v1 - try: - with open(path) as f: - return int(f.read().strip()) / 1024**3 - except Exception: # noqa: BLE001 - continue - return -1.0 - - def _loop() -> None: - while True: - total, avail = _meminfo() - used = total - avail - cg = _cgroup_used_gib() - print( - f"[hostmem] {host} used={used:.0f}GiB avail={avail:.0f}GiB " - f"total={total:.0f}GiB cgroup_used={cg:.0f}GiB", - flush=True, - ) - time.sleep(interval_s) - - threading.Thread(target=_loop, daemon=True, name="host-mem-monitor").start() diff --git a/pyproject.toml b/pyproject.toml index 46af5e79..efc6bf1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,9 @@ dependencies = [] [project.optional-dependencies] # providers/modal.py uses the Modal SDK and httpx (rollout-pool wake). modal = ["modal>=1.5.1.dev9", "httpx"] -sglang = ["fastapi", "httpx", "uvicorn"] +# The cookbook sidecars run the app under uvicorn, but Modal images install it +# explicitly; the packaged server code needs only fastapi + httpx. +sglang = ["fastapi", "httpx"] [dependency-groups] dev = ["pytest"] diff --git a/src/stitch/__init__.py b/src/stitch/__init__.py index ed24ac14..cdcc687b 100644 --- a/src/stitch/__init__.py +++ b/src/stitch/__init__.py @@ -1,27 +1 @@ """Disaggregated rollout protocol and integration helpers.""" - -from stitch.protocol import ( - Artifact, - EngineAdapter, - RolloutPoolState, - RolloutReplicaState, - SyncState, - VersionManifest, - WeightVersionPolicy, - read_latest, - version_dir, - write_latest, -) - -__all__ = [ - "Artifact", - "EngineAdapter", - "RolloutPoolState", - "RolloutReplicaState", - "SyncState", - "VersionManifest", - "WeightVersionPolicy", - "read_latest", - "version_dir", - "write_latest", -] diff --git a/src/stitch/bulletin.py b/src/stitch/bulletin.py index b91a703c..dfcb8837 100644 --- a/src/stitch/bulletin.py +++ b/src/stitch/bulletin.py @@ -56,19 +56,15 @@ class FilesystemBulletinBoard: Two on-disk layouts are supported: - - ``"stitch"`` (default): the engine-neutral protocol — ``versions/`` -nested - version dirs, a JSON ``latest.json`` pointer, and a ``manifest.json`` per - version. - - ``"slime"``: slime's native disk-delta publish output (and the customer's - object-store layout). Each run's chain lives under ``/weight_v{N:06d}/`` - and the single ``latest`` pointer holds the self-identifying snapshot identity - ``/weight_v{N:06d}`` (a bare ``weight_v{N:06d}`` for the degenerate - run-less layout). The manifest is read from each version's - ``model.safetensors.index.json``. Run-id partitioning is what makes sequential - runs collision-free: a new run writes a fresh ``/`` chain and the - pointer moves to it (a new run is not a rewind), so a finished run's chain can - never be overwritten or fast-forward a cold start. The pool reconciles against - ``latest``; the front door (or the publish hook) advances it. + - ``"stitch"`` (default): ``versions/``-nested version dirs, a JSON + ``latest.json`` pointer, and a ``manifest.json`` per version. + - ``"slime"``: slime's native disk-delta publish output. Each run's chain + lives under ``/weight_v{N:06d}/``; the single ``latest`` pointer + holds the self-identifying identity ``/weight_v{N:06d}`` (bare + ``weight_v{N:06d}`` when run-less), and the manifest is read from each + version's ``model.safetensors.index.json``. Run-id partitioning makes + sequential runs collision-free: a new run writes a fresh chain and the + pointer moves to it, which is not a rewind. """ def __init__( diff --git a/src/stitch/engines/sglang.py b/src/stitch/engines/sglang.py index 6486dd3b..fb90bdc8 100644 --- a/src/stitch/engines/sglang.py +++ b/src/stitch/engines/sglang.py @@ -19,34 +19,18 @@ def compose_extra_key( version: int, user_extra_key: str | None = None, run_id: str | None = None ) -> str: - """Compose a weight-version-namespaced SGLang ``extra_key``. - - The version segment sits at a fixed position (the prefix) and is - delimiter-terminated, so it parses unambiguously regardless of the user - key's content. sglang appends ``lora_id`` to ``extra_key`` with no - delimiter, so the version must never be parsed from the right. - Examples: ``wv7;`` (no user key), ``wv7;my-key``. This is an SGLang - radix-cache namespacing concern, not part of the engine-neutral protocol. - - ``run_id`` is folded into the key *content* (after the delimiter), so two - runs that both restart version numbering at 1 get distinct radix namespaces - (``wv1;run-a/`` vs ``wv1;run-b/``) and a stale cross-run request can never - reuse another run's same-numbered KV. ``run_id=None`` keeps the bare form. + """Compose a weight-version-namespaced SGLang radix-cache ``extra_key``, + e.g. ``wv7;my-key`` or ``wv1;run-a/my-key``. + + The version prefix namespaces the KV cache per weight version; folding + ``run_id`` into the key content keeps two runs that both restart version + numbering at 1 in distinct namespaces, so a stale cross-run request can + never reuse another run's same-numbered KV. """ run_segment = f"{run_id}/" if run_id else "" return f"wv{int(version)}{EXTRA_KEY_DELIMITER}{run_segment}{user_extra_key or ''}" -def parse_extra_key_version(extra_key: str) -> int | None: - """Inverse of :func:`compose_extra_key`. None for non-composed keys.""" - if not extra_key.startswith("wv"): - return None - head, delim, _rest = extra_key.partition(EXTRA_KEY_DELIMITER) - if not delim or not head[2:].isdigit(): - return None - return int(head[2:]) - - @dataclass class SGLangDiskDeltaAdapter: """Applies disk-delta weight versions to one local SGLang server. diff --git a/src/stitch/engines/sglang_test.py b/src/stitch/engines/sglang_test.py index 3385597d..24884cbd 100644 --- a/src/stitch/engines/sglang_test.py +++ b/src/stitch/engines/sglang_test.py @@ -4,11 +4,7 @@ import unittest from unittest import mock -from stitch.engines.sglang import ( - SGLangDiskDeltaAdapter, - compose_extra_key, - parse_extra_key_version, -) +from stitch.engines.sglang import SGLangDiskDeltaAdapter, compose_extra_key from stitch.protocol import VersionManifest @@ -72,16 +68,10 @@ async def run() -> None: class ExtraKeyTest(unittest.TestCase): - def test_compose_extra_key_round_trips_and_is_position_fixed(self) -> None: + def test_compose_extra_key_prefixes_version_and_run(self) -> None: self.assertEqual(compose_extra_key(0), "wv0;") self.assertEqual(compose_extra_key(7, "my-key"), "wv7;my-key") - self.assertEqual(parse_extra_key_version(compose_extra_key(12, None)), 12) - self.assertEqual(parse_extra_key_version(compose_extra_key(3, "wv9;decoy")), 3) - # The user key cannot shift or forge the version segment. - self.assertEqual(parse_extra_key_version("wv1;anything;else"), 1) - self.assertIsNone(parse_extra_key_version("plain-user-key")) - self.assertIsNone(parse_extra_key_version("wv12")) # no terminator - self.assertIsNone(parse_extra_key_version("wvx;k")) + self.assertEqual(compose_extra_key(1, "k", run_id="run-a"), "wv1;run-a/k") if __name__ == "__main__": diff --git a/src/stitch/protocol.py b/src/stitch/protocol.py index 195199be..580cb671 100644 --- a/src/stitch/protocol.py +++ b/src/stitch/protocol.py @@ -351,11 +351,6 @@ def to_dict(self) -> dict[str, Any]: data["metadata"] = self.metadata return data - def transition_artifact_paths(self) -> list[str]: - if self.transition_files: - return list(self.transition_files) - return [a.path for a in self.artifacts if a.kind == "transition"] - def weight_identity(version: int) -> str: """Canonical snapshot-identity string for an integer weight version.""" @@ -372,13 +367,10 @@ def parse_weight_identity(identity: str) -> int | None: def format_snapshot_identity(run_id: str | None, version: int) -> str: - """The canonical pointer/snapshot identity for a (run_id, version). - + """The canonical pointer/snapshot identity for a (run_id, version): ``/weight_v`` when a run is named, else the bare - ``weight_v`` (the degenerate single-run / customer flat layout). This - is the single self-identifying value written to the slime-layout ``latest`` - pointer: a run-scoped chain can never be mistaken for a different run's, and - an old bare pointer parses back to ``run_id=None`` rather than a phantom run. + ``weight_v``. Self-identifying, so a run-scoped chain can never be + mistaken for a different run's. """ identity = weight_identity(version) return f"{run_id}/{identity}" if run_id else identity @@ -453,9 +445,8 @@ def decide_pointer_move( ) -> PointerMove: """Decide whether the single ``latest`` writer may move to ``(run_id, version)``. - The one rule both the trainer-as-writer (bulletin board) and the - frontdoor-as-writer (hot-load API) paths share, so they can't bake in - divergent semantics: + Every ``latest`` writer (trainer publish hook, hot-load front door) shares + this rule: - A *different* run forks at base, so its version space restarts; accepting it (even at a lower number, including the ``BASE_VERSION`` claim) is a @@ -503,22 +494,11 @@ def write_latest(root: str | Path, version: int) -> None: ) -def version_not_ready_error(current: int, target: int) -> dict[str, Any]: +def _version_error(kind: str, message: str, current: int, target: int) -> dict[str, Any]: return { "error": { - "type": "WeightVersionNotReady", - "message": f"server is at version {current}, target {target} is not ready", - "current_version": int(current), - "target_version": int(target), - } - } - - -def version_too_old_error(current: int, target: int) -> dict[str, Any]: - return { - "error": { - "type": "WeightVersionTooOld", - "message": f"server is at version {current}, cannot roll back to {target}", + "type": kind, + "message": message, "current_version": int(current), "target_version": int(target), } @@ -533,17 +513,32 @@ def evaluate_version_policy( Callers decide how to react to a not-ready error (pull toward the target vs reject): the bulletin-board manager queues a sync, the hot-load shim rejects. """ + current = int(current_version) if policy.exact_version is not None: target = int(policy.exact_version) - if current_version < target: - return version_not_ready_error(current_version, target) - if current_version > target: - return version_too_old_error(current_version, target) + if current < target: + return _version_error( + "WeightVersionNotReady", + f"server is at version {current}, target {target} is not ready", + current, + target, + ) + if current > target: + return _version_error( + "WeightVersionTooOld", + f"server is at version {current}, cannot roll back to {target}", + current, + target, + ) return None - if policy.min_required_version is not None and current_version < int( - policy.min_required_version - ): - return version_not_ready_error(current_version, int(policy.min_required_version)) + if policy.min_required_version is not None and current < int(policy.min_required_version): + target = int(policy.min_required_version) + return _version_error( + "WeightVersionNotReady", + f"server is at version {current}, target {target} is not ready", + current, + target, + ) return None diff --git a/src/stitch/protocol_test.py b/src/stitch/protocol_test.py index 97acf765..21cbbdba 100644 --- a/src/stitch/protocol_test.py +++ b/src/stitch/protocol_test.py @@ -87,7 +87,7 @@ def test_manifest_round_trips_extended_and_legacy_fields(self) -> None: self.assertEqual(read_latest(root), 3) self.assertEqual(loaded.version, 3) self.assertEqual(loaded.base_version, 2) - self.assertEqual(loaded.transition_artifact_paths(), ["rank0000_flush000000.safetensors"]) + self.assertEqual(loaded.transition_files, ["rank0000_flush000000.safetensors"]) self.assertEqual(loaded.artifacts[0].checksum, "sha256:abc") self.assertEqual(loaded.run_id, "run-1") self.assertEqual(loaded.delta_encoding, "xor") diff --git a/src/stitch/servers/sglang.py b/src/stitch/servers/sglang.py index 338ecaea..9b256115 100644 --- a/src/stitch/servers/sglang.py +++ b/src/stitch/servers/sglang.py @@ -5,7 +5,7 @@ import inspect import logging import uuid -from collections.abc import Callable, Iterable +from collections.abc import Iterable from contextlib import asynccontextmanager from typing import Any @@ -44,7 +44,6 @@ def create_app( *, upstream_url: str, versioned_routes: Iterable[str] = ("generate", "v1/chat/completions"), - register_routes: Callable[[Any], None] | None = None, include_sync_routes: bool = True, upstream_timeout: float | None = 3600.0, background_sync_interval: float | None = None, @@ -54,12 +53,9 @@ def create_app( import httpx upstream_url = upstream_url.rstrip("/") - # Bound the wait for an upstream (SGLang) response. A generation that - # finishes but never delivers its HTTP body would otherwise hang this proxy - # forever (timeout=None), holding the request open and wedging the client - # awaiting it — exactly the failure mode that stalled a rollout for hours. - # On timeout the upstream call raises, surfacing as a 5xx the client can - # retry, instead of an infinite hold. connect stays short (upstream is + # Bound the wait for an upstream (SGLang) response: a generation that + # finishes but never delivers its HTTP body must surface as a retryable 5xx, + # not hold the request open forever. connect stays short (upstream is # localhost); pass None to opt out. upstream_request_timeout = httpx.Timeout(upstream_timeout, connect=10.0) versioned_route_set = {route.strip("/") for route in versioned_routes} @@ -150,9 +146,6 @@ async def rpc_sync_from_bulletin_board(request: Request) -> dict[str, Any]: "sync_state": _sync_state_value(getattr(manager, "sync_state", None)), } - if register_routes is not None: - register_routes(app) - async def _watch_disconnect(request: Request) -> None: while True: message = await request.receive() @@ -329,11 +322,8 @@ async def _upstream_call() -> Any: if versioned_route and isinstance(data, dict): # Driven by the same `versioned_route` flag that gated and stamped the - # request, so injection can't diverge from gating (previously a fixed - # path list here meant /v1/completions got version metadata while - # going ungated, and a custom versioned route got gated but no - # metadata). /generate carries it in meta_info; OpenAI-style routes - # at the top level. + # request, so injection can't diverge from gating. /generate carries + # it in meta_info; OpenAI-style routes at the top level. if route == "generate": meta = data.setdefault("meta_info", {}) meta["weight_version"] = str(start_version) diff --git a/src/stitch/sync.py b/src/stitch/sync.py index 35e0c492..f54adb19 100644 --- a/src/stitch/sync.py +++ b/src/stitch/sync.py @@ -61,9 +61,7 @@ class RolloutAdmissionGate: ``_active_cond`` acquisition the committer uses, and commits hold the gate across the engine apply *and* the version advance (cleared only after). Subclasses provide ``current_version`` and override the hooks for their - policy / admission / exact-pin specifics. ``WeightSyncManager`` composes it - (so the bulletin-board and hot-load provider sidecars, both WeightSyncManager, - share the gate semantics and the P0.1 commit-window fix in one place). + policy / admission / exact-pin specifics. """ def __init__(self, *, commit_mode: CommitMode = "quiesce") -> None: @@ -173,8 +171,7 @@ async def commit_version( a finally, with the version advanced before resume so new admissions see the new namespace. On failure the gate (and pause) are unwound and the served version is left unchanged — ``on_applied`` runs only after a - successful apply. Both the bulletin-board manager and the hot-load shim - commit through here, so the gate sequencing lives in one place. + successful apply. """ await self._begin_commit(self._commit_ready) try: @@ -262,13 +259,6 @@ def _on_policy_violation(self, error: dict[str, Any]) -> None: if error["error"]["type"] == "WeightVersionNotReady": self.queue_sync(error["error"]["target_version"]) - async def validate_policy(self, policy: WeightVersionPolicy) -> tuple[bool, int, Mapping[str, Any] | None]: - """Advisory pre-check. The authoritative check is in request_context.""" - error = self._policy_error(policy) - if error is not None and error["error"]["type"] == "WeightVersionNotReady": - self.queue_sync(error["error"]["target_version"]) - return error is None, self.current_version, error - def queue_sync(self, target_version: int | None = None) -> None: run_id, latest = self.board.read_latest() self.latest_seen_version = max(self.latest_seen_version, latest) diff --git a/src/stitch/sync_test.py b/src/stitch/sync_test.py index 1d654f93..afd3a7ee 100644 --- a/src/stitch/sync_test.py +++ b/src/stitch/sync_test.py @@ -135,30 +135,6 @@ async def run() -> None: asyncio.run(run()) - def test_exact_and_min_policy_errors_are_retryable(self) -> None: - async def run() -> None: - with tempfile.TemporaryDirectory() as tmp: - board = FilesystemBulletinBoard(tmp) - board.publish_manifest(VersionManifest(version=1, base_version=0, backend="fake", load_format="noop")) - manager = WeightSyncManager(board=board, engine=FakeEngine()) - - ok, current, error = await manager.validate_policy(WeightVersionPolicy(exact_version=1)) - self.assertFalse(ok) - self.assertEqual(current, 0) - self.assertEqual(error["error"]["type"], "WeightVersionNotReady") - - await manager.sync_to(1) - ok, current, error = await manager.validate_policy(WeightVersionPolicy(min_required_version=1)) - self.assertTrue(ok) - self.assertEqual(current, 1) - self.assertIsNone(error) - - ok, _current, error = await manager.validate_policy(WeightVersionPolicy(exact_version=0)) - self.assertFalse(ok) - self.assertEqual(error["error"]["type"], "WeightVersionTooOld") - - asyncio.run(run()) - def test_request_context_pins_and_reports_serving_version(self) -> None: async def run() -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/src/stitch/trainers/slime.py b/src/stitch/trainers/slime.py index b19c61e5..d4040f21 100644 --- a/src/stitch/trainers/slime.py +++ b/src/stitch/trainers/slime.py @@ -63,36 +63,13 @@ def publish_delta_version( def rollout_request_weight_version_hook(args: Namespace, sample: Any, request: dict[str, Any]) -> None: - """Attach provider admission constraints to one SLIME rollout request. + """Slime ``custom_rollout_request_hook_path`` hook: set retry budget and + session affinity on one rollout request. - The hook is request-level control, not the trainer's staleness policy: it - prevents an opaque rollout router from spending compute on a replica that - cannot serve a version the trainer has already decided is usable. + The pool serves whatever version each replica has hot-loaded; a lagging + replica returns a retryable 409, so the retry budget is what keeps requests + flowing across a weight update. """ - - mode = str(getattr(args, "rollout_request_weight_version_mode", "exact")) - # PR #5's per-request hook receives no rollout_id (it is per-rollout context, - # not per-request — sample.index is the batch position), so the trainer-step - # pin only runs when one is supplied. Otherwise the pool serves the latest - # hot-loaded version and a lagging replica returns a retryable 409. TODO: - # re-derive the per-request target (e.g. the latest published version) under - # the PR #5 contract if strict pinning is needed. - rollout_id = request.get("rollout_id") - if mode != "none" and rollout_id is not None: - target_version = rollout_target_weight_version( - args, - int(rollout_id), - evaluation=bool(request.get("evaluation", False)), - ) - if not bool(request.get("evaluation", False)): - target_version = max(0, target_version - int(getattr(args, "rollout_request_weight_version_lag", 0))) - if mode == "exact": - request["payload"]["weight_version"] = {"exact_version": target_version} - elif mode == "min": - request["payload"]["weight_version"] = {"min_required_version": target_version} - else: - raise ValueError(f"Unsupported rollout_request_weight_version_mode: {mode!r}") - # Generous retries on every request so a lagging/scaling replica (a 409 # weight-version reject or a transient error) is retried, not failed. request["max_retries"] = int(getattr(args, "rollout_request_retry_attempts", request.get("max_retries", 60))) @@ -110,51 +87,6 @@ def rollout_request_weight_version_hook(args: Namespace, sample: Any, request: d request["headers"] = headers -def generate_rollout( - args: Namespace, - rollout_id: int, - data_source: Any, - evaluation: bool = False, -): - """Run SLIME's default SGLang rollout. - - Kept as a compatibility wrapper for older configs. New configs should use - ``slime.rollout.sglang_rollout.generate_rollout`` directly plus - ``custom_rollout_request_hook_path`` when they need request constraints. - """ - from slime.rollout import sglang_rollout as upstream_rollout - - assert args.rollout_global_dataset - logger.info( - "Disaggregated %s rollout_id=%s", - "eval" if evaluation else "train", - rollout_id, - ) - - with upstream_rollout.rollout_request_context(args, rollout_id, evaluation=evaluation): - if evaluation: - output, _ = upstream_rollout.run(upstream_rollout.eval_rollout(args, rollout_id)) - return output - - output, aborted_samples = upstream_rollout.run( - upstream_rollout.generate_rollout_async(args, rollout_id, data_source.get_samples) - ) - if aborted_samples: - data_source.add_samples(aborted_samples) - return output - - -def rollout_target_weight_version(args: Namespace, rollout_id: int, evaluation: bool = False) -> int: - if not evaluation: - return int(rollout_id) - # Eval pins to the latest published version (the slime-native `latest`). - try: - _, version = FilesystemBulletinBoard(_bulletin_root(args), layout="slime").read_latest() - return version - except Exception: # noqa: BLE001 - return int(rollout_id) - - def _bulletin_root(args: Any) -> str: root = ( getattr(args, "update_weight_disk_dir", None) diff --git a/src/stitch/trainers/slime_test.py b/src/stitch/trainers/slime_test.py index 2592fdbf..9b269f20 100644 --- a/src/stitch/trainers/slime_test.py +++ b/src/stitch/trainers/slime_test.py @@ -10,82 +10,38 @@ class SlimeHooksTest(unittest.TestCase): - def test_rollout_request_hook_adds_exact_policy_retry_and_affinity(self) -> None: + def test_rollout_request_hook_sets_retries_and_affinity(self) -> None: args = Namespace( - rollout_request_weight_version_mode="exact", - rollout_request_weight_version_lag=1, rollout_request_retry_attempts=240, rollout_request_retry_sleep=0.25, ) sample = Namespace(session_id="session-1") - request = { - "payload": {}, - "headers": None, - "max_retries": 60, - "retry_sleep": 1.0, - "rollout_id": 3, - "evaluation": False, - } + request = {"payload": {}, "headers": None, "max_retries": 60, "retry_sleep": 1.0} rollout_request_weight_version_hook(args, sample, request) - self.assertEqual(request["payload"]["weight_version"], {"exact_version": 2}) self.assertEqual(request["max_retries"], 240) self.assertEqual(request["retry_sleep"], 0.25) self.assertEqual(request["headers"]["x-session-affinity"], "session-1") def test_rollout_request_hook_uses_configured_affinity_header(self) -> None: - args = Namespace( - rollout_request_weight_version_mode="exact", - rollout_session_affinity_header="Modal-Session-ID", - ) + args = Namespace(rollout_session_affinity_header="Modal-Session-ID") sample = Namespace(session_id="group-7") - request = { - "payload": {}, - "headers": None, - "max_retries": 60, - "retry_sleep": 1.0, - "rollout_id": 3, - "evaluation": False, - } + request = {"payload": {}, "headers": None, "max_retries": 60, "retry_sleep": 1.0} rollout_request_weight_version_hook(args, sample, request) self.assertEqual(request["headers"]["Modal-Session-ID"], "group-7") self.assertNotIn("x-session-affinity", request["headers"]) - def test_rollout_request_hook_can_add_min_policy(self) -> None: - args = Namespace(rollout_request_weight_version_mode="min") + def test_rollout_request_hook_skips_affinity_without_session(self) -> None: + args = Namespace() sample = Namespace(session_id=None) - request = { - "payload": {}, - "headers": None, - "max_retries": 60, - "retry_sleep": 1.0, - "rollout_id": 3, - "evaluation": False, - } - - rollout_request_weight_version_hook(args, sample, request) - - self.assertEqual(request["payload"]["weight_version"], {"min_required_version": 3}) - self.assertIsNone(request["headers"]) - - def test_rollout_request_hook_degrades_without_rollout_id(self) -> None: - # PR #5's request carries no rollout_id: skip the pin (no crash), still - # apply the retry budget and session affinity. - args = Namespace( - rollout_request_weight_version_mode="exact", - rollout_request_retry_attempts=240, - ) - sample = Namespace(session_id="grp-9") request = {"payload": {}, "headers": None, "max_retries": 60, "retry_sleep": 1.0} rollout_request_weight_version_hook(args, sample, request) - self.assertNotIn("weight_version", request["payload"]) - self.assertEqual(request["max_retries"], 240) - self.assertEqual(request["headers"]["x-session-affinity"], "grp-9") + self.assertIsNone(request["headers"]) def test_publish_delta_version_writes_manifest_and_latest(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/uv.lock b/uv.lock index ba7f14f1..0dd71047 100644 --- a/uv.lock +++ b/uv.lock @@ -971,7 +971,6 @@ modal = [ sglang = [ { name = "fastapi" }, { name = "httpx" }, - { name = "uvicorn" }, ] [package.dev-dependencies] @@ -985,7 +984,6 @@ requires-dist = [ { name = "httpx", marker = "extra == 'modal'" }, { name = "httpx", marker = "extra == 'sglang'" }, { name = "modal", marker = "extra == 'modal'", specifier = ">=1.5.1.dev9" }, - { name = "uvicorn", marker = "extra == 'sglang'" }, ] provides-extras = ["modal", "sglang"] @@ -1052,19 +1050,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] -[[package]] -name = "uvicorn" -version = "0.49.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", size = 91284, upload-time = "2026-06-03T22:01:30.448Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", size = 71376, upload-time = "2026-06-03T22:01:29.037Z" }, -] - [[package]] name = "watchfiles" version = "1.2.0"