diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 98e3dc06a..2404cf0d7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,6 +29,9 @@ jobs: - name: Run ruff format check run: uv run ruff format --check src/srtctl/ + - name: Check docs/schema-reference.md is regenerated + run: uv run srtctl schema-docs --check + typecheck: runs-on: ubuntu-latest steps: diff --git a/CLAUDE.md b/CLAUDE.md index e9d3c3cf0..aaac48d1a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -268,6 +268,10 @@ with patch.dict(os.environ, H100Rack.slurm_env()): 3. Add bash script to `benchmarks/scripts/mybench/bench.sh` 4. Register in benchmark type mapping +### Adding or Changing Any Config Field + +`docs/schema-reference.md` is generated from the dataclasses in `core/schema.py` and `backends/`. After adding, renaming, or re-typing a field, run `uv run srtctl schema-docs` and commit the result; CI and `tests/test_schema_docs.py` fail when the file is stale. Put the field's description in the class docstring `Attributes:` block or in a `#` comment directly above the field so it lands in the generated table. + ### Adding Config That Affects srun (Mounts, Env Vars, Options) When adding new config fields that affect what gets passed to srun (environment variables, container mounts, srun options), you must also update: diff --git a/Makefile b/Makefile index 35e43a75c..d83fb8fec 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint test test-cov ci check setup cleanup examples tachometer-scraper tachometer-scraper-download +.PHONY: lint test test-cov ci check setup cleanup examples schema-docs schema-docs-check tachometer-scraper tachometer-scraper-download NATS_VERSION ?= v2.10.28 ETCD_VERSION ?= v3.5.21 @@ -21,8 +21,16 @@ test: test-cov: uv run pytest tests/ --cov=srtctl --cov-report=term-missing --cov-report=html +# Regenerate docs/schema-reference.md from the config dataclasses +schema-docs: + uv run srtctl schema-docs + +# Fail if docs/schema-reference.md is stale (also enforced by CI and tests/test_schema_docs.py) +schema-docs-check: + uv run srtctl schema-docs --check + # Run lint + tests in one command -check: lint test +check: lint schema-docs-check test @echo "✓ All checks passed" tachometer-scraper: diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index a3a5b4a0b..d9c144664 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -9,6 +9,7 @@ ## Configuration - [Configuration Reference](config-reference.md) +- [Schema Reference (generated)](schema-reference.md) - [Parameter Sweeps](sweeps.md) - [Config Overrides](config-reference.md#config-overrides) diff --git a/docs/config-reference.md b/docs/config-reference.md index 87e279871..1011088a6 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -2,6 +2,8 @@ Complete reference for job configuration YAML files. +This page is the prose guide: what each block means, how the pieces interact, and worked examples. The authoritative field-by-field list (every key, type, and default) is generated from the code in [schema-reference.md](schema-reference.md) and checked in CI, so if this page and that one disagree, the generated one is right. + ## Table of Contents - [Overview](#overview) diff --git a/docs/schema-reference.md b/docs/schema-reference.md new file mode 100644 index 000000000..182ce25d2 --- /dev/null +++ b/docs/schema-reference.md @@ -0,0 +1,563 @@ +# Schema Reference + + + +Field-level reference for recipe YAML (`SrtConfig`) and the cluster config `srtslurm.yaml` (`ClusterConfig`), generated from the dataclasses in `srtctl.core.schema` and `srtctl.backends`. Each table lists the YAML key, the type, the default (`required` when there is none), and a description taken from the class docstring or the comment on the field. Nested types link to their own table. For prose, examples, and semantics see [config-reference.md](config-reference.md). + +## Recipe + +Top-level keys of a recipe YAML. + +| Key | Type | Default | Description | +|---|---|---|---| +| `name` | str | required | | +| `model` | [ModelConfig](#modelconfig) | required | | +| `resources` | [ResourceConfig](#resourceconfig) | required | | +| `slurm` | [SlurmConfig](#slurmconfig) | `SlurmConfig()` | | +| `backend` | [SGLangProtocol](#sglangprotocol) \| [TRTLLMProtocol](#trtllmprotocol) \| [VLLMProtocol](#vllmprotocol) \| [MockerProtocol](#mockerprotocol) | `SGLangProtocol()` | | +| `frontend` | [FrontendConfig](#frontendconfig) | `FrontendConfig()` | | +| `dynamo` | [DynamoConfig](#dynamoconfig) | `DynamoConfig()` | | +| `benchmark` | [BenchmarkConfig](#benchmarkconfig) | `BenchmarkConfig()` | | +| `profiling` | [ProfilingConfig](#profilingconfig) | `ProfilingConfig()` | | +| `output` | [OutputConfig](#outputconfig) | `OutputConfig()` | | +| `health_check` | [HealthCheckConfig](#healthcheckconfig) | `HealthCheckConfig()` | | +| `infra` | [InfraConfig](#infraconfig) | `InfraConfig()` | | +| `observability` | [ObservabilityConfig](#observabilityconfig) | `ObservabilityConfig()` | | +| `telemetry` | [TelemetryConfig](#telemetryconfig) | `TelemetryConfig()` | | +| `environment` | dict[str, str] | `{}` | | +| `container_mounts` | dict[[FormattablePath](#formattablepath), [FormattablePath](#formattablepath)] | `{}` | | +| `extra_mount` | tuple[str, ...] \| None | `None` | | +| `srun_options` | dict[str, str] | `{}` | | +| `sbatch_directives` | dict[str, str] | `{}` | | +| `enable_config_dump` | bool | `True` | | +| `setup_script` | str \| None | `None` | Custom setup script (runs before dynamo install and worker startup) e.g. "custom-setup.sh" -> runs /configs/custom-setup.sh | +| `host_setup` | [HostSetupConfig](#hostsetupconfig) | `HostSetupConfig()` | Commands run on each node's bare host, outside the container, before any worker starts. Cluster-wide default lives in srtslurm.yaml as default_host_setup; a recipe that sets this block replaces that default. | +| `identity` | [IdentityConfig](#identityconfig) | `IdentityConfig()` | Virtual identity — declares what *should* be running (verified against fingerprint) | +| `reporting` | [ReportingConfig](#reportingconfig) \| None | `None` | Reporting configuration (status API, future: logs to S3, etc.) | + +## Recipe sections + +### ModelConfig + +Model configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `path` | str | required | | +| `container` | str | required | | +| `precision` | str | required | | +| `stage_dir` | str \| None | `None` | Optional: stage the model from shared storage to this node-local dir before workers start (e.g. "/raid/scratch/models"). None = use path directly. | + +### ResourceConfig + +Resource allocation configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `gpu_type` | str | required | | +| `gpus_per_node` | int | `4` | | +| `prefill_nodes` | int \| None | `None` | Disaggregated mode | +| `decode_nodes` | int \| None | `None` | | +| `prefill_workers` | int \| None | `None` | | +| `decode_workers` | int \| None | `None` | | +| `agg_nodes` | int \| None | `None` | Aggregated mode | +| `agg_workers` | int \| None | `None` | | +| `spread_workers` | bool | `False` | If True, place each partial-node worker on its own node instead of packing multiple onto the same node. Caller must reserve enough nodes (e.g. set decode_nodes=decode_workers when gpus_per_decode``. ``benchmark.command`` has no placeholder substitution, so write the URL out literally. | +| `num_additional_frontends` | int | `9` | Additional routers beyond master (default: 9) | +| `nginx_container` | str | `'nginx:1.27.4'` | Custom nginx container image (default: nginx:1.27.4) | +| `nginx_raise_ulimit` | bool | `False` | Raise nofile before nginx and set ``worker_rlimit_nofile`` in generated nginx.conf. Off by default; enable on clusters that allow it. Override per job or set ``nginx_raise_ulimit`` in srtslurm.yaml for the cluster. | +| `nginx_session_affinity` | bool | `False` | Consistently hash ``nginx_session_affinity_header`` to a frontend. Requests without that header use a generated request ID and stay distributed. | +| `nginx_session_affinity_header` | str | `'X-Dynamo-Session-ID'` | Header hashed when affinity is on (default ``X-Dynamo-Session-ID``). Set ``X-Correlation-ID`` for clients (e.g. aiperf) that carry the session id in that header instead. | +| `nginx_keepalive_timeout` | str | `'600s'` | Idle timeout for client and upstream keepalive connections in the generated nginx.conf (default "600s"). nginx's own default is 75s, which closes a session's connection during the long recorded think-time of an agentic replay; the client's next write on that pooled socket then fails with "broken pipe" / "server disconnected" and nothing is logged server-side. | +| `args` | dict[str, Any] \| None | `None` | CLI arguments passed to the frontend/router process | +| `env` | dict[str, str] \| None | `None` | Environment variables for frontend processes | +| `container_image` | str \| None | `None` | Optional router-specific image. Static routers use the model/backend image when omitted. | +| `ctx_router` | dict[str, Any] \| None | `None` | trtllm_serve orchestrator (ser.yaml) options; ignored by other frontends. | +| `gen_router` | dict[str, Any] \| None | `None` | generation_servers.router | +| `server_config_extra` | dict[str, Any] \| None | `None` | extra top-level ser.yaml keys | +| `orchestrator_placement` | str | `'head'` | trtllm_serve: which node runs the disaggregated orchestrator. "head" (default) -> nodes.head (first prefill/CTX node) "first_decode" -> first decode/GEN worker-leader node | +| `dedicated_node` | bool | `False` | If True, reserve a node exclusively for the frontend/orchestrator instead of running it on a worker node. Requires at least 2 nodes. Not supported together with resources.het_jobs: true. Default: False. | + +### DynamoConfig + +Dynamo installation configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `install` | bool | `True` | | +| `version` | str \| None | `'0.8.0'` | | +| `hash` | str \| None | `None` | | +| `top_of_tree` | bool | `False` | | +| `wheel` | str \| None | `None` | | +| `request_plane` | str | `'tcp'` | | +| `event_plane` | str \| None | `None` | | +| `sidecar` | bool | `False` | | +| `sidecar_port` | int | `50051` | | +| `sidecar_binary` | str \| None | `None` | | +| `sidecar_startup_timeout` | int | `1200` | | +| `sidecar_context_length` | int \| None | `None` | | +| `sidecar_args` | list[str] | `[]` | | +| `cargo_patches` | list[str] \| None | `None` | Optional dependency-declaration overrides applied to the dynamo Cargo.toml tree before a source build (requires `hash`). Each entry is a full ` = ` TOML line, e.g. 'dynamo-tokenizers = { git = "https://github.com/ai-dynamo/frontend-crates", branch = "..." }' The crate's existing declaration is replaced tree-wide, letting a source build pull a crate from an unmerged branch without waiting for a crates.io release. | + +### BenchmarkConfig + +Benchmark configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `type` | str | `'manual'` | | +| `isl` | int \| None | `None` | | +| `osl` | int \| None | `None` | | +| `concurrencies` | list[int] \| str \| None | `None` | | +| `req_rate` | str \| int \| None | `'inf'` | | +| `client_placement` | str | `'head'` | Which node runs the benchmark client: "head" (default) -> nodes.head (co-located with orchestrator by default) "last_decode" -> last decode/GEN worker-leader node (isolate the client off the CTX/orchestrator node). When the client lands on a different node than the orchestrator, use the injected $SRT_FRONTEND_HOST env in the benchmark command's URL. | +| `client_dedicated_node` | bool | `False` | If True, reserve a node exclusively for the benchmark client instead of running it on a worker node. Requires at least 2 nodes. Not supported together with resources.het_jobs: true. Default: False. | +| `colocate_with_frontend` | bool | `True` | Governs how the dedicated-node flags combine when more than one of client_dedicated_node, frontend.dedicated_node, and infra.etcd_nats_dedicated_node is set. If True (default), every requested role shares a single reserved node. If False, each requested role gets its own reserved node (requires enough total nodes: worker count + number of dedicated roles). | +| `sweep` | [SweepConfig](#sweepconfig) \| None | `None` | | +| `num_examples` | int \| None | `None` | Accuracy benchmark fields | +| `max_tokens` | int \| None | `None` | | +| `repeat` | int \| None | `None` | | +| `num_threads` | int \| None | `None` | | +| `max_context_length` | int \| None | `None` | | +| `categories` | list[str] \| None | `None` | | +| `num_shots` | int \| None | `None` | GSM8K few-shot examples | +| `temperature` | float \| None | `None` | | +| `top_p` | float \| None | `None` | | +| `top_k` | int \| None | `None` | | +| `num_requests` | int \| None | `None` | Router benchmark fields | +| `concurrency` | int \| None | `None` | | +| `prefix_ratios` | list[float] \| str \| None | `None` | | +| `mooncake_workload` | str \| None | `None` | Mooncake router benchmark fields (uses aiperf with mooncake_trace) | +| `ttft_threshold_ms` | int \| None | `None` | Goodput TTFT threshold in ms (default: 2000) | +| `itl_threshold_ms` | int \| None | `None` | Goodput ITL threshold in ms (default: 25) | +| `random_range_ratio` | float \| None | `None` | Random input/output length range ratio (default: 0.8) | +| `num_prompts_mult` | int \| None | `None` | Multiplier for num_prompts = concurrency * mult (default: 10) | +| `num_warmup_mult` | int \| None | `None` | Multiplier for warmup prompts = concurrency * mult (default: 2) | +| `dataset_name` | str \| None | `None` | Custom dataset fields (sa-bench) | +| `dataset_path` | str \| None | `None` | Container path to dataset file (mount via extra_mount) | +| `agentperf_client_dir` | str \| None | `None` | AgentPerf benchmark fields (agentperf-client trajectory replay) | +| `agentperf_config` | str \| None | `None` | | +| `trace_file` | str \| None | `None` | Trace replay benchmark fields (uses aiperf with mooncake_trace dataset type) | +| `custom_tokenizer` | str \| None | `None` | Custom tokenizer class (e.g., "module.path.ClassName") | +| `use_chat_template` | bool | `True` | Pass --use-chat-template to benchmark (default: true) | +| `reuse_http_connections` | bool | `False` | SA-Bench Dynamo adapter: reuse a benchmark-scoped HTTP connection pool. Opt-in to preserve the historical per-request ClientSession behavior. | +| `command` | str \| None | `None` | Custom benchmark hook. ``command`` is passed to ``bash -lc`` verbatim; srtctl does NOT substitute placeholders like ``{nginx_url}`` or ``{slurm_job_id}``. Render any parameters when generating the recipe. See srtctl.benchmarks.custom.CustomBenchmarkRunner for details. | +| `container_image` | str \| None | `None` | | +| `env` | dict[str, str] | `{}` | | +| `aiperf_package` | str \| None | `None` | aiperf pip install spec (e.g., "aiperf>=0.7.0", "aiperf @ git+https://...@commit") If set, runs pip install before benchmarking. Upgrades if already installed. | +| `aiperf_args` | dict[str, Any] | `{}` | Extra aiperf CLI flags passed through to bench.sh (e.g., benchmark-duration: 600, workers-max: 200) | +| `export_node_metrics` | bool | `False` | Post-process: export analysis/srtlog per-node batch CSVs + gen_throughput.csv (see postprocess_stage) | +| `slow_down_sleep_time` | float \| None | `None` | SA-Bench: optional SGLang /slow_down on decode workers (sglang frontend only; see benchmark_stage) | +| `slow_down_wait_time` | float \| None | `None` | seconds until POST clears slow_down; unset = feature off | + +### ProfilingConfig + +Profiling configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `type` | str | `'none'` | "none", "nsys", "nsys-time", or "torch" | +| `extra_nsys_args` | list[str] \| None | `None` | Extra arguments passed to nsys profile (appended before `-o`; see get_nsys_prefix) | +| `prefill` | [ProfilingPhaseConfig](#profilingphaseconfig) \| None | `None` | Phase-specific profiling step configs (not used for nsys-time) | +| `decode` | [ProfilingPhaseConfig](#profilingphaseconfig) \| None | `None` | | +| `aggregated` | [ProfilingPhaseConfig](#profilingphaseconfig) \| None | `None` | | +| `delay_secs` | int \| None | `None` | nsys-time fields: time-based capture window, same on all workers | +| `duration_secs` | int \| None | `None` | nsys --duration: seconds to capture after delay | +| `benchmark_duration_secs` | int | `300` | total traffic generation duration (must cover delay + duration) | + +### OutputConfig + +Output configuration with formattable paths. + +| Key | Type | Default | Description | +|---|---|---|---| +| `log_dir` | [FormattablePath](#formattablepath) | `()` | | + +### HealthCheckConfig + +Health check configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `max_attempts` | int | `180` | 30 minutes default (large models take time to load) | +| `interval_seconds` | int | `10` | | + +### InfraConfig + +Infrastructure configuration for etcd/nats placement. + +| Key | Type | Default | Description | +|---|---|---|---| +| `etcd_nats_dedicated_node` | bool | `False` | If True, run etcd and nats on a dedicated node instead of the head node. This reserves the first node exclusively for infrastructure services. Default: False. | +| `nats_max_payload_mb` | int \| None | `None` | Maximum NATS message payload in MB. Default: None (uses NATS default of 1MB). Set to 24+ for disaggregated serving with long ISL (e.g. 65K+ tokens where prompt data exceeds 1MB in NATS messages). | + +### ObservabilityConfig + +Observability configuration for OTEL tracing. + +| Key | Type | Default | Description | +|---|---|---|---| +| `enabled` | bool | `False` | Master analytics knob. Default: False. | +| `enable_otel` | bool | `False` | If True, inject OTEL environment variables into all workers and frontends. Requires otel_endpoint to be set. Default: False. | +| `otel_endpoint` | str \| None | `None` | OTEL collector endpoint (e.g. "http://10.0.0.1:4317"). Required when enable_otel is True. | +| `tachometer` | [TachometerConfig](#tachometerconfig) | `TachometerConfig()` | Native Tachometer capture configuration. Follows ``enabled`` unless ``tachometer.enabled`` is set explicitly (see :class:`TachometerConfig`). | + +### TelemetryConfig + +DCGM power telemetry for benchmark measurement windows. + +| Key | Type | Default | Description | +|---|---|---|---| +| `enabled` | bool | `False` | | +| `dcgm_exporter` | [TelemetryExporterConfig](#telemetryexporterconfig) \| None | `None` | | +| `collect_interval_ms` | int | `1000` | Milliseconds between collector cycles. Replaces the retired ``default_frequency``, which despite its name was a period in seconds (1000ms == the old 1.0 default). | +| `storage_subdir` | str | `'power'` | | +| `required` | bool | `False` | | +| `startup_timeout_seconds` | float | `30.0` | | +| `request_timeout_seconds` | float | `2.0` | | +| `collector_join_timeout_seconds` | float \| None | `None` | None derives a safe shutdown budget from request_timeout_seconds. | + +### FormattablePath + +A path that may contain placeholders requiring formatting. + +| Key | Type | Default | Description | +|---|---|---|---| +| `template` | str | required | | + +### HostSetupConfig + +Commands run on the bare host of each allocated node, outside the container. + +| Key | Type | Default | Description | +|---|---|---|---| +| `commands` | list[str] | `[]` | Shell commands run in order on each node, joined with ``&&``. | +| `teardown` | list[str] | `[]` | Shell commands run on each node after workers stop. Runs even when the job fails, so state that outlives the allocation (locked clocks persist for the next tenant) gets reset. | +| `nodes` | one of `'all'`, `'workers'` | `'all'` | Which nodes to target. "all" covers head, infra, and workers; "workers" covers only the nodes running backend workers. | +| `ignore_failure` | bool | `False` | When True, a failing node logs a warning instead of failing the job. | +| `timeout_seconds` | int | `300` | Per-node wall-clock budget for commands and for teardown. | + +### IdentityConfig + +Virtual identity for runtime verification and reproduction. + +| Key | Type | Default | Description | +|---|---|---|---| +| `model` | [IdentityModelConfig](#identitymodelconfig) | `IdentityModelConfig()` | | +| `container` | [IdentityContainerConfig](#identitycontainerconfig) | `IdentityContainerConfig()` | | +| `frameworks` | dict[str, str] | `{}` | | + +### ReportingConfig + +Reporting configuration for status updates, AI analysis, and log exports. + +| Key | Type | Default | Description | +|---|---|---|---| +| `status` | [ReportingStatusConfig](#reportingstatusconfig) \| None | `None` | | +| `ai_analysis` | [AIAnalysisConfig](#aianalysisconfig) \| None | `None` | | +| `s3` | [S3Config](#s3config) \| None | `None` | | + +### SweepConfig + +Configuration for benchmark parameter sweeps. + +| Key | Type | Default | Description | +|---|---|---|---| +| `mode` | one of `'zip'`, `'grid'` | `'zip'` | | +| `parameters` | dict[str, list[Any]] | `{}` | | + +### ProfilingPhaseConfig + +Profiling config for a single phase (prefill/decode/aggregated). + +| Key | Type | Default | Description | +|---|---|---|---| +| `start_step` | int \| None | `None` | Step to start profiling | +| `stop_step` | int \| None | `None` | Step to stop profiling | + +### TachometerConfig + +Native Tachometer collection for an observability-enabled run. + +| Key | Type | Default | Description | +|---|---|---|---| +| `enabled` | bool \| None | `None` | | +| `binary_path` | str | `'tachometer-scraper'` | | +| `collect_interval_ms` | int | `1000` | Milliseconds between scrapes of every endpoint — the same unit and name as dcgm-exporter's --collect-interval. Replaces the retired Hz-based ``default_frequency`` (1000ms == the old 1.0 Hz default). | +| `sync_interval_secs` | int | `120` | | +| `compaction_threads` | int | `4` | | +| `storage_subdir` | str | `'tachometer'` | | +| `extra_metadata` | dict[str, str] | `{}` | | +| `default_exporters` | bool | `True` | | +| `dcgm_exporter` | [TelemetryExporterConfig](#telemetryexporterconfig) \| None | `None` | | +| `node_exporter` | [TelemetryExporterConfig](#telemetryexporterconfig) \| None | `None` | | + +### TelemetryExporterConfig + +Configuration for a metrics exporter deployed on worker nodes. + +| Key | Type | Default | Description | +|---|---|---|---| +| `container_image` | str | required | | +| `port` | int | required | | +| `command` | str \| None | `None` | | + +### IdentityModelConfig + +Virtual model identity for runtime verification. + +| Key | Type | Default | Description | +|---|---|---|---| +| `repo` | str \| None | `None` | HuggingFace model ID, e.g. "nvidia/Kimi-K2.5-NVFP4" | +| `revision` | str \| None | `None` | HuggingFace git commit SHA | + +### IdentityContainerConfig + +Container identity for reproduction (not verified at runtime). + +| Key | Type | Default | Description | +|---|---|---|---| +| `image` | str \| None | `None` | Docker URI, e.g. "gitlab-master:5005/.../trtllm-arm64" | + +### ReportingStatusConfig + +Status reporting configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `endpoint` | str \| None | `None` | | +| `endpoints` | list[str] \| None | `None` | | + +### AIAnalysisConfig + +AI-powered failure analysis configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `enabled` | bool | `False` | Whether to run AI analysis on benchmark failures | +| `openrouter_api_key` | str \| None | `None` | OpenRouter API key (falls back to OPENROUTER_API_KEY env var) | +| `gh_token` | str \| None | `None` | GitHub token for gh CLI (falls back to GH_TOKEN env var) | +| `repos_to_search` | list[str] | `()` | GitHub repos to search for related PRs | +| `pr_search_days` | int | `14` | Number of days to look back for PRs | +| `prompt` | str \| None | `None` | Custom prompt template (uses DEFAULT_AI_ANALYSIS_PROMPT if None) Available variables: {log_dir}, {repos}, {pr_days} | + +### S3Config + +S3 upload configuration for log artifacts. + +| Key | Type | Default | Description | +|---|---|---|---| +| `bucket` | str | required | S3 bucket name | +| `prefix` | str \| None | `None` | Optional prefix/path within bucket (e.g., "srtslurm/logs") | +| `region` | str \| None | `None` | AWS region (e.g., "us-west-2") | +| `endpoint_url` | str \| None | `None` | Custom S3-compatible endpoint URL (optional) | +| `access_key_id` | str \| None | `None` | AWS access key ID (falls back to AWS_ACCESS_KEY_ID env var) | +| `secret_access_key` | str \| None | `None` | AWS secret access key (falls back to AWS_SECRET_ACCESS_KEY env var) | + +## Backend types + +`backend.type` selects one of the following; the remaining `backend` keys are that type's fields. + +### SGLangProtocol + +`backend.type: sglang` + +SGLang protocol - implements BackendProtocol. + +| Key | Type | Default | Description | +|---|---|---|---| +| `type` | one of `'sglang'` | `'sglang'` | | +| `gpu_type` | str \| None | `None` | | +| `prefill_environment` | dict[str, str] | `{}` | Environment variables per mode | +| `decode_environment` | dict[str, str] | `{}` | | +| `aggregated_environment` | dict[str, str] | `{}` | | +| `sglang_config` | [SGLangServerConfig](#sglangserverconfig) \| None | `None` | SGLang server CLI config per mode | +| `kv_events_config` | bool \| dict[str, Any] \| None | `None` | KV events config - enables --kv-events-config with auto-allocated ports Per-mode: {"prefill": true, "decode": {"publisher": "zmq", "topic": "custom"}} Or global: true (enables for prefill+decode with defaults) | +| `mooncake_kv_store` | [MooncakeKVStoreConfig](#mooncakekvstoreconfig) \| None | `None` | Mooncake KV store - launches mooncake_master on infra node and injects MOONCAKE_MASTER env var on all workers automatically | + +### TRTLLMProtocol + +`backend.type: trtllm` + +TRTLLM protocol - implements BackendProtocol. + +| Key | Type | Default | Description | +|---|---|---|---| +| `type` | one of `'trtllm'` | `'trtllm'` | | +| `prefill_environment` | dict[str, str] | `{}` | | +| `decode_environment` | dict[str, str] | `{}` | | +| `aggregated_environment` | dict[str, str] | `{}` | | +| `prefill_extra_args` | list[str] | `[]` | Extra `trtllm-serve` CLI flags per mode, appended verbatim to the worker command (frontend.type: trtllm_serve only -- dynamo.trtllm takes a different CLI). `trtllm_config` already covers everything that belongs in the engine YAML, which is nearly everything: trtllm-serve merges that file into LlmArgs. But a few of its options configure the OpenAI SERVER layer rather than the engine and have no LlmArgs field, so no YAML key can reach them. The one that matters in practice is `--tool_parser` (a click.Choice consumed directly by the server constructor); note that its sibling `--reasoning_parser` IS forwarded into get_llm_args() and so remains settable from `trtllm_config`. backend: type: trtllm prefill_extra_args: ["--tool_parser", "glm47"] decode_extra_args: ["--tool_parser", "glm47"] | +| `decode_extra_args` | list[str] | `[]` | | +| `aggregated_extra_args` | list[str] | `[]` | | +| `trtllm_config` | [TRTLLMServerConfig](#trtllmserverconfig) \| None | `None` | | +| `served_model_name` | str \| None | `None` | The name clients must use in a request's "model" field. Defaults to the checkpoint directory name. backend: type: trtllm served_model_name: "deepseek-ai/deepseek-r1" Set it when the client cannot be told which name to ask for. agentperf takes the name as a flag, so it never needs this; the MLPerf harness has it fixed in the benchmark definition, so the server must match or every request 404s. Top-level rather than a trtllm_config key because trtllm_config is dumped straight into the engine's YAML file, and this is a launcher flag the engine does not recognise. | +| `publish_events_and_metrics` | bool | `False` | Whether dynamo.trtllm workers pass `--publish-events-and-metrics`. Enables the worker to publish KV-cache events (add/evict) + metrics, which the dynamo frontend consumes for KV-cache-aware routing (router-mode: kv). This may impact performance so should be disabled if exact KV aware routing is not needed. | +| `sequential_node_start` | int | `0` | Controls batched startup of workers that share the same node. 0 = start all workers in parallel (no constraint). 1 = fully sequential: one worker at a time, each must be ready before the next. N > 1 = start N workers simultaneously per batch, wait for all to be ready, then next batch. For trtllm_serve: readiness is an HTTP 200 on the worker's http_port. For dynamo.trtllm: readiness is a TCP connection on the worker's sys_port. | +| `numa_memory_bind` | bool \| None | `None` | Whether to prefix the trtllm worker command with `numactl -m 0,1`. None (default) preserves the existing auto-detected behavior (enabled only for gb200/gb300). True/False forces numactl on/off regardless of gpu_type. | +| `numa_cpu_bind` | bool | `False` | Optional stricter NUMA CPU affinity for the worker process, in addition to numa_memory_bind. A previous post-hoc `taskset -pc $PPID` approach (see bind-b300-prefill-cpus.sh) only pins the leader PID *after* launch, so secondary threads spawned by Python/UCX/MPI/TRT-LLM can still land cross-socket. When true, srtctl instead: 1. sets TLLM_NUMA_AWARE_WORKER_AFFINITY=0 (disables TRT-LLM's own internal NUMA thread-pinning, which fights with the OS-level mask) 2. wraps the worker command (prefill/decode/agg) in `taskset -c `, applied *before* exec so every spawned thread inherits the mask. The CPU list is discovered at runtime (configs/numa_cpu_bind.sh) from the physical GPU this task owns, not a static SLURM_LOCALID table — a static table assumes SLURM_LOCALID is a node-wide GPU ordinal, which breaks when two endpoints share a node (each gets its own srun step, so LOCALID restarts at 0 for both). | + +### VLLMProtocol + +`backend.type: vllm` + +vLLM protocol - implements BackendProtocol. + +| Key | Type | Default | Description | +|---|---|---|---| +| `type` | one of `'vllm'` | `'vllm'` | | +| `prefill_environment` | dict[str, str] | `{}` | Environment variables per mode | +| `decode_environment` | dict[str, str] | `{}` | | +| `aggregated_environment` | dict[str, str] | `{}` | | +| `vllm_config` | [VLLMServerConfig](#vllmserverconfig) \| None | `None` | vLLM server CLI config per mode | +| `set_cuda_visible_devices` | bool | `False` | Legacy device binding for vLLM builds without --device-ids. | +| `connector` | str \| None | `'nixl'` | Default KV connector: "nixl", "lmcache", or a raw JSON string for --kv-transfer-config. Can be overridden per mode by setting "connector" in vllm_config.prefill/decode/aggregated. dynamo 1.0.0+: translated to --kv-transfer-config (--connector was removed). | +| `mooncake_kv_store` | [VLLMMooncakeKVStoreConfig](#vllmmooncakekvstoreconfig) \| None | `None` | Mooncake KV store — when set, srtslurm launches mooncake_master on the infra node and auto-injects MOONCAKE_MASTER / MOONCAKE_TE_META_DATA_SERVER / MOONCAKE_LOCAL_HOSTNAME on every vLLM worker. | +| `kv_events_config` | bool \| dict[str, Any] \| None | `None` | KV events config - enables --kv-events-config with auto-allocated ports. Required for Dynamo's event-driven KV-aware routing. Global true enables defaults for prefill and decode workers. Per-mode: {"prefill": true, "decode": {"topic": "custom"}} | +| `allow_prefill_decode_colocation` | bool | `False` | Allow prefill and decode workers to share one node when the combined GPU request fits within gpus_per_node. Defaults off to preserve existing P/D node separation. | +| `allow_prefill_decode_colocation_across_nodes` | bool | `False` | Extend P/D colocation to multi-node topologies. When enabled together with allow_prefill_decode_colocation, workers are packed contiguously across the minimum number of nodes instead of reserving separate P/D node pools. Defaults off to preserve the original one-node-only policy. | +| `dp_launch_mode` | one of `'per_gpu'`, `'per_node'` | `'per_node'` | DP process layout. Per-node lets vLLM manage the node-local portion of a DP x TP x PP topology in one CUDA namespace and derives cross-node TP/PP rendezvous when a replica is larger than the node-local GPU allocation. Per-GPU remains available as a deprecated compatibility layout. | + +### MockerProtocol + +`backend.type: mocker` + +Dynamo Mocker protocol - implements BackendProtocol. + +| Key | Type | Default | Description | +|---|---|---|---| +| `type` | one of `'mocker'` | `'mocker'` | | +| `engine_type` | str | `'vllm'` | Simulation parameters | +| `speedup_ratio` | float | `100.0` | | +| `decode_speedup_ratio` | float | `1.0` | | +| `num_gpu_blocks_override` | int | `16384` | | +| `max_num_seqs` | int | `256` | | +| `max_num_batched_tokens` | int | `8192` | | +| `block_size` | int \| None | `None` | | +| `data_parallel_size` | int | `1` | | +| `num_workers` | int | `1` | | +| `startup_time` | float \| None | `None` | | +| `kv_transfer_bandwidth` | float \| None | `None` | | +| `kv_cache_dtype` | str \| None | `None` | | +| `enable_prefix_caching` | bool | `True` | | +| `enable_chunked_prefill` | bool | `True` | | +| `preemption_mode` | str \| None | `None` | | +| `prefill_environment` | dict[str, str] | `{}` | Environment variables per mode | +| `decode_environment` | dict[str, str] | `{}` | | +| `aggregated_environment` | dict[str, str] | `{}` | | +| `mocker_config` | [MockerServerConfig](#mockerserverconfig) \| None | `None` | Per-mode CLI overrides | + +### SGLangServerConfig + +SGLang server CLI configuration per mode (prefill/decode/aggregated). + +| Key | Type | Default | Description | +|---|---|---|---| +| `prefill` | dict[str, Any] \| None | `None` | | +| `decode` | dict[str, Any] \| None | `None` | | +| `aggregated` | dict[str, Any] \| None | `None` | | + +### MooncakeKVStoreConfig + +Mooncake KV store configuration. + +| Key | Type | Default | Description | +|---|---|---|---| +| `container` | str \| None | `None` | | +| `env` | dict[str, str] | `{}` | | +| `master_extra_args` | list[str] | `[]` | | + +### TRTLLMServerConfig + +SGLang server CLI configuration per mode (prefill/decode/aggregated). + +| Key | Type | Default | Description | +|---|---|---|---| +| `prefill` | dict[str, Any] \| None | `None` | | +| `decode` | dict[str, Any] \| None | `None` | | +| `aggregated` | dict[str, Any] \| None | `None` | | + +### VLLMServerConfig + +vLLM server CLI configuration per mode (prefill/decode/aggregated). + +| Key | Type | Default | Description | +|---|---|---|---| +| `prefill` | dict[str, Any] \| None | `None` | | +| `decode` | dict[str, Any] \| None | `None` | | +| `aggregated` | dict[str, Any] \| None | `None` | | + +### VLLMMooncakeKVStoreConfig + +Mooncake KV store config for the vLLM backend. + +| Key | Type | Default | Description | +|---|---|---|---| +| `container` | str \| None | `None` | | +| `env` | dict[str, str] | `{}` | | +| `master_extra_args` | list[str] | `[]` | | +| `store_config` | dict[str, Any] \| None | `None` | ``store_config`` values are JSON-serialized into MOONCAKE_CONFIG_PATH and parsed by vLLM's ``MooncakeStoreConfig`` dataclass — fields are a mix of str (e.g. ``protocol``), int (e.g. ``port``), and human-readable sizes (e.g. ``"4GB"``). Type as ``dict[str, Any]`` to avoid forcing users to quote numeric values. | + +### MockerServerConfig + +Mocker CLI configuration per mode (prefill/decode/aggregated). + +| Key | Type | Default | Description | +|---|---|---|---| +| `prefill` | dict[str, Any] \| None | `None` | | +| `decode` | dict[str, Any] \| None | `None` | | +| `aggregated` | dict[str, Any] \| None | `None` | | + +## Cluster config + +Top-level keys of `srtslurm.yaml`. Recipes inherit these defaults and resolve aliases through them. + +| Key | Type | Default | Description | +|---|---|---|---| +| `cluster` | str \| None | `None` | Cluster name for status reporting | +| `default_account` | str \| None | `None` | | +| `default_partition` | str \| None | `None` | | +| `default_time_limit` | str \| None | `None` | | +| `gpus_per_node` | int \| None | `None` | | +| `network_interface` | str \| None | `None` | | +| `use_gpus_per_node_directive` | bool | `True` | | +| `use_segment_sbatch_directive` | bool | `True` | | +| `use_exclusive_sbatch_directive` | bool | `False` | | +| `use_het_jobs` | bool | `False` | Default for ``ResourceConfig.het_jobs`` when the recipe doesn't set it. When True (and recipe doesn't override), the prefill side and decode side are submitted as two SLURM heterogeneous-job components, each with its own ``--segment``. Lets asymmetric layouts (e.g. prefill 12 + decode 10 nodes on GB200/GB300) preserve NVL72 affinity per side. | +| `default_sbatch_directives` | dict[str, str] \| None | `None` | | +| `default_health_check` | dict[str, int] \| None | `None` | | +| `srtctl_root` | str \| None | `None` | | +| `output_dir` | str \| None | `None` | Custom output directory for job logs | +| `model_paths` | dict[str, str] \| None | `None` | | +| `containers` | dict[str, str] \| None | `None` | | +| `cloud` | dict[str, str] \| None | `None` | | +| `default_mounts` | dict[str, str] \| None | `None` | Cluster-level container mounts (host_path -> container_path) Applied to all jobs on this cluster, useful for cluster-specific paths | +| `default_bash_preamble` | str \| None | `None` | Shell snippet prepended to every container srun (after env exports, before the main command). Useful for cluster-wide ulimits, e.g. ``"ulimit -n 1048576 -s unlimited -u 1048576"``. Silently dropped for sruns that bypass the bash wrapper (distroless containers). | +| `default_host_setup` | [HostSetupConfig](#hostsetupconfig) \| None | `None` | Commands run on every allocated node's bare host, outside the container, before workers start. Recipes override with their own `host_setup:` block. | +| `reporting` | [ReportingConfig](#reportingconfig) \| None | `None` | | +| `telemetry` | dict \| None | `None` | opaque dict, parsed by try_start_snapshotter | +| `nginx_raise_ulimit` | bool \| None | `None` | When set, applied to job configs that omit ``frontend.nginx_raise_ulimit``. Clusters that disallow raising nofile for nginx containers should use false. | +| `git_http_version` | str \| None | `None` | Works around intermittent git smart-HTTP/HTTP2 failures cloning github.com (stalls, or truncated responses git misreports as "could not read Username" auth-prompt failures). See git_clone_command_prefix() in core/config.py -- applied to every git clone/fetch srtctl performs. | diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 4782521ed..63945511b 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -1454,6 +1454,7 @@ def main(): srtctl monitor # Live job dashboard srtctl monitor --outputs /path/to/outputs # Dashboard with custom outputs dir srtctl view /path/to/run-output # Local ruter route-decision viewer + srtctl schema-docs [--check] # Regenerate (or verify) docs/schema-reference.md """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -1573,6 +1574,23 @@ def add_common_args(p): check_parser.add_argument("path", type=Path, help="Lockfile or output dir to check against") check_parser.add_argument("--json", action="store_true", dest="json_output", help="Output as JSON") + # Generated schema reference: srtctl schema-docs [--check] [--output PATH] + schema_docs_parser = subparsers.add_parser( + "schema-docs", + help="Regenerate docs/schema-reference.md from the config dataclasses", + ) + schema_docs_parser.add_argument( + "--check", + action="store_true", + help="Exit 1 if the checked-in reference is stale instead of rewriting it (used by CI)", + ) + schema_docs_parser.add_argument( + "--output", + type=Path, + default=None, + help="Write to this path instead of docs/schema-reference.md", + ) + args = parser.parse_args() json_mode = bool(getattr(args, "json_output", False)) @@ -1660,6 +1678,23 @@ def restore_console() -> None: restore_console() sys.exit(1 if all_results else 0) + if args.command == "schema-docs": + from srtctl.core.schema_docs import DEFAULT_OUTPUT, schema_reference_is_current, write_schema_reference + + output = args.output or DEFAULT_OUTPUT + if args.check: + if schema_reference_is_current(output): + console.print(f"[green]✓[/] {output} is up to date") + restore_console() + return + console.print(f"[bold red]✗[/] {output} is stale; run `srtctl schema-docs` and commit the result") + restore_console() + sys.exit(1) + written = write_schema_reference(output) + console.print(f"[green]✓[/] Wrote {written}") + restore_console() + return + if args.command == "monitor": from srtctl.cli.monitor import main as _monitor_main diff --git a/src/srtctl/core/schema_docs.py b/src/srtctl/core/schema_docs.py new file mode 100644 index 000000000..a08fd7dbc --- /dev/null +++ b/src/srtctl/core/schema_docs.py @@ -0,0 +1,364 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generate ``docs/schema-reference.md`` from the recipe and cluster dataclasses. + +The recipe schema is the dataclass tree rooted at :class:`SrtConfig`; the +cluster config (``srtslurm.yaml``) is :class:`ClusterConfig`. This module walks +both and renders one Markdown table per dataclass, so the field-level reference +cannot drift from the code: the checked-in file is regenerated by +``srtctl schema-docs`` and CI fails when it is stale. + +Descriptions come from two places, in priority order: an ``Attributes:`` block +in the class docstring (Google style), then the ``#`` comment block directly +above a field or the trailing comment on its line. +""" + +from __future__ import annotations + +import ast +import inspect +import re +import textwrap +import types +from collections.abc import Set as AbstractSet +from dataclasses import MISSING, Field, dataclass, fields, is_dataclass +from pathlib import Path +from typing import Any, Literal, get_args, get_origin, get_type_hints + +from srtctl.backends import MockerProtocol, SGLangProtocol, TRTLLMProtocol, VLLMProtocol +from srtctl.core.schema import ClusterConfig, SrtConfig + +DEFAULT_OUTPUT = Path(__file__).resolve().parents[3] / "docs" / "schema-reference.md" + +GENERATED_NOTICE = ( + "" +) + +# backend.type value -> dataclass. Order is the documentation order. +BACKEND_TYPES: tuple[tuple[str, type], ...] = ( + ("sglang", SGLangProtocol), + ("trtllm", TRTLLMProtocol), + ("vllm", VLLMProtocol), + ("mocker", MockerProtocol), +) +_BACKEND_CLASSES: set[type] = {cls for _, cls in BACKEND_TYPES} + + +@dataclass(frozen=True) +class FieldDoc: + """One row of a field table.""" + + key: str + type_label: str + default: str + description: str + + +# --------------------------------------------------------------------------- +# Type introspection +# --------------------------------------------------------------------------- + + +def _is_dataclass_type(obj: Any) -> bool: + return isinstance(obj, type) and is_dataclass(obj) + + +def _dataclass_targets(annotation: Any) -> list[type]: + """Dataclass types reachable from an annotation (through Annotated, unions, containers).""" + if annotation is None: + return [] + if _is_dataclass_type(annotation): + return [annotation] + origin = get_origin(annotation) + if origin is None: + return [] + found: list[type] = [] + for arg in get_args(annotation): + for target in _dataclass_targets(arg): + if target not in found: + found.append(target) + return found + + +def _type_label(annotation: Any) -> str: + """Human-readable type, with dataclasses linked to their section.""" + if annotation is None: + return "unknown" + if annotation is type(None): + return "None" + if _is_dataclass_type(annotation): + return f"[{annotation.__name__}](#{annotation.__name__.lower()})" + origin = get_origin(annotation) + if origin is None: + return getattr(annotation, "__name__", str(annotation)) + if str(origin) == "typing.Annotated": + return _type_label(get_args(annotation)[0]) + if origin is Literal: + return "one of " + ", ".join(f"`{value!r}`" for value in get_args(annotation)) + if origin is types.UnionType or str(origin) == "typing.Union": + return " \\| ".join(_type_label(arg) for arg in get_args(annotation)) + args = get_args(annotation) + origin_name = getattr(origin, "__name__", str(origin).replace("typing.", "")) + if not args: + return origin_name + if origin is tuple and len(args) == 2 and args[1] is Ellipsis: + return f"tuple[{_type_label(args[0])}, ...]" + return f"{origin_name}[{', '.join(_type_label(arg) for arg in args)}]" + + +def _yaml_key(item: Field) -> str | None: + """The key a recipe author writes, or None when the field is not user-facing.""" + marshmallow_field = item.metadata.get("marshmallow_field") + data_key = getattr(marshmallow_field, "data_key", None) if marshmallow_field is not None else None + if data_key: + return str(data_key) + if item.name.startswith("_"): + return None + return item.name + + +def _default_label(item: Field) -> str: + if item.default is not MISSING: + return f"`{item.default!r}`" + if item.default_factory is not MISSING: # type: ignore[attr-defined] + factory = item.default_factory # type: ignore[attr-defined] + if factory in (dict, list, tuple, set, frozenset): + return f"`{factory()!r}`" + name = getattr(factory, "__name__", type(factory).__name__) + return f"`{name}()`" + return "required" + + +# --------------------------------------------------------------------------- +# Descriptions from docstrings and source comments +# --------------------------------------------------------------------------- + +_ATTRIBUTE_LINE = re.compile(r"^\s{1,8}([A-Za-z_]\w*)\s*(?:\([^)]*\))?:\s*(.*)$") + + +def _docstring_attributes(cls: type) -> dict[str, str]: + """Parse a Google-style ``Attributes:`` block into {field: description}.""" + doc = inspect.getdoc(cls) or "" + match = re.search(r"^Attributes:\s*$", doc, re.MULTILINE) + if match is None: + return {} + descriptions: dict[str, str] = {} + current: str | None = None + for line in doc[match.end() :].splitlines(): + if not line.strip(): + continue + indent = len(line) - len(line.lstrip()) + if indent == 0: + break + attribute = _ATTRIBUTE_LINE.match(line) + if attribute and indent <= 4: + current = attribute.group(1) + descriptions[current] = attribute.group(2).strip() + elif current: + descriptions[current] = f"{descriptions[current]} {line.strip()}".strip() + return descriptions + + +def _comment_descriptions(cls: type) -> dict[str, str]: + """Comment block above each field, or the trailing comment on its line.""" + try: + source = textwrap.dedent(inspect.getsource(cls)) + except (OSError, TypeError): + return {} + try: + class_node = ast.parse(source).body[0] + except (SyntaxError, IndexError): + return {} + if not isinstance(class_node, ast.ClassDef): + return {} + lines = source.splitlines() + descriptions: dict[str, str] = {} + for node in class_node.body: + if not isinstance(node, ast.AnnAssign) or not isinstance(node.target, ast.Name): + continue + leading: list[str] = [] + index = node.lineno - 2 + while index >= 0 and lines[index].strip().startswith("#"): + leading.insert(0, lines[index].strip().lstrip("#").strip()) + index -= 1 + text = " ".join(part for part in leading if part) + if not text and node.end_lineno is not None and node.end_col_offset is not None: + tail = lines[node.end_lineno - 1][node.end_col_offset :] + if "#" in tail: + text = tail.split("#", 1)[1].strip() + if text: + descriptions[node.target.id] = text + return descriptions + + +def _descriptions(cls: type) -> dict[str, str]: + merged = _comment_descriptions(cls) + merged.update(_docstring_attributes(cls)) + return merged + + +def _class_summary(cls: type) -> str: + doc = inspect.getdoc(cls) or "" + first = doc.split("\n\n", 1)[0].strip() + return " ".join(first.split()) + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def _cell(text: str) -> str: + return " ".join(text.split()).replace("|", "\\|") + + +def field_docs(cls: Any) -> list[FieldDoc]: + """Field table rows for one dataclass, in declaration order.""" + hints = get_type_hints(cls, include_extras=True) + descriptions = _descriptions(cls) + rows: list[FieldDoc] = [] + for item in fields(cls): + key = _yaml_key(item) + if key is None: + continue + annotation = hints.get(item.name, item.type) + rows.append( + FieldDoc( + key=key, + type_label=_type_label(annotation), + default=_default_label(item), + description=descriptions.get(item.name, ""), + ) + ) + return rows + + +def _render_table(cls: type) -> list[str]: + out = ["| Key | Type | Default | Description |", "|---|---|---|---|"] + for row in field_docs(cls): + out.append(f"| `{row.key}` | {row.type_label} | {row.default} | {_cell(row.description)} |") + return out + + +def _nested_types(cls: Any) -> list[type]: + hints = get_type_hints(cls, include_extras=True) + found: list[type] = [] + for item in fields(cls): + if _yaml_key(item) is None: + continue + for target in _dataclass_targets(hints.get(item.name, item.type)): + if target not in found: + found.append(target) + return found + + +def _walk(root: type, skip: AbstractSet[type]) -> list[type]: + """Breadth-first list of dataclasses reachable from ``root`` (root excluded).""" + order: list[type] = [] + queue = [root] + seen = {root} + while queue: + current = queue.pop(0) + for nested in _nested_types(current): + if nested in seen or nested in skip: + continue + seen.add(nested) + order.append(nested) + queue.append(nested) + return order + + +def _render_class_section(cls: type, level: int) -> list[str]: + heading = "#" * level + lines = [f"{heading} {cls.__name__}", ""] + summary = _class_summary(cls) + if summary: + lines.extend([summary, ""]) + lines.extend(_render_table(cls)) + lines.append("") + return lines + + +def render_schema_reference() -> str: + """Render the full Markdown reference for recipes and the cluster config.""" + lines: list[str] = [ + "# Schema Reference", + "", + GENERATED_NOTICE, + "", + ( + "Field-level reference for recipe YAML (`SrtConfig`) and the cluster config " + "`srtslurm.yaml` (`ClusterConfig`), generated from the dataclasses in " + "`srtctl.core.schema` and `srtctl.backends`. Each table lists the YAML key, " + "the type, the default (`required` when there is none), and a description " + "taken from the class docstring or the comment on the field. Nested types " + "link to their own table. For prose, examples, and semantics see " + "[config-reference.md](config-reference.md)." + ), + "", + "## Recipe", + "", + "Top-level keys of a recipe YAML.", + "", + ] + lines.extend(_render_table(SrtConfig)) + lines.append("") + + nested = _walk(SrtConfig, skip=_BACKEND_CLASSES) + if nested: + lines.extend(["## Recipe sections", ""]) + for cls in nested: + lines.extend(_render_class_section(cls, level=3)) + + lines.extend( + [ + "## Backend types", + "", + "`backend.type` selects one of the following; the remaining `backend` keys are that type's fields.", + "", + ] + ) + backend_nested: list[type] = [] + for type_name, cls in BACKEND_TYPES: + lines.extend([f"### {cls.__name__}", "", f"`backend.type: {type_name}`", ""]) + summary = _class_summary(cls) + if summary: + lines.extend([summary, ""]) + lines.extend(_render_table(cls)) + lines.append("") + for extra in _walk(cls, skip=_BACKEND_CLASSES | set(nested)): + if extra not in backend_nested: + backend_nested.append(extra) + for cls in backend_nested: + lines.extend(_render_class_section(cls, level=3)) + + lines.extend( + [ + "## Cluster config", + "", + "Top-level keys of `srtslurm.yaml`. Recipes inherit these defaults and resolve aliases through them.", + "", + ] + ) + lines.extend(_render_table(ClusterConfig)) + lines.append("") + for cls in _walk(ClusterConfig, skip=_BACKEND_CLASSES | set(nested) | set(backend_nested)): + lines.extend(_render_class_section(cls, level=3)) + + return "\n".join(lines).rstrip() + "\n" + + +def write_schema_reference(output: Path = DEFAULT_OUTPUT) -> Path: + """Write the rendered reference to ``output`` and return the path.""" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(render_schema_reference(), encoding="utf-8") + return output + + +def schema_reference_is_current(output: Path = DEFAULT_OUTPUT) -> bool: + """True when the checked-in file matches what the code renders.""" + if not output.exists(): + return False + return output.read_text(encoding="utf-8") == render_schema_reference() diff --git a/tests/test_schema_docs.py b/tests/test_schema_docs.py new file mode 100644 index 000000000..dea28b64e --- /dev/null +++ b/tests/test_schema_docs.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from srtctl.cli import submit as submit_cli +from srtctl.core.schema import ObservabilityConfig, ResourceConfig, SrtConfig +from srtctl.core.schema_docs import ( + DEFAULT_OUTPUT, + field_docs, + render_schema_reference, + schema_reference_is_current, + write_schema_reference, +) + + +def test_checked_in_schema_reference_is_current() -> None: + """docs/schema-reference.md must be regenerated whenever the schema changes. + + Fix with: uv run srtctl schema-docs + """ + assert DEFAULT_OUTPUT.exists(), f"{DEFAULT_OUTPUT} is missing; run `srtctl schema-docs`" + assert schema_reference_is_current(), ( + f"{DEFAULT_OUTPUT.name} is stale relative to srtctl.core.schema; run `srtctl schema-docs` and commit the result" + ) + + +def test_render_is_deterministic() -> None: + assert render_schema_reference() == render_schema_reference() + + +def test_top_level_recipe_keys_are_documented() -> None: + rows = {row.key for row in field_docs(SrtConfig)} + for key in ("name", "model", "resources", "backend", "frontend", "benchmark", "observability", "host_setup"): + assert key in rows, key + + +def test_marshmallow_data_key_wins_over_private_attribute_name() -> None: + rows = {row.key: row for row in field_docs(ResourceConfig)} + assert "gpus_per_prefill" in rows + assert "gpus_per_decode" in rows + assert "_explicit_gpus_per_prefill" not in rows + assert rows["gpus_per_node"].default == "`4`" + assert rows["gpu_type"].default == "required" + + +def test_docstring_attributes_become_descriptions() -> None: + rows = {row.key: row for row in field_docs(ObservabilityConfig)} + assert "Master analytics knob" in rows["enabled"].description + + +def test_field_comments_become_descriptions() -> None: + rows = {row.key: row for row in field_docs(SrtConfig)} + assert "Custom setup script" in rows["setup_script"].description + + +def test_backend_types_and_cluster_config_are_rendered() -> None: + text = render_schema_reference() + for heading in ( + "## Recipe", + "## Backend types", + "### SGLangProtocol", + "### TRTLLMProtocol", + "### VLLMProtocol", + "### MockerProtocol", + "## Cluster config", + ): + assert heading in text, heading + assert "`backend.type: sglang`" in text + assert "`sglang_config`" in text + assert "`default_account`" in text + assert "