From 5cd8d5b155113df202090ac4b64ad652e3323ae2 Mon Sep 17 00:00:00 2001 From: Yuewei Na Date: Mon, 12 Oct 2026 02:58:00 -0700 Subject: [PATCH 1/2] feat: capture Dynamo lifecycle OTel traces by default Signed-off-by: Yuewei Na --- .gitignore | 2 + Makefile | 29 ++++- docs/observability.md | 27 +++++ docs/schema-reference.md | 8 +- docs/schema/recipe.schema.json | 8 +- docs/services.md | 8 ++ examples/sglang/dynamo-agg.yaml | 3 + src/srtctl/cli/mixins/worker_stage.py | 2 + src/srtctl/cli/submit.py | 4 +- src/srtctl/core/schema.py | 18 ++- src/srtctl/frontends/dynamo.py | 16 ++- src/srtctl/mock.py | 2 + src/srtctl/ports.py | 1 + src/srtctl/services/__init__.py | 2 + src/srtctl/services/otel.py | 107 ++++++++++++++++++ src/srtctl/services/registry.py | 2 + .../launch/features__dynamo-source.txt | 23 +++- .../launch/features__infra-services.txt | 24 +++- .../features__lmcache-server-disagg.txt | 23 +++- .../launch/features__lmcache-server.txt | 23 +++- .../launch/features__mlperf-client.txt | 23 +++- .../launch/features__observability-vllm.txt | 19 +++- ...es__sglang-sidecar-multinode-disagg-dp.txt | 32 +++++- .../features__sglang-sidecar-multinode-dp.txt | 24 +++- .../launch/features__vllm-failover.txt | 31 ++++- .../launch/features__worker-restart.txt | 23 +++- tests/snapshots/launch/mocker__dynamo-agg.txt | 19 +++- tests/snapshots/launch/sglang__dynamo-agg.txt | 23 +++- .../launch/sglang__dynamo-disagg.txt | 23 +++- .../launch/tokenspeed__dynamo-agg.txt | 23 +++- .../launch/tokenspeed__dynamo-disagg.txt | 23 +++- tests/snapshots/launch/trtllm__dynamo-agg.txt | 23 +++- .../trtllm__dynamo-disagg-multinode.txt | 23 +++- .../launch/trtllm__dynamo-disagg.txt | 23 +++- tests/snapshots/launch/vllm__dynamo-agg.txt | 23 +++- .../snapshots/launch/vllm__dynamo-disagg.txt | 23 +++- tests/test_benchmarks.py | 2 + tests/test_dry_run.py | 10 ++ tests/test_failover.py | 7 +- tests/test_frontends.py | 9 ++ tests/test_observability_nsys.py | 1 + tests/test_schema_docs.py | 2 +- tests/test_services.py | 93 +++++++++++++-- tests/test_slurm.py | 3 + 44 files changed, 787 insertions(+), 50 deletions(-) create mode 100644 src/srtctl/services/otel.py diff --git a/.gitignore b/.gitignore index 3f6078bc7..844612478 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,8 @@ configs/nats-server configs/etcd configs/etcdctl configs/process-exporter +configs/otelcol-contrib +configs/.otelcol-contrib.release configs/*.whl configs/*.deb configs/*.tar.gz diff --git a/Makefile b/Makefile index 8250f25d0..b2aefb1e5 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ NATS_VERSION ?= v2.10.28 ETCD_VERSION ?= v3.5.21 PROCESS_EXPORTER_VERSION ?= 0.8.7 +OTEL_COLLECTOR_VERSION ?= 0.146.1 LOGS_DIR ?= logs ARCH ?= $(shell uname -m) TACHOMETER_RELEASE ?= latest @@ -127,7 +128,33 @@ cpu-power-exporter-setup: $(MAKE) --no-print-directory cpu-power-exporter-download; \ fi -setup: tachometer-scraper-download cpu-power-exporter-setup +.PHONY: otel-collector-download +otel-collector-download: + @set -eu; \ + case "$(ARCH)" in \ + x86_64) arch_short=amd64; file_pattern=x86-64 ;; \ + aarch64) arch_short=arm64; file_pattern=aarch64 ;; \ + *) echo "Unsupported architecture: $(ARCH)"; exit 1 ;; \ + esac; \ + marker=configs/.otelcol-contrib.release; \ + if [ -x configs/otelcol-contrib ] && \ + [ "$$(cat "$$marker" 2>/dev/null || true)" = "$(OTEL_COLLECTOR_VERSION)" ] && \ + file configs/otelcol-contrib | grep -q "$$file_pattern"; then \ + echo "OTel Collector $(OTEL_COLLECTOR_VERSION) already installed ($(ARCH))"; exit 0; \ + fi; \ + asset="otelcol-contrib_$(OTEL_COLLECTOR_VERSION)_linux_$$arch_short.tar.gz"; \ + base_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v$(OTEL_COLLECTOR_VERSION)"; \ + tmp_dir=$$(mktemp -d); \ + trap 'rm -rf "$$tmp_dir"' EXIT; \ + curl --fail --location --retry 3 "$$base_url/$$asset" --output "$$tmp_dir/$$asset"; \ + curl --fail --location --retry 3 "$$base_url/opentelemetry-collector-releases_otelcol-contrib_checksums.txt" --output "$$tmp_dir/checksums.txt"; \ + awk -v asset="$$asset" '$$2 == asset {print}' "$$tmp_dir/checksums.txt" > "$$tmp_dir/selected.sha256"; \ + (cd "$$tmp_dir" && sha256sum --check selected.sha256); \ + tar -xzf "$$tmp_dir/$$asset" -C "$$tmp_dir" otelcol-contrib; \ + install -Dm755 "$$tmp_dir/otelcol-contrib" configs/otelcol-contrib; \ + printf '%s' "$(OTEL_COLLECTOR_VERSION)" > "$$marker" + +setup: tachometer-scraper-download cpu-power-exporter-setup otel-collector-download @echo "📦 Setting up configs and logs directories..." @mkdir -p logs @echo "🖥️ Using architecture: $(ARCH)" diff --git a/docs/observability.md b/docs/observability.md index 5b567e9d9..2f58f207a 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -4,6 +4,33 @@ The `observability:` preset (nsys, OTEL, Tachometer) and the `telemetry:` provid ## observability +**Dynamo request-lifecycle OTel tracing is on by default**, including when +`observability.enabled: false` or the entire block is omitted. It applies only to +the Dynamo stack (`frontend.type: dynamo`). No `enable_otel`, `otel_endpoint`, or +`DYN_LIFECYCLE_TRACE_ENABLED` recipe setting is needed. + +The implied `otel-collector` service starts on the head node before workers, +receives OTLP/gRPC on port 4317, and saves `/otel/traces.jsonl` and +`logs.jsonl`. Dynamo frontends and workers receive `DYN_LIFECYCLE_TRACE_ENABLED=1`, +`OTEL_EXPORT_ENABLED=1`, the head-node OTLP endpoint, and a component service name. +The collector stops after the serving processes so they can flush spans. This +uses Dynamo's native `dynamo.request_lifecycle` instrumentation (Dynamo PR +[#14101](https://github.com/ai-dynamo/dynamo/pull/14101)); the installed Dynamo build +must include it. The default lifecycle mode is Dynamo's `core` mode. + +`make setup ARCH=` installs the pinned, checksum-verified +`otelcol-contrib` host binary. A missing binary fails startup with a setup hint. +An air-gapped installation can set `services[name: otel-collector].options.binary` +to a shared executable; `options.port` changes both its listener and the automatic +endpoint. `srtctl dry-run` shows the implied collector regardless of the analytics +level. [DSight](dsight.md) discovers the saved OTLP JSONL when building the report. + +Explicit recipe environment still wins. The existing `enable_otel: true` plus +`otel_endpoint` redirects traces to that external endpoint, so those traces are +not saved locally. To opt out of automatic collection, declare the +`otel-collector` service with `enabled: false`; this also removes its default +lifecycle/export environment. Neither option changes other stacks' defaults. + Tachometer collection is **on by default for every run** (no configuration needed; `observability.tachometer.enabled: false` opts out). `observability.enabled` turns on the server metrics *content* (the TRT-LLM publish flag and engine statistics) and the trace surfaces: ```yaml diff --git a/docs/schema-reference.md b/docs/schema-reference.md index 45781d6ce..1518fbcf1 100644 --- a/docs/schema-reference.md +++ b/docs/schema-reference.md @@ -269,13 +269,13 @@ Health check configuration. ### ObservabilityConfig -Observability configuration for OTEL tracing. +Optional analytics capture and external OTEL export. | Key | Type | Default | Description | |---|---|---|---| -| `enabled` | bool | `False` | Master analytics knob. Default: False. | -| `enable_otel` | bool | `False` | If True, inject OTEL environment variables into all workers and frontends. Requires otel_endpoint to be set. Default: False. | -| `otel_endpoint` | str \| None | `None` | OTEL collector endpoint (e.g. "http://10.0.0.1:4317"). Required when enable_otel is True. | +| `enabled` | bool | `False` | Additional analytics (including nsys); does not gate Dynamo lifecycle tracing or Tachometer. Default: False. | +| `enable_otel` | bool | `False` | Export traces to an explicit endpoint and enable JSONL logs. Not needed for default Dynamo lifecycle capture. Requires otel_endpoint to be set. Default: False. | +| `otel_endpoint` | str \| None | `None` | Explicit OTEL trace endpoint (e.g. "http://10.0.0.1:4317"). Used with enable_otel; overrides Dynamo's local trace destination. | | `tachometer` | [TachometerConfig](#tachometerconfig) | `TachometerConfig()` | Native Tachometer capture configuration. Collects on every run, independent of ``enabled``, unless ``tachometer.enabled: false`` (see :class:`TachometerConfig`). | | `nsys` | [NsysObservabilityConfig](#nsysobservabilityconfig) | `NsysObservabilityConfig()` | Automatic Nsight Systems capture, enabled with the master switch. | diff --git a/docs/schema/recipe.schema.json b/docs/schema/recipe.schema.json index e12ed1756..a3f168db8 100644 --- a/docs/schema/recipe.schema.json +++ b/docs/schema/recipe.schema.json @@ -1521,12 +1521,12 @@ "properties": { "enabled": { "type": "boolean", - "description": "Master analytics knob. Default: False.", + "description": "Additional analytics (including nsys); does not gate Dynamo lifecycle tracing or Tachometer. Default: False.", "default": false }, "enable_otel": { "type": "boolean", - "description": "If True, inject OTEL environment variables into all workers and frontends. Requires otel_endpoint to be set. Default: False.", + "description": "Export traces to an explicit endpoint and enable JSONL logs. Not needed for default Dynamo lifecycle capture. Requires otel_endpoint to be set. Default: False.", "default": false }, "otel_endpoint": { @@ -1538,7 +1538,7 @@ "type": "null" } ], - "description": "OTEL collector endpoint (e.g. \"http://10.0.0.1:4317\"). Required when enable_otel is True.", + "description": "Explicit OTEL trace endpoint (e.g. \"http://10.0.0.1:4317\"). Used with enable_otel; overrides Dynamo's local trace destination.", "default": null }, "tachometer": { @@ -1550,7 +1550,7 @@ "description": "Automatic Nsight Systems capture, enabled with the master switch." } }, - "description": "Observability configuration for OTEL tracing." + "description": "Optional analytics capture and external OTEL export." }, "OutputConfig": { "type": "object", diff --git a/docs/services.md b/docs/services.md index 0105ba292..edcb578ac 100644 --- a/docs/services.md +++ b/docs/services.md @@ -126,6 +126,13 @@ services: ## Implicit Services +Dynamo stacks also imply a critical, host-native `otel-collector` on the head +node, independent of `observability.enabled`. It starts `before_workers`, stays +alive through their shutdown, and writes `otel/traces.jsonl` and `otel/logs.jsonl` +under the log directory. `make setup` installs its binary; `options.binary` and +`options.port` override the executable and OTLP/gRPC port (4317). See +[Dynamo lifecycle tracing](observability.md#observability). + Things the recipe asks for elsewhere are services the job runs without an entry; the Mooncake master is the one built-in kind that is always declared. NATS is not implied by the Dynamo frontend alone: the default request plane is `tcp` and KV events travel over direct ZMQ, so a plain Dynamo job runs etcd only. Declare a `nats` service to run one regardless. | Implied by | Services | Where | @@ -297,6 +304,7 @@ environment its process needs; the launch path is shared by every kind. Register | Type | Default command | Start | Critical | Notes | | --- | --- | --- | --- | --- | | `generic` | none (required) | `after_frontend` | `false` | Launches exactly what you wrote. | +| `otel-collector` | `configs/otelcol-contrib --config /otel-collector.yaml` on the bare node | `before_workers` | `true` | Implied only by Dynamo, independent of observability. Head node, OTLP/gRPC port 4317. `options`: `binary`, `port`. Writes DSight-compatible OTLP JSONL and stops after workers. | | `etcd` | `/configs/etcd` from the job container, advertising the node's IP | `infra` | `true` | Implied by the Dynamo frontend. Placement `head`, `infra`, or `dedicated`; supports `external`. Fresh data dir on node-local `/tmp` each job. | | `nats` | `/configs/nats-server -js` from the job container | `infra` | `true` | Implied by the Dynamo frontend. `options.max_payload_mb` writes a server config. Same placements as etcd; supports `external`. | | `mooncake-master` | `mooncake_master` with the RPC, HTTP metadata, and metrics ports srtctl owns | `before_workers` | `true` | Declared by name; see [Mooncake KV Store](mooncake-kv-store.md). `args` are appended; `options.store_config` is the vLLM connector JSON. Container falls back to the job container. Supports `dedicated` and `external`. | diff --git a/examples/sglang/dynamo-agg.yaml b/examples/sglang/dynamo-agg.yaml index 2ae273ab2..86f1c9bd1 100644 --- a/examples/sglang/dynamo-agg.yaml +++ b/examples/sglang/dynamo-agg.yaml @@ -6,6 +6,9 @@ # container already ships it. # # Aliases (`qwen3-0.6b`, `sglang`) resolve through srtslurm.yaml; see examples/README.md. +# Dynamo lifecycle OTel capture is automatic, even with observability.enabled: false. +# Use a Dynamo build containing PR #14101 for native lifecycle spans; make setup +# installs the local collector. No OTEL endpoint or lifecycle environment is needed. schema: 2 name: "qwen3-0.6b-sglang-dynamo-agg" diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index d3c13be5b..bf1cce8fa 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -21,6 +21,7 @@ from srtctl.core.observability_nsys import wrap_observability_nsys from srtctl.core.processes import ManagedProcess, NamedProcesses, ProcessRegistry from srtctl.core.schema import build_otel_env, installs_dynamo +from srtctl.services.otel import lifecycle_otel_env from srtctl.core.slurm import CONTAINER_REMAP_ROOT_EXPORT, get_hostname_ip, start_srun_process from srtctl.core.supervisor import EndpointKey, WorkerSupervisor from srtctl.frontends import get_frontend @@ -199,6 +200,7 @@ def _worker_environment_defaults(self, process: "Process") -> dict[str, str]: env = {"HEAD_NODE_IP": self.runtime.head_node_ip} if get_frontend(self.config.frontend.type).worker_launch == "dynamo": env.update(discovery_env(self.config, self.runtime)) + env.update(lifecycle_otel_env(self.config.services, self.runtime.head_node_ip, process.endpoint_mode)) env.update( DYN_SYSTEM_PORT=str(process.sys_port), DYN_REQUEST_PLANE=self.config.dynamo.request_plane, diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 1d263c1e8..4708e2b2d 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -554,7 +554,9 @@ def show_config_details(config: SrtConfig) -> None: continue console.print(f" [yellow]command:[/] {shlex.join(service.preview_command())}", crop=False) container = service.container or get_service_kind(service.type).container_fallback(config) - console.print(f" [yellow]container:[/] {container or ''}") + launch_kind = get_service_kind(service.type) + container_label = "" if launch_kind.host_native(service) else container or "" + console.print(f" [yellow]container:[/] {container_label}") if service.source is not None: console.print(f" [yellow]source:[/] {service.source.git} @ {service.source.rev}", crop=False) if service.source.path: diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 317bc6786..6534bf97a 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1833,7 +1833,11 @@ def terminate_timeout(self) -> int: @dataclass(frozen=True) class ObservabilityConfig: - """Observability configuration for OTEL tracing. + """Optional analytics capture and external OTEL export. + + Dynamo stacks always enable native request-lifecycle OTEL tracing to a + managed local collector, independent of ``enabled`` and ``enable_otel``. + Other frontend stacks do not receive that collector or its environment. When enable_otel is True, OTEL environment variables (DYN_LOGGING_JSONL, OTEL_EXPORT_ENABLED, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_SERVICE_NAME) @@ -1876,11 +1880,13 @@ class ObservabilityConfig: ``BenchmarkStageMixin``.) Attributes: - enabled: Master analytics knob. Default: False. - enable_otel: If True, inject OTEL environment variables into all workers - and frontends. Requires otel_endpoint to be set. Default: False. - otel_endpoint: OTEL collector endpoint (e.g. "http://10.0.0.1:4317"). - Required when enable_otel is True. + enabled: Additional analytics (including nsys); does not gate Dynamo + lifecycle tracing or Tachometer. Default: False. + enable_otel: Export traces to an explicit endpoint and enable JSONL logs. + Not needed for default Dynamo lifecycle capture. Requires + otel_endpoint to be set. Default: False. + otel_endpoint: Explicit OTEL trace endpoint (e.g. "http://10.0.0.1:4317"). + Used with enable_otel; overrides Dynamo's local trace destination. nsys: Automatic Nsight Systems capture, enabled with the master switch. tachometer: Native Tachometer capture configuration. Collects on every run, independent of ``enabled``, unless ``tachometer.enabled: false`` (see diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py index 07dd42124..d574f3c19 100644 --- a/src/srtctl/frontends/dynamo.py +++ b/src/srtctl/frontends/dynamo.py @@ -30,6 +30,7 @@ infra_placement, nats_implied_reasons, ) +from srtctl.services.otel import OTEL_SERVICE_NAME, lifecycle_otel_env if TYPE_CHECKING: from srtctl.backends.base import WorkerMode @@ -131,11 +132,12 @@ def worker_ready_port(self, process: "Process") -> int: return process.sys_port def implied_services(self, config: Any) -> list[EffectiveService]: - """The discovery plane: etcd always, NATS only when a Dynamo plane rides on it. + """Discovery and lifecycle tracing, independent of the observability preset. The default request plane is tcp and KV events go over direct ZMQ, so a - plain Dynamo job runs etcd alone; a recipe declares a ``nats`` service to - force one anyway. + plain Dynamo job needs only etcd for discovery; a recipe declares a + ``nats`` service to force one anyway. The collector is independent of + the discovery planes and the observability preset. """ placement = infra_placement(config) implied = [ @@ -143,7 +145,12 @@ def implied_services(self, config: Any) -> list[EffectiveService]: ServiceConfig(name=ETCD_SERVICE_NAME, type="etcd", placement=placement), implicit=True, reason="frontend.type dynamo", - ) + ), + EffectiveService( + ServiceConfig(name=OTEL_SERVICE_NAME, type="otel-collector"), + implicit=True, + reason="Dynamo lifecycle tracing (independent of observability.enabled)", + ), ] nats_reasons = nats_implied_reasons(config) if nats_reasons: @@ -241,6 +248,7 @@ def start_frontends( env_to_set["DYN_EVENT_PLANE"] = config.dynamo.event_plane # Add OTEL env vars (before frontend env so OTEL_SERVICE_NAME can be overridden) + env_to_set.update(lifecycle_otel_env(config.services, runtime.head_node_ip, "frontend")) env_to_set.update(build_otel_env(config.observability, "frontend")) env_to_set.update(nsys_env) diff --git a/src/srtctl/mock.py b/src/srtctl/mock.py index a9f06d785..b4b301446 100644 --- a/src/srtctl/mock.py +++ b/src/srtctl/mock.py @@ -303,6 +303,8 @@ def _fake_post(url, json=None, timeout=None, **_kwargs): ("srtctl.frontends.trtllm_serve.wait_for_health", _fake_wait_for_port), # Service readiness probes (etcd, NATS, exporters, declared services). ("srtctl.cli.mixins.service_stage.wait_until_ready", _fake_wait_for_port), + # The mock records the collector launch without needing a downloaded binary. + ("srtctl.services.otel.resolve_host_binary", lambda binary: Path(binary)), # The ray kind resolves the head IP itself and gates on the dashboard's node summary. ("srtctl.services.ray.get_hostname_ip", _fake_hostname_ip), ("srtctl.services.ray.RayService.wait_fleet_ready", lambda *_args, **_kwargs: None), diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index cbfde1a48..8831571cd 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -13,6 +13,7 @@ ETCD_CLIENT_PORT = 2379 ETCD_PEER_PORT = 2380 NATS_PORT = 4222 +OTEL_GRPC_PORT = 4317 # Frontend service ports. FRONTEND_PUBLIC_PORT = 8000 diff --git a/src/srtctl/services/__init__.py b/src/srtctl/services/__init__.py index 5e9c82103..b5073e319 100644 --- a/src/srtctl/services/__init__.py +++ b/src/srtctl/services/__init__.py @@ -13,6 +13,7 @@ lmcache_server, mooncake_master, mooncake_store, + otel, ray, ) from srtctl.services.config import ( @@ -56,6 +57,7 @@ "lmcache_server", "mooncake_master", "mooncake_store", + "otel", "ray", "register_service", ] diff --git a/src/srtctl/services/otel.py b/src/srtctl/services/otel.py new file mode 100644 index 000000000..b98d725be --- /dev/null +++ b/src/srtctl/services/otel.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Job-local OTLP capture for Dynamo's native request-lifecycle spans.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import yaml +from marshmallow import ValidationError + +from srtctl.ports import OTEL_GRPC_PORT +from srtctl.services.config import ServiceReadinessConfig, TcpProbe +from srtctl.services.exporters import resolve_host_binary +from srtctl.services.registry import ServiceKind, ServiceLaunchContext, register_service + +if TYPE_CHECKING: + from srtctl.core.runtime import RuntimeContext + from srtctl.core.schema import SrtConfig + from srtctl.services.config import ServiceConfig + +OTEL_SERVICE_NAME = "otel-collector" +DEFAULT_OTEL_BINARY = "configs/otelcol-contrib" +OTEL_CONFIG_NAME = "otel-collector.yaml" + + +def lifecycle_otel_env(services: Sequence[ServiceConfig], head_ip: str, component: str) -> dict[str, str]: + """Defaults for Dynamo launches; explicit recipe environment wins at the caller.""" + service = next((s for s in services if s.name == OTEL_SERVICE_NAME), None) + if service is not None and (not service.enabled or service.type != "otel-collector"): + return {} + port = _port(service) if service is not None else OTEL_GRPC_PORT + return { + "DYN_LIFECYCLE_TRACE_ENABLED": "1", + "OTEL_EXPORT_ENABLED": "1", + "OTEL_EXPORTER_OTLP_ENDPOINT": f"http://{head_ip}:{port}", + "OTEL_SERVICE_NAME": f"dynamo-{component}", + } + + +def _port(service: ServiceConfig) -> int: + return int(service.options.get("port", OTEL_GRPC_PORT)) + + +@register_service("otel-collector") +class OtelCollectorService(ServiceKind): + """Host-native collector, ready before Dynamo starts and stopped after it flushes.""" + + builds_command = True + default_start = "before_workers" + default_critical = True + option_keys = ("binary", "port") + + def validate(self, service: ServiceConfig, config: SrtConfig) -> None: + if service.effective_placement != "head" or service.effective_pool is not None or service.nodes is not None: + raise ValidationError("otel-collector runs on the head node, where Dynamo's default OTLP endpoint points") + if service.effective_start != "before_workers": + raise ValidationError("otel-collector must start before_workers to capture the full lifecycle") + if service.container is not None: + raise ValidationError("otel-collector runs host-native; set options.binary instead of container") + try: + port = _port(service) + except (TypeError, ValueError): + raise ValidationError("otel-collector options.port must be an integer between 1 and 65535") from None + if not 1 <= port <= 65535: + raise ValidationError("otel-collector options.port must be between 1 and 65535") + + def host_native(self, service: ServiceConfig) -> bool: + return True + + def prepare(self, service: ServiceConfig, runtime: RuntimeContext, config: SrtConfig) -> None: + binary = str(service.options.get("binary", DEFAULT_OTEL_BINARY)) + if resolve_host_binary(binary) is None: + raise RuntimeError(f"OTel collector binary {binary!r} not found; run make setup ARCH=") + output = runtime.log_dir / "otel" + output.mkdir(parents=True, exist_ok=True) + signals = ("traces", "logs") + collector_config = { + "receivers": {"otlp": {"protocols": {"grpc": {"endpoint": f"0.0.0.0:{_port(service)}"}}}}, + "exporters": { + f"file/{signal}": { + "path": str(output / f"{signal}.jsonl"), + "format": "json", + "append": True, + "flush_interval": "1s", + } + for signal in signals + }, + "service": { + "telemetry": {"logs": {"level": "info"}, "metrics": {"level": "none"}}, + "pipelines": { + signal: {"receivers": ["otlp"], "processors": [], "exporters": [f"file/{signal}"]} + for signal in signals + }, + }, + } + (runtime.log_dir / OTEL_CONFIG_NAME).write_text(yaml.safe_dump(collector_config, sort_keys=False)) + + def build_command(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> list[str]: + binary = str(service.options.get("binary", DEFAULT_OTEL_BINARY)) + resolved = resolve_host_binary(binary) + return [str(resolved or binary), "--config", str(ctx.runtime.log_dir / OTEL_CONFIG_NAME)] + + def readiness(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> ServiceReadinessConfig: + return ServiceReadinessConfig(tcp=TcpProbe(port=_port(service))) diff --git a/src/srtctl/services/registry.py b/src/srtctl/services/registry.py index bfdfee438..6561f3b61 100644 --- a/src/srtctl/services/registry.py +++ b/src/srtctl/services/registry.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, ClassVar, cast if TYPE_CHECKING: @@ -54,6 +55,7 @@ def preview(cls, node: str = "") -> ServiceLaunchContext: head_node_ip="", infra_node_ip="", job_id="", + log_dir=Path(""), ) return cls( runtime=cast("RuntimeContext", runtime), diff --git a/tests/snapshots/launch/features__dynamo-source.txt b/tests/snapshots/launch/features__dynamo-source.txt index af5f5e596..25f60dc89 100644 --- a/tests/snapshots/launch/features__dynamo-source.txt +++ b/tests/snapshots/launch/features__dynamo-source.txt @@ -1,6 +1,6 @@ # examples/features/dynamo-source.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -28,12 +37,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -58,12 +71,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7501 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -87,9 +104,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/features__infra-services.txt b/tests/snapshots/launch/features__infra-services.txt index 23af8ba55..d98645403 100644 --- a/tests/snapshots/launch/features__infra-services.txt +++ b/tests/snapshots/launch/features__infra-services.txt @@ -1,6 +1,6 @@ # examples/features/infra-services.yaml # exit_code: 0 -# srun calls: 11 +# srun calls: 12 ## service_etcd nodelist: node-01 @@ -34,6 +34,16 @@ bash_preamble: | command: | /configs/nats-server -c /tmp/nats.conf +## service_otel-collector +nodelist: node-02 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 + NATS_SERVER: nats://127.0.0.1:4222 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-02 nodelist: node-02 output: /logs/node-02_agg_w0.out @@ -45,6 +55,7 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -52,6 +63,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -76,6 +90,7 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -83,6 +98,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -106,10 +124,14 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/features__lmcache-server-disagg.txt b/tests/snapshots/launch/features__lmcache-server-disagg.txt index 14996a8cc..1254b693d 100644 --- a/tests/snapshots/launch/features__lmcache-server-disagg.txt +++ b/tests/snapshots/launch/features__lmcache-server-disagg.txt @@ -1,6 +1,6 @@ # examples/features/lmcache-server-disagg.yaml # exit_code: 0 -# srun calls: 13 +# srun calls: 14 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## service_lmcache nodelist: node-01 output: /logs/service_lmcache.out @@ -46,6 +55,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -53,6 +63,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5200 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONHASHSEED: 0 PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 @@ -79,6 +92,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -86,6 +100,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5201 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONHASHSEED: 0 PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 @@ -111,9 +128,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend PYTHONHASHSEED: 0 srun_export_env: ENROOT_REMAP_ROOT: yes diff --git a/tests/snapshots/launch/features__lmcache-server.txt b/tests/snapshots/launch/features__lmcache-server.txt index f6dd68e13..874a24278 100644 --- a/tests/snapshots/launch/features__lmcache-server.txt +++ b/tests/snapshots/launch/features__lmcache-server.txt @@ -1,6 +1,6 @@ # examples/features/lmcache-server.yaml # exit_code: 0 -# srun calls: 15 +# srun calls: 16 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## service_lmcache-server_node-01 nodelist: node-01 output: /logs/service_lmcache-server_node-01.out @@ -82,6 +91,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -89,6 +99,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5200 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONHASHSEED: 0 PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 @@ -115,6 +128,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -122,6 +136,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5201 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONHASHSEED: 0 PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 @@ -147,9 +164,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend PYTHONHASHSEED: 0 srun_export_env: ENROOT_REMAP_ROOT: yes diff --git a/tests/snapshots/launch/features__mlperf-client.txt b/tests/snapshots/launch/features__mlperf-client.txt index ee032f4ae..d655788c4 100644 --- a/tests/snapshots/launch/features__mlperf-client.txt +++ b/tests/snapshots/launch/features__mlperf-client.txt @@ -1,6 +1,6 @@ # examples/features/mlperf-client.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -38,6 +38,16 @@ bash_preamble: | command: | /configs/nats-server -js -sd /tmp/nats +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 + NATS_SERVER: nats://127.0.0.1:4222 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodes: 1 ntasks: 4 @@ -53,6 +63,7 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -64,6 +75,9 @@ env_to_set: NATS_SERVER: nats://127.0.0.1:4222 NCCL_CUMEM_ENABLE: 1 NCCL_MNNVL_ENABLE: 1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg TLLM_LOG_LEVEL: INFO TRTLLM_ENABLE_PDL: 1 TRTLLM_EPLB_SHM_NAME: @@ -94,10 +108,14 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 @@ -260,6 +278,9 @@ env_to_set: SRT_SERVICE_NODE_EXPORTER_IPS: 127.0.0.1,127.0.0.1,127.0.0.1 SRT_SERVICE_NODE_EXPORTER_NODES: node-01,node-02,node-03 SRT_SERVICE_NODE_EXPORTER_NODE_COUNT: 3 + SRT_SERVICE_OTEL_COLLECTOR_IPS: 127.0.0.1 + SRT_SERVICE_OTEL_COLLECTOR_NODES: node-01 + SRT_SERVICE_OTEL_COLLECTOR_NODE_COUNT: 1 SRT_SERVICE_PROCESS_EXPORTER_IPS: 127.0.0.1,127.0.0.1,127.0.0.1 SRT_SERVICE_PROCESS_EXPORTER_NODES: node-01,node-02,node-03 SRT_SERVICE_PROCESS_EXPORTER_NODE_COUNT: 3 diff --git a/tests/snapshots/launch/features__observability-vllm.txt b/tests/snapshots/launch/features__observability-vllm.txt index 88c12153f..4f763dcb6 100644 --- a/tests/snapshots/launch/features__observability-vllm.txt +++ b/tests/snapshots/launch/features__observability-vllm.txt @@ -1,6 +1,6 @@ # examples/features/observability-vllm.yaml # exit_code: 0 -# srun calls: 11 +# srun calls: 12 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -28,6 +37,7 @@ container_mounts: /logs: /logs env_to_set: DYN_ENABLE_RUST_NVTX: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: debug DYN_LOGGING_JSONL: true DYN_LOGGING_SPAN_EVENTS: true @@ -38,6 +48,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5200 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SRT_NSYS_REPORT_BARRIER_DIR: /logs/profiles/.stopped/ SRT_NSYS_REPORT_EXPECTED: 1 @@ -68,6 +81,7 @@ container_mounts: /logs: /logs env_to_set: DYN_ENABLE_RUST_NVTX: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: debug DYN_LOGGING_JSONL: true DYN_LOGGING_SPAN_EVENTS: true @@ -77,6 +91,9 @@ env_to_set: DYN_REQUEST_TRACE_FILE_PATH: /logs/dynamo-request-trace DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend SRT_NSYS_REPORT_BARRIER_DIR: /logs/profiles/.stopped/ SRT_NSYS_REPORT_EXPECTED: 1 SRT_NSYS_REPORT_STOP_TIMEOUT: 1800 diff --git a/tests/snapshots/launch/features__sglang-sidecar-multinode-disagg-dp.txt b/tests/snapshots/launch/features__sglang-sidecar-multinode-disagg-dp.txt index 887d0b89b..62deb1024 100644 --- a/tests/snapshots/launch/features__sglang-sidecar-multinode-disagg-dp.txt +++ b/tests/snapshots/launch/features__sglang-sidecar-multinode-disagg-dp.txt @@ -1,6 +1,6 @@ # examples/features/sglang-sidecar-multinode-disagg-dp.yaml # exit_code: 0 -# srun calls: 21 +# srun calls: 22 ## service_etcd nodelist: node-01 @@ -32,6 +32,16 @@ bash_preamble: | command: | /configs/nats-server -js -sd /tmp/nats +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 + NATS_SERVER: nats://127.0.0.1:4222 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## prefill_0_node-01 nodelist: node-01 output: /logs/node-01_prefill_w0.out @@ -43,6 +53,7 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -50,6 +61,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -132,6 +146,7 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -139,6 +154,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -221,6 +239,7 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -228,6 +247,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -310,6 +332,7 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -317,6 +340,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -340,10 +366,14 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/features__sglang-sidecar-multinode-dp.txt b/tests/snapshots/launch/features__sglang-sidecar-multinode-dp.txt index b5e2b53dd..5e5adc164 100644 --- a/tests/snapshots/launch/features__sglang-sidecar-multinode-dp.txt +++ b/tests/snapshots/launch/features__sglang-sidecar-multinode-dp.txt @@ -1,6 +1,6 @@ # examples/features/sglang-sidecar-multinode-dp.yaml # exit_code: 0 -# srun calls: 15 +# srun calls: 16 ## service_etcd nodelist: node-01 @@ -32,6 +32,16 @@ bash_preamble: | command: | /configs/nats-server -js -sd /tmp/nats +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 + NATS_SERVER: nats://127.0.0.1:4222 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -43,6 +53,7 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -50,6 +61,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -132,6 +146,7 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -139,6 +154,9 @@ env_to_set: ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -221,10 +239,14 @@ container_mounts: /logs: /logs env_to_set: DYN_EVENT_PLANE: nats + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: nats DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 NATS_SERVER: nats://127.0.0.1:4222 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/features__vllm-failover.txt b/tests/snapshots/launch/features__vllm-failover.txt index 3c1cfdd86..14b87f39e 100644 --- a/tests/snapshots/launch/features__vllm-failover.txt +++ b/tests/snapshots/launch/features__vllm-failover.txt @@ -1,6 +1,6 @@ # examples/features/vllm-failover.yaml # exit_code: 0 -# srun calls: 15 +# srun calls: 16 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## service_gms_agg_0_node-01 nodelist: node-01 output: /logs/service_gms_agg_0_node-01.out @@ -143,6 +152,7 @@ container_mounts: env_to_set: CUDA_VISIBLE_DEVICES: 0 DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -155,6 +165,9 @@ env_to_set: FAILOVER_LOCK_PATH: /dev/shm/srtctl-4242/agg_0/failover.lock GMS_SOCKET_DIR: /dev/shm/srtctl-4242/agg_0 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 @@ -179,6 +192,7 @@ container_mounts: env_to_set: CUDA_VISIBLE_DEVICES: 0 DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -191,6 +205,9 @@ env_to_set: FAILOVER_LOCK_PATH: /dev/shm/srtctl-4242/agg_0/failover.lock GMS_SOCKET_DIR: /dev/shm/srtctl-4242/agg_0 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 @@ -215,6 +232,7 @@ container_mounts: env_to_set: CUDA_VISIBLE_DEVICES: 1 DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -227,6 +245,9 @@ env_to_set: FAILOVER_LOCK_PATH: /dev/shm/srtctl-4242/agg_1/failover.lock GMS_SOCKET_DIR: /dev/shm/srtctl-4242/agg_1 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5402 @@ -251,6 +272,7 @@ container_mounts: env_to_set: CUDA_VISIBLE_DEVICES: 1 DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -263,6 +285,9 @@ env_to_set: FAILOVER_LOCK_PATH: /dev/shm/srtctl-4242/agg_1/failover.lock GMS_SOCKET_DIR: /dev/shm/srtctl-4242/agg_1 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5403 @@ -285,9 +310,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode round-robin diff --git a/tests/snapshots/launch/features__worker-restart.txt b/tests/snapshots/launch/features__worker-restart.txt index da044bf01..2a945a578 100644 --- a/tests/snapshots/launch/features__worker-restart.txt +++ b/tests/snapshots/launch/features__worker-restart.txt @@ -1,6 +1,6 @@ # examples/features/worker-restart.yaml # exit_code: 0 -# srun calls: 11 +# srun calls: 12 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -28,12 +37,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -58,12 +71,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7501 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -87,9 +104,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/mocker__dynamo-agg.txt b/tests/snapshots/launch/mocker__dynamo-agg.txt index 7ce69cf7e..8b5f67716 100644 --- a/tests/snapshots/launch/mocker__dynamo-agg.txt +++ b/tests/snapshots/launch/mocker__dynamo-agg.txt @@ -1,6 +1,6 @@ # examples/mocker/dynamo-agg.yaml # exit_code: 0 -# srun calls: 11 +# srun calls: 12 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -27,12 +36,16 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg srun_export_env: ENROOT_REMAP_ROOT: yes bash_preamble: | @@ -53,9 +66,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/sglang__dynamo-agg.txt b/tests/snapshots/launch/sglang__dynamo-agg.txt index a3a6f0e4a..5bcaab22c 100644 --- a/tests/snapshots/launch/sglang__dynamo-agg.txt +++ b/tests/snapshots/launch/sglang__dynamo-agg.txt @@ -1,6 +1,6 @@ # examples/sglang/dynamo-agg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -28,12 +37,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -58,12 +71,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7501 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -87,9 +104,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/sglang__dynamo-disagg.txt b/tests/snapshots/launch/sglang__dynamo-disagg.txt index e8b55eb58..afe41c891 100644 --- a/tests/snapshots/launch/sglang__dynamo-disagg.txt +++ b/tests/snapshots/launch/sglang__dynamo-disagg.txt @@ -1,6 +1,6 @@ # examples/sglang/dynamo-disagg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## prefill_0_node-01 nodelist: node-01 output: /logs/node-01_prefill_w0.out @@ -28,12 +37,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -58,12 +71,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7501 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 SGLANG_CUDA_COREDUMP_BEFORE_CRASH: 0 SGLANG_PYSPY_DUMP_BEFORE_CRASH: 0 @@ -87,9 +104,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/tokenspeed__dynamo-agg.txt b/tests/snapshots/launch/tokenspeed__dynamo-agg.txt index 3d1799677..306834cb7 100644 --- a/tests/snapshots/launch/tokenspeed__dynamo-agg.txt +++ b/tests/snapshots/launch/tokenspeed__dynamo-agg.txt @@ -1,6 +1,6 @@ # examples/tokenspeed/dynamo-agg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -28,12 +37,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 bash_preamble: | ( python3 <(cat <<'__FINGERPRINT_EOF__' @@ -54,12 +67,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7501 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 bash_preamble: | ( python3 <(cat <<'__FINGERPRINT_EOF__' @@ -79,9 +96,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/tokenspeed__dynamo-disagg.txt b/tests/snapshots/launch/tokenspeed__dynamo-disagg.txt index 5ffd73f8f..2a242b495 100644 --- a/tests/snapshots/launch/tokenspeed__dynamo-disagg.txt +++ b/tests/snapshots/launch/tokenspeed__dynamo-disagg.txt @@ -1,6 +1,6 @@ # examples/tokenspeed/dynamo-disagg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## prefill_0_node-01 nodelist: node-01 output: /logs/node-01_prefill_w0.out @@ -28,12 +37,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7500 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 TOKENSPEED_SKIP_GRPC_WARMUP: 1 bash_preamble: | @@ -55,12 +68,16 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 DYN_SYSTEM_PORT: 7501 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 TOKENSPEED_SKIP_GRPC_WARMUP: 1 bash_preamble: | @@ -81,9 +98,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/trtllm__dynamo-agg.txt b/tests/snapshots/launch/trtllm__dynamo-agg.txt index c747b6161..4b1fe9333 100644 --- a/tests/snapshots/launch/trtllm__dynamo-agg.txt +++ b/tests/snapshots/launch/trtllm__dynamo-agg.txt @@ -1,6 +1,6 @@ # examples/trtllm/dynamo-agg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodes: 1 ntasks: 1 @@ -30,6 +39,7 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -39,6 +49,9 @@ env_to_set: HEAD_NODE_IP: 127.0.0.1 MASTER_ADDR: 127.0.0.1 MASTER_PORT: 29500 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 TLLM_LOG_LEVEL: INFO TRTLLM_EPLB_SHM_NAME: @@ -69,6 +82,7 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 1 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -78,6 +92,9 @@ env_to_set: HEAD_NODE_IP: 127.0.0.1 MASTER_ADDR: 127.0.0.1 MASTER_PORT: 29501 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 TLLM_LOG_LEVEL: INFO TRTLLM_EPLB_SHM_NAME: @@ -105,9 +122,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/trtllm__dynamo-disagg-multinode.txt b/tests/snapshots/launch/trtllm__dynamo-disagg-multinode.txt index 47d89bd5b..53d6f4d73 100644 --- a/tests/snapshots/launch/trtllm__dynamo-disagg-multinode.txt +++ b/tests/snapshots/launch/trtllm__dynamo-disagg-multinode.txt @@ -1,6 +1,6 @@ # examples/trtllm/dynamo-disagg-multinode.yaml # exit_code: 0 -# srun calls: 16 +# srun calls: 17 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## prefill_0_node-01 nodes: 1 ntasks: 8 @@ -29,6 +38,7 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -37,6 +47,9 @@ env_to_set: HEAD_NODE_IP: 127.0.0.1 MASTER_ADDR: 127.0.0.1 MASTER_PORT: 29500 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 TLLM_LOG_LEVEL: INFO TRTLLM_EPLB_SHM_NAME: @@ -66,6 +79,7 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -74,6 +88,9 @@ env_to_set: HEAD_NODE_IP: 127.0.0.1 MASTER_ADDR: 127.0.0.1 MASTER_PORT: 29501 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 TLLM_LOG_LEVEL: INFO TRTLLM_EPLB_SHM_NAME: @@ -102,9 +119,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/trtllm__dynamo-disagg.txt b/tests/snapshots/launch/trtllm__dynamo-disagg.txt index 3771b0d80..eb73c296e 100644 --- a/tests/snapshots/launch/trtllm__dynamo-disagg.txt +++ b/tests/snapshots/launch/trtllm__dynamo-disagg.txt @@ -1,6 +1,6 @@ # examples/trtllm/dynamo-disagg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## prefill_0_node-01 nodes: 1 ntasks: 1 @@ -30,6 +39,7 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -38,6 +48,9 @@ env_to_set: HEAD_NODE_IP: 127.0.0.1 MASTER_ADDR: 127.0.0.1 MASTER_PORT: 29500 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 TLLM_LOG_LEVEL: INFO TRTLLM_EPLB_SHM_NAME: @@ -68,6 +81,7 @@ container_mounts: /logs: /logs env_to_set: CUDA_VISIBLE_DEVICES: 0 + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -76,6 +90,9 @@ env_to_set: HEAD_NODE_IP: 127.0.0.1 MASTER_ADDR: 127.0.0.1 MASTER_PORT: 29501 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 TLLM_LOG_LEVEL: INFO TRTLLM_EPLB_SHM_NAME: @@ -103,9 +120,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend mpi: pmix command: | python3 -m dynamo.frontend --http-port=8000 --router-mode kv diff --git a/tests/snapshots/launch/vllm__dynamo-agg.txt b/tests/snapshots/launch/vllm__dynamo-agg.txt index a40d033a8..05e014bcf 100644 --- a/tests/snapshots/launch/vllm__dynamo-agg.txt +++ b/tests/snapshots/launch/vllm__dynamo-agg.txt @@ -1,6 +1,6 @@ # examples/vllm/dynamo-agg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## agg_0_node-01 nodelist: node-01 output: /logs/node-01_agg_w0.out @@ -28,6 +37,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -35,6 +45,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5200 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 @@ -60,6 +73,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -67,6 +81,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5201 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-agg PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 @@ -91,9 +108,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/snapshots/launch/vllm__dynamo-disagg.txt b/tests/snapshots/launch/vllm__dynamo-disagg.txt index bdd38117c..21817175c 100644 --- a/tests/snapshots/launch/vllm__dynamo-disagg.txt +++ b/tests/snapshots/launch/vllm__dynamo-disagg.txt @@ -1,6 +1,6 @@ # examples/vllm/dynamo-disagg.yaml # exit_code: 0 -# srun calls: 12 +# srun calls: 13 ## service_etcd nodelist: node-01 @@ -17,6 +17,15 @@ bash_preamble: | command: | /configs/etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379 +## service_otel-collector +nodelist: node-01 +output: /logs/service_otel-collector.out +env_to_set: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: True +command: | + configs/otelcol-contrib --config /logs/otel-collector.yaml + ## prefill_0_node-01 nodelist: node-01 output: /logs/node-01_prefill_w0.out @@ -28,6 +37,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -35,6 +45,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5200 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-prefill PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 @@ -60,6 +73,7 @@ container_mounts: /logs: /logs env_to_set: DYN_HEALTH_CHECK_ENABLED: false + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 @@ -67,6 +81,9 @@ env_to_set: DYN_VLLM_KV_EVENT_PORT: 5201 ETCD_ENDPOINTS: http://127.0.0.1:2379 HEAD_NODE_IP: 127.0.0.1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-decode PYTHONUNBUFFERED: 1 VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 @@ -91,9 +108,13 @@ container_mounts: : /model /logs: /logs env_to_set: + DYN_LIFECYCLE_TRACE_ENABLED: 1 DYN_REQUEST_PLANE: tcp DYN_SKIP_SGLANG_LOG_FORMATTING: 1 ETCD_ENDPOINTS: http://127.0.0.1:2379 + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4317 + OTEL_EXPORT_ENABLED: 1 + OTEL_SERVICE_NAME: dynamo-frontend srun_export_env: ENROOT_REMAP_ROOT: yes mpi: pmix diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 0341c442b..0b475337a 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -1822,6 +1822,7 @@ def _make_orchestrator(): from srtctl.cli.do_sweep import SweepOrchestrator from srtctl.core.runtime import Nodes, RuntimeContext + from srtctl.services.config import ServiceConfig from srtctl.core.schema import ( BenchmarkConfig, FrontendConfig, @@ -1840,6 +1841,7 @@ def _make_orchestrator(): # These tests exercise the eval flow, not telemetry; opt out of the # default-on Tachometer so run() needs no scraper mocks. observability=ObservabilityConfig(tachometer=TachometerConfig(enabled=False)), + services=[ServiceConfig(name="otel-collector", type="otel-collector", enabled=False)], resources=ResourceConfig(gpu_type="h100", gpus_per_node=8), roles={"prefill": RoleConfig(nodes=1, workers=1), "decode": RoleConfig(nodes=2, workers=2)}, benchmark=BenchmarkConfig(type="sa-bench", isl=1024, osl=1024, concurrencies="128x256x512"), diff --git a/tests/test_dry_run.py b/tests/test_dry_run.py index b7f7df978..64364f999 100644 --- a/tests/test_dry_run.py +++ b/tests/test_dry_run.py @@ -88,6 +88,16 @@ def test_default_restart_policy_keeps_dry_run_unchanged(capsys): assert "restart=" not in capsys.readouterr().out +@pytest.mark.parametrize("enabled", [False, True]) +def test_dynamo_lifecycle_collector_visible_independently_of_observability(capsys, enabled): + show_config_details(_make_config({"observability": {"enabled": enabled}})) + output = " ".join(capsys.readouterr().out.split()) + assert "otel-collector" in output + assert "Dynamo lifecycle tracing (independent of observability.enabled)" in output + assert "otel-collector.yaml" in output + assert "" in output + + def test_role_engines_images_and_environments_are_visible(capsys): data = yaml.safe_load(Path("examples/vllm/vllm-router-disagg.yaml").read_text()) engine = data.pop("engine") diff --git a/tests/test_failover.py b/tests/test_failover.py index e5f115ff3..f11d1dbe2 100644 --- a/tests/test_failover.py +++ b/tests/test_failover.py @@ -379,7 +379,10 @@ def _orchestrator(config: SrtConfig, tmp_path: Path) -> SweepOrchestrator: def test_gms_service_launches_one_pinned_instance_per_worker(tmp_path: Path) -> None: - orchestrator = _orchestrator(_load(), tmp_path) + # Isolate the GMS launch from the independent default lifecycle collector. + orchestrator = _orchestrator( + _load(services=[{"name": "otel-collector", "type": "otel-collector", "enabled": False}]), tmp_path + ) with ( patch(SRUN_SERVICE, return_value=_proc()) as srun, patch(WAIT_SERVICE, return_value=True) as wait, @@ -438,7 +441,7 @@ def test_worker_stage_launches_only_engines(tmp_path: Path) -> None: def test_full_node_worker_pins_nothing(tmp_path: Path) -> None: - data = _data() + data = _data(services=[{"name": "otel-collector", "type": "otel-collector", "enabled": False}]) data["roles"]["agg"] = {"nodes": 1, "workers": 1, "gpus": 8, "args": {"tensor-parallel-size": 8}} orchestrator = _orchestrator(_load_data(data), tmp_path) with ( diff --git a/tests/test_frontends.py b/tests/test_frontends.py index 67b8ba686..403f0760e 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -310,6 +310,7 @@ def test_frontend_metrics_port_and_implied_services(self): implied = get_frontend("dynamo").implied_services(dynamo_config) assert [(entry.service.name, entry.service.type, entry.reason) for entry in implied] == [ ("etcd", "etcd", "frontend.type dynamo"), + ("otel-collector", "otel-collector", "Dynamo lifecycle tracing (independent of observability.enabled)"), ("nats", "nats", "dynamo.request_plane nats"), ] for frontend_type in ("vllm", "sglang", "sglang-router", "trtllm_serve", "vllm-router"): @@ -909,6 +910,7 @@ def test_dynamo_frontend_prefixed_when_enabled(self): log_dir=Path("/logs"), nodes=SimpleNamespace(infra="infra-node", het_group_for=lambda node: None), infra_node_ip="10.0.0.9", + head_node_ip="10.0.0.8", container_image=Path("/container.sqsh"), container_mounts={}, environment={}, @@ -916,6 +918,7 @@ def test_dynamo_frontend_prefixed_when_enabled(self): config = SimpleNamespace( frontend=SimpleNamespace(args=None, env=None, worker_selection=None, numa_bind=True), observability=ObservabilityConfig(), + services=[], dynamo=SimpleNamespace( install=False, get_install_commands=lambda: "", request_plane="tcp", event_plane=None ), @@ -944,6 +947,7 @@ def test_dynamo_frontend_is_a_named_step_with_a_drain_timeout(): log_dir=Path("/logs"), nodes=SimpleNamespace(infra="infra-node", het_group_for=lambda node: None), infra_node_ip="10.0.0.9", + head_node_ip="10.0.0.8", container_image=Path("/container.sqsh"), container_mounts={}, environment={}, @@ -951,6 +955,7 @@ def test_dynamo_frontend_is_a_named_step_with_a_drain_timeout(): config = SimpleNamespace( frontend=SimpleNamespace(args=None, env=None), observability=ObservabilityConfig(), + services=[], dynamo=SimpleNamespace(install=False, get_install_commands=lambda: "", request_plane="tcp", event_plane=None), setup_script=None, ) @@ -970,6 +975,7 @@ def _dynamo_frontend_call(*, dynamo_install: bool, event_plane: str | None = "zm log_dir=Path("/logs"), nodes=SimpleNamespace(infra="infra-node", het_group_for=lambda node: None), infra_node_ip="10.0.0.9", + head_node_ip="10.0.0.8", container_image=Path("/container.sqsh"), container_mounts={}, environment={}, @@ -977,6 +983,7 @@ def _dynamo_frontend_call(*, dynamo_install: bool, event_plane: str | None = "zm config = SimpleNamespace( frontend=SimpleNamespace(type="dynamo", args=None, env=None, worker_selection=None), observability=ObservabilityConfig(), + services=[], dynamo=SimpleNamespace( install=dynamo_install, get_install_commands=lambda: "echo install-dynamo", @@ -1028,6 +1035,7 @@ def test_dynamo_frontend_materializes_inline_worker_selection(tmp_path): runtime = SimpleNamespace( log_dir=tmp_path, infra_node_ip="10.0.0.9", + head_node_ip="10.0.0.8", nodes=SimpleNamespace(infra="infra-node", het_group_for=lambda node: None), container_image=Path("/container.sqsh"), container_mounts={tmp_path: Path("/logs")}, @@ -1053,6 +1061,7 @@ def test_dynamo_frontend_materializes_inline_worker_selection(tmp_path): type="dynamo", args={"router-mode": "kv"}, env=None, worker_selection=worker_selection ), observability=ObservabilityConfig(), + services=[], dynamo=SimpleNamespace( install=False, get_install_commands=lambda: "", diff --git a/tests/test_observability_nsys.py b/tests/test_observability_nsys.py index 20622203e..af1dec587 100644 --- a/tests/test_observability_nsys.py +++ b/tests/test_observability_nsys.py @@ -144,6 +144,7 @@ def test_every_dynamo_frontend_is_wrapped_and_gets_shutdown_budget(tmp_path, ena log_dir=tmp_path, nodes=SimpleNamespace(infra="head", het_group_for=lambda node: None), infra_node_ip="10.0.0.9", + head_node_ip="10.0.0.8", container_image=Path("/container.sqsh"), container_mounts={}, environment={}, diff --git a/tests/test_schema_docs.py b/tests/test_schema_docs.py index 6a978e0a6..a3fa32fdd 100644 --- a/tests/test_schema_docs.py +++ b/tests/test_schema_docs.py @@ -249,7 +249,7 @@ def test_required_field_renders_as_required() -> None: def test_docstring_attributes_become_descriptions() -> None: rows = {row.key: row for row in field_docs(ObservabilityConfig)} - assert "Master analytics knob" in rows["enabled"].description + assert "does not gate Dynamo lifecycle tracing" in rows["enabled"].description def test_field_comments_become_descriptions() -> None: diff --git a/tests/test_services.py b/tests/test_services.py index a3fe6a856..4caaecc8a 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -118,6 +118,7 @@ def test_registered_kinds() -> None: "mooncake-store", "nats", "node-exporter", + "otel-collector", "process-exporter", "ray", ] @@ -246,7 +247,9 @@ def _orchestrator(config: SrtConfig, tmp_path: Path) -> SweepOrchestrator: def test_no_matching_services_is_a_noop(tmp_path: Path) -> None: - orchestrator = _orchestrator(_load("services:\n - name: a\n command: [/bin/true]\n"), tmp_path) + orchestrator = _orchestrator( + _load("frontend:\n type: sglang-router\nservices:\n - name: a\n command: [/bin/true]\n"), tmp_path + ) with patch(SRUN) as srun: assert orchestrator.start_services("before_workers") == [] srun.assert_not_called() @@ -456,6 +459,9 @@ def test_clone_and_build_steps_are_registered_and_bounded(tmp_path: Path) -> Non STORES = """ services: + - name: otel-collector + type: otel-collector + enabled: false - name: store-prefill type: mooncake-store placement: @@ -560,7 +566,8 @@ def test_colocated_roles_with_same_port_rejected_before_launch(tmp_path: Path) - def test_workers_placement_deduplicates_shared_nodes(tmp_path: Path) -> None: config = _load( "services:\n - name: store\n type: mooncake-store\n placement:\n node: workers\n" - " readiness:\n port: 8800\n", + " readiness:\n port: 8800\n" + " - name: otel-collector\n type: otel-collector\n enabled: false\n", head=MOONCAKE_HEAD, engine=MOONCAKE_ENGINE, ) @@ -583,6 +590,78 @@ def test_service_config_direct_construction() -> None: ServiceConfig(name="x", command=["python", ""]) +@pytest.mark.parametrize("enabled", [False, True]) +def test_dynamo_lifecycle_defaults_reach_frontend_and_workers(tmp_path, enabled): + from types import SimpleNamespace + + from srtctl.frontends.dynamo import DynamoFrontend + + head = DISAGG_HEAD.replace("observability:\n", f"observability:\n enabled: {str(enabled).lower()}\n") + config = _load("", head=head) + orchestrator = _orchestrator(config, tmp_path) + runtime = orchestrator.runtime + topology = SimpleNamespace(frontend_nodes=["node0"], frontend_port=8180) + with patch("srtctl.frontends.dynamo.start_srun_process", return_value=_proc()) as launch: + DynamoFrontend().start_frontends(topology, runtime, config, config.backend, []) + envs = [launch.call_args.kwargs["env_to_set"]] + for mode in ("prefill", "decode"): + envs.append(orchestrator._worker_environment_defaults(SimpleNamespace(endpoint_mode=mode, sys_port=8081))) + for env, component in zip(envs, ("frontend", "prefill", "decode"), strict=True): + assert env["DYN_LIFECYCLE_TRACE_ENABLED"] == "1" + assert env["OTEL_EXPORT_ENABLED"] == "1" + assert env["OTEL_EXPORTER_OTLP_ENDPOINT"] == "http://10.0.0.10:4317" + assert env["OTEL_SERVICE_NAME"] == f"dynamo-{component}" + + +def test_direct_stack_does_not_receive_lifecycle_defaults(tmp_path): + from types import SimpleNamespace + + config = _load("frontend:\n type: sglang-router\n") + env = _orchestrator(config, tmp_path)._worker_environment_defaults( + SimpleNamespace(endpoint_mode="decode", sys_port=8081) + ) + assert not any("OTEL" in key or "LIFECYCLE" in key for key in env) + assert "otel-collector" not in dict(_names(config)) + + +def test_otel_collector_launch_writes_dsight_input_and_survives_worker_shutdown(tmp_path): + config = _load("services:\n - name: otel-collector\n type: otel-collector\n options:\n port: 14317\n") + with ( + patch("srtctl.services.otel.resolve_host_binary", return_value=Path("/srt/configs/otelcol-contrib")), + patch(SRUN, return_value=_proc()) as launch, + patch(HOST_IP, return_value="10.0.0.10"), + patch(WAIT, return_value=True) as wait, + ): + (proc,) = _orchestrator(config, tmp_path).start_services("before_workers") + assert proc.critical and proc.shutdown_tier == 1 + assert launch.call_args.kwargs["container_image"] is None + assert launch.call_args.kwargs["command"] == [ + "/srt/configs/otelcol-contrib", "--config", str(tmp_path / "otel-collector.yaml") + ] + assert wait.call_args.args[0].port == 14317 + collector = yaml.safe_load((tmp_path / "otel-collector.yaml").read_text()) + assert collector["exporters"]["file/traces"]["path"] == str(tmp_path / "otel/traces.jsonl") + assert collector["exporters"]["file/traces"]["flush_interval"] == "1s" + assert collector["receivers"]["otlp"]["protocols"]["grpc"]["endpoint"] == "0.0.0.0:14317" + from srtctl.services.otel import lifecycle_otel_env + + assert lifecycle_otel_env(config.services, "10.0.0.10", "decode")["OTEL_EXPORTER_OTLP_ENDPOINT"].endswith(":14317") + + +def test_otel_collector_missing_binary_fails_with_setup_hint(tmp_path): + with patch("srtctl.services.otel.resolve_host_binary", return_value=None): + with pytest.raises(RuntimeError, match="make setup ARCH"): + _orchestrator(_load(""), tmp_path).start_services("before_workers") + + +def test_explicit_otel_service_opt_out_removes_default_environment(): + from srtctl.services.otel import lifecycle_otel_env + + config = _load("services:\n - name: otel-collector\n type: otel-collector\n enabled: false\n") + assert "otel-collector" not in dict(_names(config)) + assert lifecycle_otel_env(config.services, "10.0.0.10", "frontend") == {} + + # --- implicit services ------------------------------------------------------------ @@ -599,12 +678,12 @@ def _names(config: SrtConfig) -> list[tuple[str, bool]]: def test_dynamo_frontend_implies_etcd_and_nats_on_the_infra_node(tmp_path: Path) -> None: config = _load("") # frontend defaults to dynamo; request plane defaults to tcp - assert _names(config) == [("etcd", True)] + assert _names(config) == [("etcd", True), ("otel-collector", True)] assert uses_discovery_plane(config) # NATS is implied only when a plane rides on it. for extra in ("dynamo:\n request_plane: nats\n", "dynamo:\n event_plane: nats\n"): with_nats = _load(extra) - assert _names(with_nats) == [("etcd", True), ("nats", True)] + assert _names(with_nats) == [("etcd", True), ("otel-collector", True), ("nats", True)] reasons = {e.service.name: e.reason for e in effective_services(with_nats)} assert reasons["nats"].startswith("dynamo.") @@ -653,7 +732,7 @@ def test_static_frontend_implies_no_discovery_plane(tmp_path: Path) -> None: def test_declared_etcd_takes_over_and_external_is_not_launched(tmp_path: Path) -> None: config = _load("services:\n - name: etcd\n type: etcd\n external: http://etcd.shared:2379\n") - assert _names(config) == [("etcd", False)] + assert _names(config) == [("etcd", False), ("otel-collector", True)] orchestrator = _orchestrator(config, tmp_path) with ( patch(SRUN, return_value=_proc()) as srun, @@ -668,7 +747,7 @@ def test_declared_etcd_takes_over_and_external_is_not_launched(tmp_path: Path) - def test_enabled_false_drops_an_implicit_service() -> None: config = _load("services:\n - name: nats\n type: nats\n enabled: false\n") - assert _names(config) == [("etcd", True)] + assert _names(config) == [("etcd", True), ("otel-collector", True)] def test_nats_max_payload_renders_a_config_file(tmp_path: Path) -> None: @@ -696,7 +775,7 @@ def test_declared_infra_services_drive_placement_and_payload(tmp_path: Path) -> ) assert config.infra_dedicated_node is True assert config.nats_max_payload_mb == 24 - assert [entry.service.effective_placement for entry in effective_services(config)] == ["dedicated", "dedicated"] + assert [entry.service.effective_placement for entry in effective_services(config)] == ["dedicated", "head", "dedicated"] def test_declared_etcd_dedicated_must_agree_with_nats(tmp_path: Path) -> None: diff --git a/tests/test_slurm.py b/tests/test_slurm.py index 2bf440437..fbdc46581 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -200,6 +200,7 @@ def test_worker_stage_wraps_nonfatal_fingerprint_hook(tmp_path: Path) -> None: frontend=SimpleNamespace(type="sglang"), dynamo=SimpleNamespace(install=False, sidecar=False, request_plane="nats", event_plane="zmq"), observability=ObservabilityConfig(), + services=[], profiling=SimpleNamespace(enabled=False, is_nsys=False), resources=ResourceConfig(), topology=Topology(roles={}, gpus_per_node=4), @@ -274,6 +275,7 @@ def _remap_worker_mixin(tmp_path: Path, *, frontend_type: str, dynamo_install: b event_plane="zmq", ), observability=ObservabilityConfig(), + services=[], profiling=SimpleNamespace(enabled=False, is_nsys=False), resources=ResourceConfig(), topology=Topology(roles={}, gpus_per_node=4), @@ -855,6 +857,7 @@ def test_worker_stage_unsets_vllm_port_for_multinode_endpoint(tmp_path: Path) -> frontend=SimpleNamespace(type="sglang"), dynamo=SimpleNamespace(install=False, sidecar=False, request_plane="nats", event_plane=None), observability=ObservabilityConfig(), + services=[], profiling=SimpleNamespace(enabled=False, is_nsys=False), resources=ResourceConfig(), topology=Topology(roles={}, gpus_per_node=4), From ff6743fab447899f254a8f4d81f9f6e481599146 Mon Sep 17 00:00:00 2001 From: Yuewei Na Date: Mon, 12 Oct 2026 03:01:23 -0700 Subject: [PATCH 2/2] style: apply pre-push formatting Signed-off-by: Yuewei Na --- src/srtctl/cli/mixins/worker_stage.py | 2 +- tests/test_schema_docs.py | 6 +++--- tests/test_services.py | 10 ++++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index bf1cce8fa..cf0c5f46a 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -21,11 +21,11 @@ from srtctl.core.observability_nsys import wrap_observability_nsys from srtctl.core.processes import ManagedProcess, NamedProcesses, ProcessRegistry from srtctl.core.schema import build_otel_env, installs_dynamo -from srtctl.services.otel import lifecycle_otel_env from srtctl.core.slurm import CONTAINER_REMAP_ROOT_EXPORT, get_hostname_ip, start_srun_process from srtctl.core.supervisor import EndpointKey, WorkerSupervisor from srtctl.frontends import get_frontend from srtctl.services.implicit import discovery_env +from srtctl.services.otel import lifecycle_otel_env if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/test_schema_docs.py b/tests/test_schema_docs.py index a3fa32fdd..4817bd2cc 100644 --- a/tests/test_schema_docs.py +++ b/tests/test_schema_docs.py @@ -58,9 +58,9 @@ def test_checked_in_schema_reference_is_current() -> None: Fix with: uv run srtctl schema-docs """ assert DEFAULT_OUTPUT.exists(), f"{DEFAULT_OUTPUT} is missing; run `srtctl schema-docs`" - assert ( - schema_reference_is_current() - ), f"{DEFAULT_OUTPUT.name} is stale relative to the code; run `srtctl schema-docs` and commit the result" + assert schema_reference_is_current(), ( + f"{DEFAULT_OUTPUT.name} is stale relative to the code; run `srtctl schema-docs` and commit the result" + ) def test_render_is_deterministic() -> None: diff --git a/tests/test_services.py b/tests/test_services.py index 4caaecc8a..0ceec609b 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -636,7 +636,9 @@ def test_otel_collector_launch_writes_dsight_input_and_survives_worker_shutdown( assert proc.critical and proc.shutdown_tier == 1 assert launch.call_args.kwargs["container_image"] is None assert launch.call_args.kwargs["command"] == [ - "/srt/configs/otelcol-contrib", "--config", str(tmp_path / "otel-collector.yaml") + "/srt/configs/otelcol-contrib", + "--config", + str(tmp_path / "otel-collector.yaml"), ] assert wait.call_args.args[0].port == 14317 collector = yaml.safe_load((tmp_path / "otel-collector.yaml").read_text()) @@ -775,7 +777,11 @@ def test_declared_infra_services_drive_placement_and_payload(tmp_path: Path) -> ) assert config.infra_dedicated_node is True assert config.nats_max_payload_mb == 24 - assert [entry.service.effective_placement for entry in effective_services(config)] == ["dedicated", "head", "dedicated"] + assert [entry.service.effective_placement for entry in effective_services(config)] == [ + "dedicated", + "head", + "dedicated", + ] def test_declared_etcd_dedicated_must_agree_with_nats(tmp_path: Path) -> None: