feat: enable Dynamo lifecycle OTel capture by default

.gitignore

@@ -52,6 +52,8 @@ configs/nats-server
 configs/etcd
 configs/etcdctl
 configs/process-exporter
+configs/otelcol-contrib
+configs/.otelcol-contrib.release
 configs/*.whl
 configs/*.deb
 configs/*.tar.gz

Makefile

@@ -3,6 +3,7 @@
 NATS_VERSION ?= v2.10.28
 ETCD_VERSION ?= v3.5.21
 PROCESS_EXPORTER_VERSION ?= 0.8.7
+OTEL_COLLECTOR_VERSION ?= 0.146.1
 LOGS_DIR ?= logs
 ARCH ?= $(shell uname -m)
 TACHOMETER_RELEASE ?= latest
@@ -127,7 +128,33 @@ cpu-power-exporter-setup:
 		$(MAKE) --no-print-directory cpu-power-exporter-download; \
 	fi
 
-setup: tachometer-scraper-download cpu-power-exporter-setup
+.PHONY: otel-collector-download
+otel-collector-download:
+	@set -eu; \
+	case "$(ARCH)" in \
+		x86_64) arch_short=amd64; file_pattern=x86-64 ;; \
+		aarch64) arch_short=arm64; file_pattern=aarch64 ;; \
+		*) echo "Unsupported architecture: $(ARCH)"; exit 1 ;; \
+	esac; \
+	marker=configs/.otelcol-contrib.release; \
+	if [ -x configs/otelcol-contrib ] && \
+	   [ "$$(cat "$$marker" 2>/dev/null || true)" = "$(OTEL_COLLECTOR_VERSION)" ] && \
+	   file configs/otelcol-contrib | grep -q "$$file_pattern"; then \
+		echo "OTel Collector $(OTEL_COLLECTOR_VERSION) already installed ($(ARCH))"; exit 0; \
+	fi; \
+	asset="otelcol-contrib_$(OTEL_COLLECTOR_VERSION)_linux_$$arch_short.tar.gz"; \
+	base_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v$(OTEL_COLLECTOR_VERSION)"; \
+	tmp_dir=$$(mktemp -d); \
+	trap 'rm -rf "$$tmp_dir"' EXIT; \
+	curl --fail --location --retry 3 "$$base_url/$$asset" --output "$$tmp_dir/$$asset"; \
+	curl --fail --location --retry 3 "$$base_url/opentelemetry-collector-releases_otelcol-contrib_checksums.txt" --output "$$tmp_dir/checksums.txt"; \
+	awk -v asset="$$asset" '$$2 == asset {print}' "$$tmp_dir/checksums.txt" > "$$tmp_dir/selected.sha256"; \
+	(cd "$$tmp_dir" && sha256sum --check selected.sha256); \
+	tar -xzf "$$tmp_dir/$$asset" -C "$$tmp_dir" otelcol-contrib; \
+	install -Dm755 "$$tmp_dir/otelcol-contrib" configs/otelcol-contrib; \
+	printf '%s' "$(OTEL_COLLECTOR_VERSION)" > "$$marker"
+
+setup: tachometer-scraper-download cpu-power-exporter-setup otel-collector-download
 	@echo "📦 Setting up configs and logs directories..."
 	@mkdir -p logs
 	@echo "🖥️  Using architecture: $(ARCH)"

docs/observability.md

@@ -4,6 +4,33 @@ The `observability:` preset (nsys, OTEL, Tachometer) and the `telemetry:` provid
 
 ## observability
 
+**Dynamo request-lifecycle OTel tracing is on by default**, including when
+`observability.enabled: false` or the entire block is omitted. It applies only to
+the Dynamo stack (`frontend.type: dynamo`). No `enable_otel`, `otel_endpoint`, or
+`DYN_LIFECYCLE_TRACE_ENABLED` recipe setting is needed.
+
+The implied `otel-collector` service starts on the head node before workers,
+receives OTLP/gRPC on port 4317, and saves `<log_dir>/otel/traces.jsonl` and
+`logs.jsonl`. Dynamo frontends and workers receive `DYN_LIFECYCLE_TRACE_ENABLED=1`,
+`OTEL_EXPORT_ENABLED=1`, the head-node OTLP endpoint, and a component service name.
+The collector stops after the serving processes so they can flush spans. This
+uses Dynamo's native `dynamo.request_lifecycle` instrumentation (Dynamo PR
+[#14101](https://github.com/ai-dynamo/dynamo/pull/14101)); the installed Dynamo build
+must include it. The default lifecycle mode is Dynamo's `core` mode.
+
+`make setup ARCH=<compute_arch>` installs the pinned, checksum-verified
+`otelcol-contrib` host binary. A missing binary fails startup with a setup hint.
+An air-gapped installation can set `services[name: otel-collector].options.binary`
+to a shared executable; `options.port` changes both its listener and the automatic
+endpoint. `srtctl dry-run` shows the implied collector regardless of the analytics
+level. [DSight](dsight.md) discovers the saved OTLP JSONL when building the report.
+
+Explicit recipe environment still wins. The existing `enable_otel: true` plus
+`otel_endpoint` redirects traces to that external endpoint, so those traces are
+not saved locally. To opt out of automatic collection, declare the
+`otel-collector` service with `enabled: false`; this also removes its default
+lifecycle/export environment. Neither option changes other stacks' defaults.
+
 Tachometer collection is **on by default for every run** (no configuration needed; `observability.tachometer.enabled: false` opts out). `observability.enabled` turns on the server metrics *content* (the TRT-LLM publish flag and engine statistics) and the trace surfaces:
 
 ```yaml

docs/schema-reference.md

@@ -269,13 +269,13 @@ Health check configuration.
 
 ### ObservabilityConfig
 
-Observability configuration for OTEL tracing.
+Optional analytics capture and external OTEL export.
 
 | Key | Type | Default | Description |
 |---|---|---|---|
-| `enabled` | bool | `False` | Master analytics knob. Default: False. |
-| `enable_otel` | bool | `False` | If True, inject OTEL environment variables into all workers and frontends. Requires otel_endpoint to be set. Default: False. |
-| `otel_endpoint` | str \| None | `None` | OTEL collector endpoint (e.g. "http://10.0.0.1:4317"). Required when enable_otel is True. |
+| `enabled` | bool | `False` | Additional analytics (including nsys); does not gate Dynamo lifecycle tracing or Tachometer. Default: False. |
+| `enable_otel` | bool | `False` | Export traces to an explicit endpoint and enable JSONL logs. Not needed for default Dynamo lifecycle capture. Requires otel_endpoint to be set. Default: False. |
+| `otel_endpoint` | str \| None | `None` | Explicit OTEL trace endpoint (e.g. "http://10.0.0.1:4317"). Used with enable_otel; overrides Dynamo's local trace destination. |
 | `tachometer` | [TachometerConfig](#tachometerconfig) | `TachometerConfig()` | Native Tachometer capture configuration. Collects on every run, independent of ``enabled``, unless ``tachometer.enabled: false`` (see :class:`TachometerConfig`). |
 | `nsys` | [NsysObservabilityConfig](#nsysobservabilityconfig) | `NsysObservabilityConfig()` | Automatic Nsight Systems capture, enabled with the master switch. |
 

docs/schema/recipe.schema.json

@@ -1521,12 +1521,12 @@
       "properties": {
         "enabled": {
           "type": "boolean",
-          "description": "Master analytics knob. Default: False.",
+          "description": "Additional analytics (including nsys); does not gate Dynamo lifecycle tracing or Tachometer. Default: False.",
           "default": false
         },
         "enable_otel": {
           "type": "boolean",
-          "description": "If True, inject OTEL environment variables into all workers and frontends. Requires otel_endpoint to be set. Default: False.",
+          "description": "Export traces to an explicit endpoint and enable JSONL logs. Not needed for default Dynamo lifecycle capture. Requires otel_endpoint to be set. Default: False.",
           "default": false
         },
         "otel_endpoint": {
@@ -1538,7 +1538,7 @@
               "type": "null"
             }
           ],
-          "description": "OTEL collector endpoint (e.g. \"http://10.0.0.1:4317\"). Required when enable_otel is True.",
+          "description": "Explicit OTEL trace endpoint (e.g. \"http://10.0.0.1:4317\"). Used with enable_otel; overrides Dynamo's local trace destination.",
           "default": null
         },
         "tachometer": {
@@ -1550,7 +1550,7 @@
           "description": "Automatic Nsight Systems capture, enabled with the master switch."
         }
       },
-      "description": "Observability configuration for OTEL tracing."
+      "description": "Optional analytics capture and external OTEL export."
     },
     "OutputConfig": {
       "type": "object",

docs/services.md

@@ -126,6 +126,13 @@ services:
 
 ## Implicit Services
 
+Dynamo stacks also imply a critical, host-native `otel-collector` on the head
+node, independent of `observability.enabled`. It starts `before_workers`, stays
+alive through their shutdown, and writes `otel/traces.jsonl` and `otel/logs.jsonl`
+under the log directory. `make setup` installs its binary; `options.binary` and
+`options.port` override the executable and OTLP/gRPC port (4317). See
+[Dynamo lifecycle tracing](observability.md#observability).
+
 Things the recipe asks for elsewhere are services the job runs without an entry; the Mooncake master is the one built-in kind that is always declared. NATS is not implied by the Dynamo frontend alone: the default request plane is `tcp` and KV events travel over direct ZMQ, so a plain Dynamo job runs etcd only. Declare a `nats` service to run one regardless.
 
 | Implied by | Services | Where |
@@ -297,6 +304,7 @@ environment its process needs; the launch path is shared by every kind. Register
 | Type | Default command | Start | Critical | Notes |
 | --- | --- | --- | --- | --- |
 | `generic` | none (required) | `after_frontend` | `false` | Launches exactly what you wrote. |
+| `otel-collector` | `configs/otelcol-contrib --config <log_dir>/otel-collector.yaml` on the bare node | `before_workers` | `true` | Implied only by Dynamo, independent of observability. Head node, OTLP/gRPC port 4317. `options`: `binary`, `port`. Writes DSight-compatible OTLP JSONL and stops after workers. |
 | `etcd` | `/configs/etcd` from the job container, advertising the node's IP | `infra` | `true` | Implied by the Dynamo frontend. Placement `head`, `infra`, or `dedicated`; supports `external`. Fresh data dir on node-local `/tmp` each job. |
 | `nats` | `/configs/nats-server -js` from the job container | `infra` | `true` | Implied by the Dynamo frontend. `options.max_payload_mb` writes a server config. Same placements as etcd; supports `external`. |
 | `mooncake-master` | `mooncake_master` with the RPC, HTTP metadata, and metrics ports srtctl owns | `before_workers` | `true` | Declared by name; see [Mooncake KV Store](mooncake-kv-store.md). `args` are appended; `options.store_config` is the vLLM connector JSON. Container falls back to the job container. Supports `dedicated` and `external`. |

examples/sglang/dynamo-agg.yaml

@@ -6,6 +6,9 @@
 # container already ships it.
 #
 # Aliases (`qwen3-0.6b`, `sglang`) resolve through srtslurm.yaml; see examples/README.md.
+# Dynamo lifecycle OTel capture is automatic, even with observability.enabled: false.
+# Use a Dynamo build containing PR #14101 for native lifecycle spans; make setup
+# installs the local collector. No OTEL endpoint or lifecycle environment is needed.
 
 schema: 2
 name: "qwen3-0.6b-sglang-dynamo-agg"

src/srtctl/cli/mixins/worker_stage.py

@@ -25,6 +25,7 @@
 from srtctl.core.supervisor import EndpointKey, WorkerSupervisor
 from srtctl.frontends import get_frontend
 from srtctl.services.implicit import discovery_env
+from srtctl.services.otel import lifecycle_otel_env
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -199,6 +200,7 @@ def _worker_environment_defaults(self, process: "Process") -> dict[str, str]:
         env = {"HEAD_NODE_IP": self.runtime.head_node_ip}
         if get_frontend(self.config.frontend.type).worker_launch == "dynamo":
             env.update(discovery_env(self.config, self.runtime))
+            env.update(lifecycle_otel_env(self.config.services, self.runtime.head_node_ip, process.endpoint_mode))
             env.update(
                 DYN_SYSTEM_PORT=str(process.sys_port),
                 DYN_REQUEST_PLANE=self.config.dynamo.request_plane,

src/srtctl/cli/submit.py

@@ -554,7 +554,9 @@ def show_config_details(config: SrtConfig) -> None:
                 continue
             console.print(f"    [yellow]command:[/] {shlex.join(service.preview_command())}", crop=False)
             container = service.container or get_service_kind(service.type).container_fallback(config)
-            console.print(f"    [yellow]container:[/] {container or '<job container>'}")
+            launch_kind = get_service_kind(service.type)
+            container_label = "<host-native>" if launch_kind.host_native(service) else container or "<job container>"
+            console.print(f"    [yellow]container:[/] {container_label}")
             if service.source is not None:
                 console.print(f"    [yellow]source:[/] {service.source.git} @ {service.source.rev}", crop=False)
                 if service.source.path:

src/srtctl/core/schema.py

@@ -1833,7 +1833,11 @@ def terminate_timeout(self) -> int:
 
 @dataclass(frozen=True)
 class ObservabilityConfig:
-    """Observability configuration for OTEL tracing.
+    """Optional analytics capture and external OTEL export.
+
+    Dynamo stacks always enable native request-lifecycle OTEL tracing to a
+    managed local collector, independent of ``enabled`` and ``enable_otel``.
+    Other frontend stacks do not receive that collector or its environment.
 
     When enable_otel is True, OTEL environment variables (DYN_LOGGING_JSONL,
     OTEL_EXPORT_ENABLED, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_SERVICE_NAME)
@@ -1876,11 +1880,13 @@ class ObservabilityConfig:
     ``BenchmarkStageMixin``.)
 
     Attributes:
-        enabled: Master analytics knob. Default: False.
-        enable_otel: If True, inject OTEL environment variables into all workers
-            and frontends. Requires otel_endpoint to be set. Default: False.
-        otel_endpoint: OTEL collector endpoint (e.g. "http://10.0.0.1:4317").
-            Required when enable_otel is True.
+        enabled: Additional analytics (including nsys); does not gate Dynamo
+            lifecycle tracing or Tachometer. Default: False.
+        enable_otel: Export traces to an explicit endpoint and enable JSONL logs.
+            Not needed for default Dynamo lifecycle capture. Requires
+            otel_endpoint to be set. Default: False.
+        otel_endpoint: Explicit OTEL trace endpoint (e.g. "http://10.0.0.1:4317").
+            Used with enable_otel; overrides Dynamo's local trace destination.
         nsys: Automatic Nsight Systems capture, enabled with the master switch.
         tachometer: Native Tachometer capture configuration. Collects on every run,
             independent of ``enabled``, unless ``tachometer.enabled: false`` (see

src/srtctl/frontends/dynamo.py

@@ -30,6 +30,7 @@
     infra_placement,
     nats_implied_reasons,
 )
+from srtctl.services.otel import OTEL_SERVICE_NAME, lifecycle_otel_env
 
 if TYPE_CHECKING:
     from srtctl.backends.base import WorkerMode
@@ -131,19 +132,25 @@ def worker_ready_port(self, process: "Process") -> int:
         return process.sys_port
 
     def implied_services(self, config: Any) -> list[EffectiveService]:
-        """The discovery plane: etcd always, NATS only when a Dynamo plane rides on it.
+        """Discovery and lifecycle tracing, independent of the observability preset.
 
         The default request plane is tcp and KV events go over direct ZMQ, so a
-        plain Dynamo job runs etcd alone; a recipe declares a ``nats`` service to
-        force one anyway.
+        plain Dynamo job needs only etcd for discovery; a recipe declares a
+        ``nats`` service to force one anyway. The collector is independent of
+        the discovery planes and the observability preset.
         """
         placement = infra_placement(config)
         implied = [
             EffectiveService(
                 ServiceConfig(name=ETCD_SERVICE_NAME, type="etcd", placement=placement),
                 implicit=True,
                 reason="frontend.type dynamo",
-            )
+            ),
+            EffectiveService(
+                ServiceConfig(name=OTEL_SERVICE_NAME, type="otel-collector"),
+                implicit=True,
+                reason="Dynamo lifecycle tracing (independent of observability.enabled)",
+            ),
         ]
         nats_reasons = nats_implied_reasons(config)
         if nats_reasons:
@@ -241,6 +248,7 @@ def start_frontends(
                 env_to_set["DYN_EVENT_PLANE"] = config.dynamo.event_plane
 
             # Add OTEL env vars (before frontend env so OTEL_SERVICE_NAME can be overridden)
+            env_to_set.update(lifecycle_otel_env(config.services, runtime.head_node_ip, "frontend"))
             env_to_set.update(build_otel_env(config.observability, "frontend"))
             env_to_set.update(nsys_env)
 

src/srtctl/mock.py

@@ -303,6 +303,8 @@ def _fake_post(url, json=None, timeout=None, **_kwargs):
         ("srtctl.frontends.trtllm_serve.wait_for_health", _fake_wait_for_port),
         # Service readiness probes (etcd, NATS, exporters, declared services).
         ("srtctl.cli.mixins.service_stage.wait_until_ready", _fake_wait_for_port),
+        # The mock records the collector launch without needing a downloaded binary.
+        ("srtctl.services.otel.resolve_host_binary", lambda binary: Path(binary)),
         # The ray kind resolves the head IP itself and gates on the dashboard's node summary.
         ("srtctl.services.ray.get_hostname_ip", _fake_hostname_ip),
         ("srtctl.services.ray.RayService.wait_fleet_ready", lambda *_args, **_kwargs: None),

src/srtctl/ports.py

@@ -13,6 +13,7 @@
 ETCD_CLIENT_PORT = 2379
 ETCD_PEER_PORT = 2380
 NATS_PORT = 4222
+OTEL_GRPC_PORT = 4317
 
 # Frontend service ports.
 FRONTEND_PUBLIC_PORT = 8000

src/srtctl/services/__init__.py

@@ -13,6 +13,7 @@
     lmcache_server,
     mooncake_master,
     mooncake_store,
+    otel,
     ray,
 )
 from srtctl.services.config import (
@@ -56,6 +57,7 @@
     "lmcache_server",
     "mooncake_master",
     "mooncake_store",
+    "otel",
     "ray",
     "register_service",
 ]