diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index e3c77a0380..e92d0584f2 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -24,19 +24,23 @@ COPY agents/langchain-deepagents-code/generate-config.ts /opt/nemoclaw-deepagent COPY agents/langchain-deepagents-code/managed-dcode-runtime.py /opt/nemoclaw-deepagents-code/managed-dcode-runtime.py COPY agents/langchain-deepagents-code/patch-managed-deepagents-code.py /opt/nemoclaw-deepagents-code/patch-managed-deepagents-code.py COPY agents/langchain-deepagents-code/progressive_tool_disclosure.py /opt/nemoclaw-deepagents-code/progressive_tool_disclosure.py +COPY agents/langchain-deepagents-code/nemoclaw_observability.py /opt/nemoclaw-deepagents-code/nemoclaw_observability.py COPY agents/langchain-deepagents-code/validate-progressive-tool-disclosure.py /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py +COPY agents/langchain-deepagents-code/validate-observability.py /opt/nemoclaw-deepagents-code/validate-observability.py COPY agents/langchain-deepagents-code/dcode-wrapper.sh /usr/local/lib/nemoclaw/dcode-wrapper.sh COPY agents/langchain-deepagents-code/dcode-launcher.sh /usr/local/lib/nemoclaw/dcode-launcher.sh COPY agents/langchain-deepagents-code/start.sh /usr/local/bin/nemoclaw-start COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/ -RUN chmod 444 /opt/nemoclaw-deepagents-code/generate-config.ts /opt/nemoclaw-deepagents-code/managed-dcode-runtime.py /opt/nemoclaw-deepagents-code/patch-managed-deepagents-code.py /opt/nemoclaw-deepagents-code/progressive_tool_disclosure.py /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py \ +RUN chmod 444 /opt/nemoclaw-deepagents-code/generate-config.ts /opt/nemoclaw-deepagents-code/managed-dcode-runtime.py /opt/nemoclaw-deepagents-code/patch-managed-deepagents-code.py /opt/nemoclaw-deepagents-code/progressive_tool_disclosure.py /opt/nemoclaw-deepagents-code/nemoclaw_observability.py /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py /opt/nemoclaw-deepagents-code/validate-observability.py \ && chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/dcode-wrapper.sh /usr/local/lib/nemoclaw/dcode-launcher.sh \ && chmod -R a+rX /opt/nemoclaw-blueprint \ && python3 /opt/nemoclaw-deepagents-code/patch-managed-deepagents-code.py \ && install -d -m 0700 /tmp/nemoclaw-progressive-validation \ && TMPDIR=/tmp/nemoclaw-progressive-validation python3 /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py \ && rm -rf /tmp/nemoclaw-progressive-validation \ + && /opt/venv/bin/python3 -I /opt/nemoclaw-deepagents-code/validate-observability.py \ && rm -f /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py \ + && rm -f /opt/nemoclaw-deepagents-code/validate-observability.py \ && rm -f /usr/local/bin/dcode /usr/local/bin/deepagents-code /opt/venv/bin/dcode /opt/venv/bin/deepagents-code \ && install -m 0755 /usr/local/lib/nemoclaw/dcode-launcher.sh /usr/local/bin/dcode \ && install -m 0755 /usr/local/lib/nemoclaw/dcode-launcher.sh /usr/local/bin/dcode.real \ diff --git a/agents/langchain-deepagents-code/dcode-launcher.sh b/agents/langchain-deepagents-code/dcode-launcher.sh index 2e92280e62..5c1943df69 100755 --- a/agents/langchain-deepagents-code/dcode-launcher.sh +++ b/agents/langchain-deepagents-code/dcode-launcher.sh @@ -8,9 +8,26 @@ set -euo pipefail unset BASH_ENV ENV readonly MANAGED_DCODE_WRAPPER="/usr/local/lib/nemoclaw/dcode-wrapper.sh" +readonly MANAGED_OBSERVABILITY_MARKER="/tmp/nemoclaw-observability-enabled" export HOME=/sandbox export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" +# Invalid state: raw OpenShell exec processes do not inherit the sandbox +# entrypoint's environment, so an opted-in direct dcode exec can lose tracing. +# Source boundary: start.sh materializes only the credential-free enable bit; +# this launcher recovers it only from a regular, non-symlink marker. +# Source-fix constraint: NemoClaw cannot make OpenShell preserve entrypoint env. +# Regression: the proxy-launcher tests cover exact values and unsafe file types. +# Removal condition: OpenShell propagates the bit to every exec/login process. +# The marker is convenience state, not an authorization boundary; the +# host-selected network policy controls whether local OTLP egress exists. +unset NEMOCLAW_OBSERVABILITY +if [ -f "$MANAGED_OBSERVABILITY_MARKER" ] \ + && [ ! -L "$MANAGED_OBSERVABILITY_MARKER" ] \ + && [ "$(<"$MANAGED_OBSERVABILITY_MARKER")" = "1" ]; then + export NEMOCLAW_OBSERVABILITY=1 +fi + # Raw OpenShell exec processes do not inherit the entrypoint's environment or # source shell startup files. Rebuild the proxy-only dcode contract here so a # direct exec cannot retain the host seed and bypass the managed proxy for a diff --git a/agents/langchain-deepagents-code/dependency-review.md b/agents/langchain-deepagents-code/dependency-review.md index e5119986fc..4a3c232038 100644 --- a/agents/langchain-deepagents-code/dependency-review.md +++ b/agents/langchain-deepagents-code/dependency-review.md @@ -7,9 +7,9 @@ This file records the reviewed dependency baseline for the Deep Agents Code sand Update it whenever `requirements.lock` changes. - Lockfile: `agents/langchain-deepagents-code/requirements.lock` -- Lockfile SHA-256: `229efec862ec10e6b128525e95c8fb8b44cdef8285a6cee78e3a7c73af780a9b` -- Audit command: `uvx --python 3.13 pip-audit -r agents/langchain-deepagents-code/requirements.lock --progress-spinner off` -- Audit date: 2026-07-03 +- Lockfile SHA-256: `6fde7b3188137ab5669898a552d5b12c7def2560cb4c861e8ed3563d35a5bcb9` +- Audit command: `uv tool run --python 3.13 pip-audit -r agents/langchain-deepagents-code/requirements.lock --progress-spinner off` +- Audit date: 2026-07-06 - Audit result: `No known vulnerabilities found` The Dockerfile installs this lockfile with `pip3 install --require-hashes`, so this review covers the exact package versions selected for the managed image install. diff --git a/agents/langchain-deepagents-code/nemoclaw_observability.py b/agents/langchain-deepagents-code/nemoclaw_observability.py new file mode 100644 index 0000000000..0eb6b1c14a --- /dev/null +++ b/agents/langchain-deepagents-code/nemoclaw_observability.py @@ -0,0 +1,1104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Backend-neutral, bounded observability for managed Deep Agents Code.""" + +from __future__ import annotations + +import atexit +import json +import logging +import math +import os +import re +import threading +from types import TracebackType +from typing import Any +from typing import NoReturn + +_OBSERVABILITY_ENV = "NEMOCLAW_OBSERVABILITY" +_OTLP_ENDPOINT = "http://host.openshell.internal:4318/v1/traces" +_SERVICE_NAME = "nemoclaw-langchain-deepagents-code" +_SUBSCRIBER_NAME = "nemoclaw-dcode-openinference" +_GUARDRAIL_NAME = "nemoclaw-dcode-bounded-content" +_EXPORT_TIMEOUT_MILLIS = 1_000 +_REDACTED_EXCEPTION_MESSAGE = ( + "NEMOCLAW_DCODE_OPERATION_FAILED: managed operation failed (details redacted)" +) +_SCOPE_NAME_UNSAFE = re.compile(r"[^A-Za-z0-9_.:/-]+") +_CAPTURE_KEY_ACRONYM_BOUNDARY = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])") +_CAPTURE_KEY_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_CAPTURE_KEY_DELIMITER = re.compile(r"[^A-Za-z0-9]+") +_UNICODE_SURROGATE = re.compile(r"[\ud800-\udfff]") +_MAX_SCOPE_NAME_CHARS = 128 +_MAX_CAPTURE_DEPTH = 8 +_MAX_CAPTURE_ITEMS = 50 +_MAX_CAPTURE_NODES = 2_048 +_MAX_CAPTURE_STRING_CHARS = 8_000 +_MAX_CAPTURE_AGGREGATE_STRING_CHARS = 50_000 +_MAX_CAPTURE_JSON_CHARS = 50_000 +_MAX_CAPTURE_PREVIEW_CHARS = 16_000 +_MIN_RELAY_JSON_INTEGER = -(1 << 63) +_MAX_RELAY_JSON_INTEGER = (1 << 64) - 1 +_AMBIENT_OTEL_PREFIX = "OTEL_" +_REDACTED_VALUE = "" +_OUT_OF_RANGE_INTEGER = "" +_UNSAFE_RELAY_SERIALIZATION_TAGS = { + "__nv_fallback_str__", + "__nv_pickle__", +} +_RESULT_UNSET = object() +_SENSITIVE_CAPTURE_KEYS = { + "api_key", + "auth", + "authorization", + "cookie", + "credential", + "credentials", + "headers", + "password", + "proxy_authorization", + "secret", + "set_cookie", + "token", +} +_STATE_CAPTURE_KEYS = { + "__interrupt__", + "channel_values", + "checkpoint", + "checkpoint_id", + "checkpoint_ns", + "interrupt", + "interrupts", + "pending_sends", + "resume", +} + +logger = logging.getLogger(__name__) + +_lifecycle_lock = threading.RLock() + + +class _LifecycleState: + """Mutable exporter state guarded by ``_lifecycle_lock``.""" + + def __init__(self) -> None: + self.initialization_attempted = False + self.active = False + self.subscriber: Any = None + + +_lifecycle = _LifecycleState() + + +class _CaptureBudget: + """Bound aggregate traversal and repeated container expansion.""" + + def __init__(self) -> None: + self.remaining_nodes = _MAX_CAPTURE_NODES + self.remaining_string_chars = _MAX_CAPTURE_AGGREGATE_STRING_CHARS + self.seen_containers: set[int] = set() + + def claim_node(self) -> bool: + if self.remaining_nodes <= 0: + return False + self.remaining_nodes -= 1 + return True + + def claim_container(self, value: Any) -> bool: + identity = id(value) + if identity in self.seen_containers: + return False + self.seen_containers.add(identity) + return True + + +def observability_requested(env: dict[str, str] | None = None) -> bool: + """Return whether the host requested the fixed managed observability path.""" + source = os.environ if env is None else env + return source.get(_OBSERVABILITY_ENV) == "1" + + +def _safe_identifier(value: Any, fallback: str) -> str: + """Sanitize and cap identifiers at 128 characters before Relay receives them.""" + if type(value) is not str: + return fallback + normalized = _SCOPE_NAME_UNSAFE.sub("_", value[:_MAX_SCOPE_NAME_CHARS]).strip("_") + return normalized or fallback + + +def _bounded_string(value: str, budget: _CaptureBudget | None = None) -> str: + limit = min(len(value), _MAX_CAPTURE_STRING_CHARS) + if budget is not None: + limit = min(limit, budget.remaining_string_chars) + budget.remaining_string_chars -= limit + bounded = ( + value + if limit == len(value) + else f"{value[:limit]}...[truncated {len(value) - limit} chars]" + ) + # Relay's native JSON bridge requires valid UTF-8. Replace unpaired UTF-16 + # surrogates without rejecting the application value or mutating it in place. + return _UNICODE_SURROGATE.sub("\ufffd", bounded) + + +def _redact_capture_key(key: Any) -> bool: + if type(key) is not str: + return True + segmented = _CAPTURE_KEY_ACRONYM_BOUNDARY.sub("_", key.strip()) + normalized = _CAPTURE_KEY_DELIMITER.sub( + "_", _CAPTURE_KEY_CAMEL_BOUNDARY.sub("_", segmented) + ).strip("_").lower() + segments = set(normalized.split("_")) + return ( + normalized in _SENSITIVE_CAPTURE_KEYS + or normalized in _STATE_CAPTURE_KEYS + or bool( + segments + & { + "auth", + "authentication", + "authorization", + "bearer", + "cookie", + "credential", + "credentials", + "header", + "password", + "passwd", + "secret", + "token", + } + ) + or ("key" in segments and bool(segments & {"access", "api", "private", "signing"})) + or normalized.endswith("_api_key") + or normalized.endswith("_access_key") + or normalized.endswith("_headers") + or normalized.endswith("_password") + or normalized.endswith("_private_key") + or normalized.endswith("_secret") + or normalized.endswith("_token") + or normalized.startswith("checkpoint_") + ) + + +def _opaque_capture_marker(_value: Any) -> dict[str, str]: + # Keep this marker constant. Even type-name lookup can invoke attacker-owned + # metaclass behavior, and the concrete class name is not useful trace data. + return {"_omitted_type": "opaque"} + + +def _capture_jsonable( + value: Any, + *, + depth: int = 0, + budget: _CaptureBudget | None = None, +) -> Any: + """Bound arbitrary Relay values and redact credential/checkpoint-shaped keys.""" + if budget is None: + budget = _CaptureBudget() + if depth >= _MAX_CAPTURE_DEPTH: + return {"_omitted_at_depth": _MAX_CAPTURE_DEPTH} + if not budget.claim_node(): + return {"_truncated_by_budget": True} + if value is None or type(value) is bool: + return value + if type(value) is int: + if _MIN_RELAY_JSON_INTEGER <= value <= _MAX_RELAY_JSON_INTEGER: + return value + return _OUT_OF_RANGE_INTEGER + if type(value) is float: + return value if math.isfinite(value) else "" + if type(value) is str: + return _bounded_string(value, budget) + if type(value) in (bytes, bytearray): + return f"<{len(value)} bytes>" + if type(value) is dict: + # Relay's best-effort arbitrary-object codec can encode opaque values as + # base64 pickle or attacker-controlled string output before guardrails + # run. Never inspect or export either fallback representation. + if any(tag in value for tag in _UNSAFE_RELAY_SERIALIZATION_TAGS): + return _opaque_capture_marker(value) + if not budget.claim_container(value): + return {"_omitted_reference": "shared_or_cycle"} + captured: dict[str, Any] = {} + omitted_items = 0 + inspected_items = 0 + for key, item in value.items(): + if inspected_items >= _MAX_CAPTURE_ITEMS: + break + inspected_items += 1 + if type(key) is not str: + omitted_items += 1 + continue + bounded_key = _bounded_string(key, budget) + captured[bounded_key] = ( + _REDACTED_VALUE + if _redact_capture_key(key) + else _capture_jsonable(item, depth=depth + 1, budget=budget) + ) + truncated_items = len(value) - inspected_items + if truncated_items > 0: + captured["_truncated_items"] = truncated_items + if omitted_items > 0: + captured["_omitted_non_string_keys"] = omitted_items + return captured + if type(value) in (list, tuple): + if not budget.claim_container(value): + return {"_omitted_reference": "shared_or_cycle"} + captured_items: list[Any] = [] + inspected_items = 0 + for item in value: + if inspected_items >= _MAX_CAPTURE_ITEMS or budget.remaining_nodes <= 0: + break + inspected_items += 1 + captured_items.append( + _capture_jsonable(item, depth=depth + 1, budget=budget) + ) + if len(value) > inspected_items: + captured_items.append({"_truncated_items": len(value) - inspected_items}) + return captured_items + return _opaque_capture_marker(value) + + +def _finalize_capture(captured: Any, original: Any) -> Any: + try: + encoded = json.dumps( + captured, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + except Exception: # noqa: BLE001 - preserve a bounded diagnostic shape + return {"_truncated": True, **_opaque_capture_marker(original)} + if len(encoded) <= _MAX_CAPTURE_JSON_CHARS: + return captured + return { + "_truncated": True, + **_opaque_capture_marker(original), + "preview": encoded[:_MAX_CAPTURE_PREVIEW_CHARS], + } + + +def _bounded_capture(value: Any, *, budget: _CaptureBudget | None = None) -> Any: + active_budget = budget or _CaptureBudget() + return _finalize_capture( + _capture_jsonable(value, budget=active_budget), + value, + ) + + +def _bounded_llm_request(request: Any) -> Any: + """Capture the model payload without transport headers or ambient credentials.""" + import nemo_relay + + content = request.content if type(getattr(request, "content", None)) is dict else {} + model = _safe_identifier(content.get("model"), "unknown") + messages = _bounded_capture(content.get("messages", [])) + return nemo_relay.LLMRequest({}, {"messages": messages, "model": model}) + + +def _bounded_llm_response(response: Any) -> dict[str, Any]: + """Capture bounded LangChain output while preserving its observable shape.""" + captured = _bounded_capture(response) + return captured if type(captured) is dict else {"content": captured} + + +def _bounded_tool_request(_tool_name: str, args: Any) -> Any: + """Capture bounded tool arguments for the emitted event only.""" + return _bounded_capture(args) + + +def _bounded_tool_response(_tool_name: str, result: Any) -> Any: + """Capture bounded tool results for the emitted event only.""" + return _bounded_capture(result) + + +def _safe_object_attribute(value: Any, name: str, default: Any = None) -> Any: + """Read a framework-owned field without invoking an instance override.""" + try: + return object.__getattribute__(value, name) + except Exception: # noqa: BLE001 - an unreadable field is omitted from telemetry + return default + + +def _bounded_langchain_message( + message: Any, budget: _CaptureBudget +) -> dict[str, Any]: + """Project known LangChain messages without generic model serialization.""" + try: + from langchain_core.messages import AIMessage + from langchain_core.messages import ChatMessage + from langchain_core.messages import FunctionMessage + from langchain_core.messages import HumanMessage + from langchain_core.messages import SystemMessage + from langchain_core.messages import ToolMessage + except Exception: # noqa: BLE001 - observability remains fail-safe + return _opaque_capture_marker(message) + + message_type = type(message) + roles = { + HumanMessage: "user", + AIMessage: "assistant", + SystemMessage: "system", + ToolMessage: "tool", + FunctionMessage: "function", + ChatMessage: "chat", + } + role = roles.get(message_type) + if role is None: + return _opaque_capture_marker(message) + + captured: dict[str, Any] = { + "content": _capture_jsonable( + _safe_object_attribute(message, "content"), budget=budget + ), + "role": role, + } + name = _safe_object_attribute(message, "name") + if type(name) is str: + captured["name"] = _bounded_string( + _safe_identifier(name, "unknown"), budget + ) + if message_type is AIMessage: + captured["tool_calls"] = _capture_jsonable( + _safe_object_attribute(message, "tool_calls", []), budget=budget + ) + if message_type is ToolMessage: + captured["artifact"] = _capture_jsonable( + _safe_object_attribute(message, "artifact"), budget=budget + ) + captured["status"] = _bounded_string( + _safe_identifier(_safe_object_attribute(message, "status"), "unknown"), + budget, + ) + captured["tool_call_id"] = _bounded_string( + _safe_identifier( + _safe_object_attribute(message, "tool_call_id"), "unknown" + ), + budget, + ) + return captured + + +def _bounded_langchain_messages( + messages: Any, + *, + budget: _CaptureBudget, + prefix: tuple[Any, ...] = (), +) -> Any: + raw_messages = messages if type(messages) in (list, tuple) else () + total_items = len(prefix) + len(raw_messages) + captured = [ + _bounded_langchain_message(message, budget) + for message in (*prefix, *raw_messages[:_MAX_CAPTURE_ITEMS])[ + :_MAX_CAPTURE_ITEMS + ] + ] + if total_items > len(captured): + captured.append({"_truncated_items": total_items - len(captured)}) + return _finalize_capture(captured, messages) + + +def _managed_model_name(request: Any) -> str: + model = _safe_object_attribute(request, "model") + for field in ("model", "model_name", "model_id", "deployment_name"): + value = _safe_object_attribute(model, field) + if type(value) is str and value: + return _safe_identifier(value, "unknown") + return "unknown" + + +def _bounded_model_call_request(request: Any) -> tuple[str, Any]: + """Build a telemetry-only request without model settings, schemas, or tools.""" + import nemo_relay + + budget = _CaptureBudget() + system_message = _safe_object_attribute(request, "system_message") + request_messages = _safe_object_attribute(request, "messages", []) + messages = _bounded_langchain_messages( + request_messages, + budget=budget, + prefix=(() if system_message is None else (system_message,)), + ) + model_name = _managed_model_name(request) + return model_name, nemo_relay.LLMRequest( + {}, + {"messages": messages, "model": model_name}, + ) + + +def _bounded_model_call_response(response: Any) -> dict[str, Any]: + """Project a ModelResponse without Relay's arbitrary-object codec.""" + try: + from langchain.agents.middleware import ModelResponse + except Exception: # noqa: BLE001 - observability remains fail-safe + ModelResponse = None # type: ignore[assignment,misc] + + if ModelResponse is not None and type(response) is ModelResponse: + budget = _CaptureBudget() + raw_messages = _safe_object_attribute(response, "result", []) + captured = { + "messages": _bounded_langchain_messages( + raw_messages, + budget=budget, + ), + "structured_response": _capture_jsonable( + _safe_object_attribute(response, "structured_response"), + budget=budget, + ), + } + finalized = _finalize_capture(captured, response) + return finalized if type(finalized) is dict else {"content": finalized} + + captured = _bounded_capture(response) + return captured if type(captured) is dict else {"content": captured} + + +def _bounded_tool_call_response(response: Any) -> Any: + """Project a ToolMessage while leaving graph-control objects opaque.""" + try: + from langchain_core.messages import ToolMessage + except Exception: # noqa: BLE001 - observability remains fail-safe + ToolMessage = None # type: ignore[assignment,misc] + if ToolMessage is not None and type(response) is ToolMessage: + budget = _CaptureBudget() + return _finalize_capture( + _bounded_langchain_message(response, budget), response + ) + return _bounded_capture(response) + + +class _MetadataOnlyGraphCallbacks: + """LangGraph callback methods that never serialize graph data or errors.""" + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._nemoclaw_scope_handles: dict[Any, Any] = {} + self._nemoclaw_scope_lock = threading.RLock() + + def on_chain_start( + self, + _serialized: dict[str, Any] | None, + _inputs: dict[str, Any], + *, + run_id: Any, + parent_run_id: Any | None = None, + **kwargs: Any, + ) -> None: + """Open a scope identified only by its bounded graph node name.""" + import nemo_relay + + name = _safe_identifier(kwargs.get("name"), "LangGraph") + with self._nemoclaw_scope_lock: + parent = self._nemoclaw_scope_handles.get(parent_run_id) + try: + handle = nemo_relay.scope.push( + name, + nemo_relay.ScopeType.Agent, + handle=parent, + ) + except Exception: # noqa: BLE001 - observability must not fail agent work + logger.debug("NeMo Relay scope start failed") + return + with self._nemoclaw_scope_lock: + self._nemoclaw_scope_handles[run_id] = handle + + def on_chain_end( + self, + _outputs: dict[str, Any], + *, + run_id: Any, + **_kwargs: Any, + ) -> None: + """Close a successful scope without recording graph outputs.""" + self._nemoclaw_pop_scope(run_id, "OK") + + def on_chain_error( + self, + _error: BaseException, + *, + run_id: Any, + **_kwargs: Any, + ) -> None: + """Close a failed scope without recording exception text.""" + self._nemoclaw_pop_scope(run_id, "ERROR") + + def _nemoclaw_pop_scope(self, run_id: Any, status: str) -> None: + import nemo_relay + + with self._nemoclaw_scope_lock: + handle = self._nemoclaw_scope_handles.pop(run_id, None) + if handle is None: + return + try: + nemo_relay.scope.pop( + handle, + metadata={ + "integration": "langgraph", + "otel.status_code": status, + }, + ) + except Exception: # noqa: BLE001 - observability must not fail agent work + logger.debug("NeMo Relay scope end failed") + + def on_interrupt(self, _event: Any) -> None: + """Record an interrupt mark without its potentially sensitive payload.""" + self._nemoclaw_graph_mark("Graph Interrupt") + + def on_resume(self, _event: Any) -> None: + """Record a resume mark without checkpoint or interrupt payloads.""" + self._nemoclaw_graph_mark("Graph Resume") + + @staticmethod + def _nemoclaw_graph_mark(name: str) -> None: + import nemo_relay + + try: + nemo_relay.scope.event( + name, + metadata={"integration": "langgraph"}, + ) + except Exception: # noqa: BLE001 - observability must not fail agent work + logger.debug("NeMo Relay graph mark failed") + + +def new_metadata_only_callback_handler() -> Any: + """Create an isolated metadata-only callback for one compiled graph.""" + from langgraph.callbacks import GraphCallbackHandler + + class MetadataOnlyGraphCallbackHandler( + _MetadataOnlyGraphCallbacks, GraphCallbackHandler + ): + pass + + return MetadataOnlyGraphCallbackHandler() + + +def new_metadata_only_callback_manager() -> Any: + """Create the locked base manager for pinned self-config-first graph merges.""" + from langchain_core.callbacks import CallbackManager + + class MetadataOnlyCallbackManager(CallbackManager): + """Keep exactly one managed handler while preserving config context.""" + + def __init__( + self, + handlers: list[Any], + inheritable_handlers: list[Any] | None = None, + parent_run_id: Any | None = None, + *, + tags: list[str] | None = None, + inheritable_tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + inheritable_metadata: dict[str, Any] | None = None, + ) -> None: + candidates = [*handlers, *(inheritable_handlers or ())] + managed_handlers: list[Any] = [] + for handler in candidates: + if isinstance(handler, _MetadataOnlyGraphCallbacks) and not any( + existing is handler for existing in managed_handlers + ): + managed_handlers.append(handler) + if len(managed_handlers) != 1: + raise RuntimeError( + "managed observability callback manager requires exactly one handler" + ) + managed_handler = managed_handlers[0] + super().__init__( + handlers=[managed_handler], + inheritable_handlers=[managed_handler], + parent_run_id=parent_run_id, + tags=list(tags or ()), + inheritable_tags=list(inheritable_tags or ()), + metadata=dict(metadata or {}), + inheritable_metadata=dict(inheritable_metadata or {}), + ) + + def copy(self) -> MetadataOnlyCallbackManager: + return self.__class__( + handlers=self.handlers.copy(), + inheritable_handlers=self.inheritable_handlers.copy(), + parent_run_id=self.parent_run_id, + tags=self.tags.copy(), + inheritable_tags=self.inheritable_tags.copy(), + metadata=self.metadata.copy(), + inheritable_metadata=self.inheritable_metadata.copy(), + ) + + def merge(self, other: Any) -> MetadataOnlyCallbackManager: + """Merge tags and metadata while discarding external handlers.""" + # LangGraph 1.2.6 calls this locked manager as the base manager. + return self.__class__( + handlers=self.handlers.copy(), + inheritable_handlers=self.inheritable_handlers.copy(), + parent_run_id=self.parent_run_id or other.parent_run_id, + tags=list(dict.fromkeys([*self.tags, *other.tags])), + inheritable_tags=list( + dict.fromkeys([*self.inheritable_tags, *other.inheritable_tags]) + ), + metadata={**self.metadata, **other.metadata}, + inheritable_metadata={ + **self.inheritable_metadata, + **other.inheritable_metadata, + }, + ) + + def add_handler(self, _handler: Any, inherit: bool = True) -> None: + """Reject handler additions performed while runnable configs merge.""" + + def remove_handler(self, _handler: Any) -> None: + """Keep the managed handler installed for the graph lifetime.""" + + def set_handler(self, _handler: Any, inherit: bool = True) -> None: + """Reject attempts to replace the managed handler.""" + + def set_handlers(self, _handlers: list[Any], inherit: bool = True) -> None: + """Reject attempts to replace the managed handler set.""" + + return MetadataOnlyCallbackManager(handlers=[new_metadata_only_callback_handler()]) + + +class _CaptureCallbackException: + def __init__(self, boundary: _RelayExceptionBoundary) -> None: + self._boundary = boundary + + def __enter__(self) -> None: + return None + + def __exit__( + self, + _error_type: type[BaseException] | None, + error: BaseException | None, + _traceback: TracebackType | None, + ) -> bool: + if error is None: + return False + self._boundary.capture(error) + return True + + +class _SuppressRelayException: + def __init__(self, boundary: _RelayExceptionBoundary) -> None: + self._boundary = boundary + + def __enter__(self) -> None: + return None + + def __exit__( + self, + _error_type: type[BaseException] | None, + error: BaseException | None, + _traceback: TracebackType | None, + ) -> bool: + return isinstance(error, Exception) and self._boundary.has_original + + +class _RelayExceptionBoundary: + """Hide callback exceptions from Relay, then restore them for the agent.""" + + def __init__(self) -> None: + self._original: tuple[BaseException, TracebackType | None] | None = None + + @property + def has_original(self) -> bool: + return self._original is not None + + def capture(self, error: BaseException) -> None: + if self._original is None: + # Bypass attacker-controlled exception-subclass dispatch. A custom + # ``__getattribute__`` must not replace the application exception + # with a secret-bearing failure that Relay can observe. + traceback = BaseException.__traceback__.__get__(error, BaseException) + self._original = (error, traceback) + + def capture_callback_exception(self) -> _CaptureCallbackException: + return _CaptureCallbackException(self) + + def suppress_relay_exception(self) -> _SuppressRelayException: + return _SuppressRelayException(self) + + @staticmethod + def raise_redacted() -> NoReturn: + # This method is called only after leaving the handler's ``except`` + # block. The constant exception therefore has no ``__context__`` link + # back to the original exception for Relay to inspect or serialize. + raise RuntimeError(_REDACTED_EXCEPTION_MESSAGE) + + def restore_original(self) -> NoReturn: + if self._original is None: + raise RuntimeError("NemoClaw Relay exception boundary is empty") + error, traceback = self._original + self._original = None + # Call the base implementation directly so an exception subclass cannot + # intercept restoration. A plain raise preserves an explicit __cause__. + BaseException.with_traceback(error, traceback) + raise error + + +def new_relay_middleware() -> Any: + """Create Relay middleware that never exposes agent exception text.""" + import nemo_relay + from nemo_relay.integrations.langchain import NemoRelayMiddleware + from nemo_relay.utils import run_sync + + class BoundedNemoRelayMiddleware(NemoRelayMiddleware): + def wrap_model_call(self, request: Any, handler: Any) -> Any: + prepared_request: tuple[str, Any] | None = None + try: + prepared_request = _bounded_model_call_request(request) + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + pass + if prepared_request is None: + return handler(request) + model_name, relay_request = prepared_request + + original_result: Any = _RESULT_UNSET + callback_started = False + callback_completed = False + + async def bounded_call(_relay_request: Any) -> Any: + nonlocal callback_completed, callback_started, original_result + if callback_started: + if callback_completed: + return _bounded_model_call_response(original_result) + return {"content": _opaque_capture_marker(None)} + callback_started = True + original_result = handler(request) + callback_completed = True + return _bounded_model_call_response(original_result) + + invoke_fallback = False + try: + run_sync( + self._llm_execute( + model_name=model_name, + request=relay_request, + codec=None, + response_codec=None, + func=bounded_call, + ) + ) + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + if callback_completed: + return original_result + if callback_started: + raise + invoke_fallback = True + if invoke_fallback: + return handler(request) + if not callback_completed: + return handler(request) + return original_result + + async def awrap_model_call(self, request: Any, handler: Any) -> Any: + prepared_request: tuple[str, Any] | None = None + try: + prepared_request = _bounded_model_call_request(request) + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + pass + if prepared_request is None: + return await handler(request) + model_name, relay_request = prepared_request + + original_result: Any = _RESULT_UNSET + callback_started = False + callback_completed = False + + async def bounded_call(_relay_request: Any) -> Any: + nonlocal callback_completed, callback_started, original_result + if callback_started: + if callback_completed: + return _bounded_model_call_response(original_result) + return {"content": _opaque_capture_marker(None)} + callback_started = True + original_result = await handler(request) + callback_completed = True + return _bounded_model_call_response(original_result) + + invoke_fallback = False + try: + await self._llm_execute( + model_name=model_name, + request=relay_request, + codec=None, + response_codec=None, + func=bounded_call, + ) + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + if callback_completed: + return original_result + if callback_started: + raise + invoke_fallback = True + if invoke_fallback: + return await handler(request) + if not callback_completed: + return await handler(request) + return original_result + + async def _llm_execute( + self, + model_name: str, + request: Any, + codec: Any, + response_codec: Any, + func: Any, + ) -> Any: + boundary = _RelayExceptionBoundary() + + async def redacted_call(*args: Any, **kwargs: Any) -> Any: + callback_result: Any = None + with boundary.capture_callback_exception(): + callback_result = await func(*args, **kwargs) + if boundary.has_original: + boundary.raise_redacted() + return callback_result + + result: Any = None + with boundary.suppress_relay_exception(): + result = await super()._llm_execute( + model_name=_safe_identifier(model_name, "unknown"), + request=request, + codec=codec, + response_codec=response_codec, + func=redacted_call, + ) + if boundary.has_original: + boundary.restore_original() + return result + + def wrap_tool_call(self, request: Any, handler: Any) -> Any: + prepared_call: tuple[Any, Any, Any, Any] | None = None + try: + prepared_call = self._prepare_tool_call(request) + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + pass + if prepared_call is None: + return handler(request) + parent, _codec, tool_name, tool_args = prepared_call + + boundary = _RelayExceptionBoundary() + original_result: Any = _RESULT_UNSET + callback_started = False + callback_completed = False + + def redacted_call(_args: Any) -> Any: + nonlocal callback_completed, callback_started, original_result + if callback_started: + if callback_completed: + return _bounded_tool_call_response(original_result) + return _opaque_capture_marker(None) + + callback_result: Any = None + with boundary.capture_callback_exception(): + callback_request = request.override( + tool_call={**request.tool_call, "args": tool_args} + ) + callback_started = True + callback_result = handler(callback_request) + if boundary.has_original: + boundary.raise_redacted() + original_result = callback_result + callback_completed = True + return _bounded_tool_call_response(callback_result) + + async def execute_tool() -> Any: + return await nemo_relay.tools.execute( + name=_safe_identifier(tool_name, "unknown"), + args=_bounded_capture(tool_args), + func=redacted_call, + handle=parent, + ) + + invoke_fallback = False + try: + with boundary.suppress_relay_exception(): + run_sync(execute_tool()) + if boundary.has_original: + boundary.restore_original() + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + if callback_completed: + return original_result + if callback_started: + raise + invoke_fallback = True + if invoke_fallback: + return handler(request) + if not callback_completed: + return handler(request) + return original_result + + async def awrap_tool_call(self, request: Any, handler: Any) -> Any: + prepared_call: tuple[Any, Any, Any, Any] | None = None + try: + prepared_call = self._prepare_tool_call(request) + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + pass + if prepared_call is None: + return await handler(request) + parent, _codec, tool_name, tool_args = prepared_call + + boundary = _RelayExceptionBoundary() + original_result: Any = _RESULT_UNSET + callback_started = False + callback_completed = False + + async def redacted_call(_args: Any) -> Any: + nonlocal callback_completed, callback_started, original_result + if callback_started: + if callback_completed: + return _bounded_tool_call_response(original_result) + return _opaque_capture_marker(None) + + callback_result: Any = None + with boundary.capture_callback_exception(): + callback_request = request.override( + tool_call={**request.tool_call, "args": tool_args} + ) + callback_started = True + callback_result = await handler(callback_request) + if boundary.has_original: + boundary.raise_redacted() + original_result = callback_result + callback_completed = True + return _bounded_tool_call_response(callback_result) + + invoke_fallback = False + try: + with boundary.suppress_relay_exception(): + await nemo_relay.tools.execute( + name=_safe_identifier(tool_name, "unknown"), + args=_bounded_capture(tool_args), + func=redacted_call, + handle=parent, + ) + if boundary.has_original: + boundary.restore_original() + except Exception: # noqa: BLE001 - optional instrumentation is fail-open + if callback_completed: + return original_result + if callback_started: + raise + invoke_fallback = True + if invoke_fallback: + return await handler(request) + if not callback_completed: + return await handler(request) + return original_result + + return BoundedNemoRelayMiddleware(name="NemoClawObservabilityMiddleware") + + +def _deregister_guardrails() -> None: + try: + import nemo_relay + + nemo_relay.guardrails.deregister_llm_sanitize_request(_GUARDRAIL_NAME) + nemo_relay.guardrails.deregister_llm_sanitize_response(_GUARDRAIL_NAME) + nemo_relay.guardrails.deregister_tool_sanitize_request(_GUARDRAIL_NAME) + nemo_relay.guardrails.deregister_tool_sanitize_response(_GUARDRAIL_NAME) + except Exception: # noqa: BLE001 - best-effort cleanup + logger.debug("NeMo Relay guardrail cleanup failed") + + +def _new_managed_subscriber(nemo_relay: Any) -> Any: + """Construct Relay without inheriting ambient OpenTelemetry configuration.""" + # Relay 0.4's native exporter reads OTEL_* independently of config.headers, + # so an empty managed header map alone does not clear ambient credentials. + ambient = { + name: value + for name, value in os.environ.items() + if name.startswith(_AMBIENT_OTEL_PREFIX) + } + for name in ambient: + os.environ.pop(name, None) + try: + config = nemo_relay.OpenInferenceConfig() + config.transport = "http_binary" + config.endpoint = _OTLP_ENDPOINT + config.headers = {} + config.service_name = _SERVICE_NAME + config.timeout_millis = _EXPORT_TIMEOUT_MILLIS + return nemo_relay.OpenInferenceSubscriber(config) + finally: + for name, value in ambient.items(): + if value is not None: + os.environ[name] = value + + +def shutdown_observability() -> None: + """Flush and tear down the local exporter without blocking agent shutdown.""" + with _lifecycle_lock: + subscriber = _lifecycle.subscriber + if subscriber is None: + return + _lifecycle.subscriber = None + _lifecycle.active = False + + try: + import nemo_relay + + nemo_relay.subscribers.flush() + except Exception: # noqa: BLE001 - shutdown remains fail-open + logger.debug("NeMo Relay subscriber flush failed") + try: + subscriber.force_flush() + except Exception: # noqa: BLE001 - bounded exporter failure is non-fatal + logger.debug("NeMo Relay OTLP force-flush failed") + try: + subscriber.deregister(_SUBSCRIBER_NAME) + except Exception: # noqa: BLE001 - best-effort cleanup + logger.debug("NeMo Relay subscriber deregistration failed") + try: + subscriber.shutdown() + except Exception: # noqa: BLE001 - best-effort cleanup + logger.debug("NeMo Relay subscriber shutdown failed") + _deregister_guardrails() + + +def initialize_observability() -> bool: + """Enable the fixed bounded-content Relay exporter when explicitly requested.""" + if not observability_requested(): + return False + with _lifecycle_lock: + if _lifecycle.initialization_attempted: + return _lifecycle.active + _lifecycle.initialization_attempted = True + + subscriber: Any = None + try: + import nemo_relay + + nemo_relay.guardrails.register_llm_sanitize_request( + _GUARDRAIL_NAME, 0, _bounded_llm_request + ) + nemo_relay.guardrails.register_llm_sanitize_response( + _GUARDRAIL_NAME, 0, _bounded_llm_response + ) + nemo_relay.guardrails.register_tool_sanitize_request( + _GUARDRAIL_NAME, 0, _bounded_tool_request + ) + nemo_relay.guardrails.register_tool_sanitize_response( + _GUARDRAIL_NAME, 0, _bounded_tool_response + ) + + subscriber = _new_managed_subscriber(nemo_relay) + subscriber.register(_SUBSCRIBER_NAME) + except Exception: # noqa: BLE001 - tracing setup must not stop the agent + logger.warning( + "Managed observability could not be initialized; continuing without tracing" + ) + if subscriber is not None: + try: + subscriber.shutdown() + except Exception: # noqa: BLE001 - best-effort rollback + logger.debug("NeMo Relay rollback failed") + _deregister_guardrails() + return False + + _lifecycle.subscriber = subscriber + _lifecycle.active = True + atexit.register(shutdown_observability) + return True diff --git a/agents/langchain-deepagents-code/patch-managed-deepagents-code.py b/agents/langchain-deepagents-code/patch-managed-deepagents-code.py index 654fd9c223..09316c4730 100644 --- a/agents/langchain-deepagents-code/patch-managed-deepagents-code.py +++ b/agents/langchain-deepagents-code/patch-managed-deepagents-code.py @@ -26,7 +26,9 @@ EXPECTED_DCODE_VERSION = "0.1.30" PATCH_MARKER = "NemoClaw-managed Deep Agents Code hardening v2." TOOL_DISCLOSURE_PATCH_MARKER = "NemoClaw-managed progressive tool disclosure." +OBSERVABILITY_PATCH_MARKER = "NemoClaw-managed backend-neutral observability." MIDDLEWARE_MODULE = "progressive_tool_disclosure.py" +OBSERVABILITY_MODULE = "nemoclaw_observability.py" MANAGED_RUNTIME_SOURCE_PATH = Path(__file__).with_name("managed-dcode-runtime.py") MAIN_MARKER = " args = parser.parse_args()\n" @@ -424,17 +426,21 @@ def _nemoclaw_get_class_path(self, provider_name: str): ModelConfig.get_class_path = _nemoclaw_get_class_path ''' -# Source-of-truth boundary: pinned upstream deepagents-code==0.1.30 has no -# supported managed progressive-disclosure middleware hook in its agent factory -# API, and this repository cannot change that third-party package source. -# Patcher/unit guards plus validate-progressive-tool-disclosure.py cover this -# fail-closed integration. Remove it once upstream provides a supported hook -# preserving managed MCP, credentials, approvals, executor, sandbox, and private -# checkpoint-state boundaries. +# Source-of-truth boundary: pinned upstream deepagents-code==0.1.30 cannot inject +# managed progressive-disclosure or Relay middleware into both main and subagent +# graphs, nor attach a metadata-only callback to the compiled graph. Without this +# root-owned image patch, those graphs omit NemoClaw's runtime controls; this repo +# cannot change the third-party package source. Patcher shape guards, direct-patch +# tests, progressive-disclosure tests, and observability conformance tests fail +# closed on upstream drift. Remove each injection once an upstream agent-factory +# API can preserve the managed MCP, credential, approval, executor, sandbox, +# private checkpoint-state, bounded model/tool content, and metadata-only graph +# trace boundaries end to end. AGENT_PATCH = r''' # NemoClaw-managed Deep Agents Code hardening v2. # NemoClaw-managed progressive tool disclosure. +# NemoClaw-managed backend-neutral observability. from contextvars import ContextVar as _NemoClawContextVar _nemoclaw_original_create_cli_agent = create_cli_agent @@ -442,20 +448,31 @@ def _nemoclaw_get_class_path(self, provider_name: str): _nemoclaw_progressive_disclosure_active = _NemoClawContextVar( "nemoclaw_progressive_disclosure_active", default=False ) +_nemoclaw_observability_active = _NemoClawContextVar( + "nemoclaw_observability_active", default=False +) def _nemoclaw_create_deep_agent(*args, **kwargs): - """Install distinct disclosure middleware in the main and local subagent graphs.""" + """Install managed middleware in the main and local subagent graphs.""" if _nemoclaw_original_create_deep_agent is None: raise RuntimeError("Deep Agents Code create_deep_agent boundary is unavailable") - if not _nemoclaw_progressive_disclosure_active.get(): + progressive_active = _nemoclaw_progressive_disclosure_active.get() + observability_active = _nemoclaw_observability_active.get() + if not progressive_active and not observability_active: return _nemoclaw_original_create_deep_agent(*args, **kwargs) - from deepagents_code.progressive_tool_disclosure import ( - ProgressiveToolDisclosureMiddleware, - ) middleware = list(kwargs.get("middleware") or ()) - middleware.append(ProgressiveToolDisclosureMiddleware()) + if progressive_active: + from deepagents_code.progressive_tool_disclosure import ( + ProgressiveToolDisclosureMiddleware, + ) + + middleware.append(ProgressiveToolDisclosureMiddleware()) + if observability_active: + from deepagents_code.nemoclaw_observability import new_relay_middleware + + middleware.append(new_relay_middleware()) kwargs["middleware"] = middleware subagents = kwargs.get("subagents") @@ -464,7 +481,10 @@ def _nemoclaw_create_deep_agent(*args, **kwargs): for subagent in subagents: if isinstance(subagent, dict): subagent_middleware = list(subagent.get("middleware") or ()) - subagent_middleware.append(ProgressiveToolDisclosureMiddleware()) + if progressive_active: + subagent_middleware.append(ProgressiveToolDisclosureMiddleware()) + if observability_active: + subagent_middleware.append(new_relay_middleware()) subagent = {**subagent, "middleware": subagent_middleware} patched_subagents.append(subagent) kwargs["subagents"] = patched_subagents @@ -477,7 +497,7 @@ def _nemoclaw_create_deep_agent(*args, **kwargs): def create_cli_agent(model, assistant_id, *args, **kwargs): - """Keep managed graph posture and progressively disclose loaded MCP tools.""" + """Keep managed graph posture, disclosure, and observability boundaries.""" kwargs["rubric_model"] = None kwargs["async_subagents"] = None from deepagents_code.progressive_tool_disclosure import ( @@ -500,13 +520,38 @@ def create_cli_agent(model, assistant_id, *args, **kwargs): progressive_active = False if progressive_active and _nemoclaw_original_create_deep_agent is None: raise RuntimeError("Deep Agents Code create_deep_agent boundary is unavailable") - token = _nemoclaw_progressive_disclosure_active.set(progressive_active) + from deepagents_code.nemoclaw_observability import ( + initialize_observability, + new_metadata_only_callback_manager, + ) + + observability_active = initialize_observability() + if observability_active and _nemoclaw_original_create_deep_agent is None: + raise RuntimeError("Deep Agents Code create_deep_agent boundary is unavailable") + progressive_token = _nemoclaw_progressive_disclosure_active.set( + progressive_active + ) + observability_token = _nemoclaw_observability_active.set(observability_active) try: - return _nemoclaw_original_create_cli_agent( + result = _nemoclaw_original_create_cli_agent( model, assistant_id, *args, **kwargs ) finally: - _nemoclaw_progressive_disclosure_active.reset(token) + _nemoclaw_observability_active.reset(observability_token) + _nemoclaw_progressive_disclosure_active.reset(progressive_token) + if not observability_active: + return result + agent, backend = result + # Copy the graph, then replace callbacks directly. with_config would merge a + # pre-bound manager first and retain its handlers. Invocation safety then + # relies on pinned LangGraph 1.2.6 calling ensure_config(self.config, + # input_config); validate-observability.py locks that merge path. + agent = agent.with_config({}) + agent.config = { + **agent.config, + "callbacks": new_metadata_only_callback_manager(), + } + return agent, backend def _resolve_ptc_option(*args, **kwargs): @@ -945,6 +990,33 @@ def _package_root() -> Path: return Path(roots[0]) +def _load_managed_module( + root: Path, + module_name: str, + source_boundary_name: str, + installed_boundary_name: str | None = None, +) -> tuple[Path, str]: + source_path = Path(__file__).with_name(module_name) + destination_path = root / module_name + if not source_path.is_file(): + raise RuntimeError( + f"NemoClaw {source_boundary_name} source not found at {source_path}" + ) + source = source_path.read_text(encoding="utf-8") + compile(source, str(destination_path), "exec") + if destination_path.exists() or destination_path.is_symlink(): + if ( + not destination_path.is_file() + or destination_path.is_symlink() + or destination_path.read_text(encoding="utf-8") != source + ): + raise RuntimeError( + "Refusing to overwrite unexpected " + f"{installed_boundary_name or source_boundary_name} at {destination_path}" + ) + return destination_path, source + + def main() -> None: actual_version = importlib.metadata.version("deepagents-code") if actual_version != EXPECTED_DCODE_VERSION: @@ -989,23 +1061,12 @@ def main() -> None: } texts = {name: path.read_text(encoding="utf-8") for name, path in paths.items()} - module_source_path = Path(__file__).with_name(MIDDLEWARE_MODULE) - module_destination_path = root / MIDDLEWARE_MODULE - if not module_source_path.is_file(): - raise RuntimeError( - f"NemoClaw middleware source not found at {module_source_path}" - ) - module_source = module_source_path.read_text(encoding="utf-8") - compile(module_source, str(module_destination_path), "exec") - if module_destination_path.exists() or module_destination_path.is_symlink(): - if ( - not module_destination_path.is_file() - or module_destination_path.is_symlink() - or module_destination_path.read_text(encoding="utf-8") != module_source - ): - raise RuntimeError( - f"Refusing to overwrite unexpected middleware at {module_destination_path}" - ) + module_destination_path, module_source = _load_managed_module( + root, MIDDLEWARE_MODULE, "middleware" + ) + observability_destination_path, observability_source = _load_managed_module( + root, OBSERVABILITY_MODULE, "observability", "observability module" + ) marker_states = {PATCH_MARKER in text for text in texts.values()} helper_path = root / "_nemoclaw_managed.py" @@ -1016,6 +1077,18 @@ def main() -> None: raise RuntimeError("Managed package patch is partial: helper is missing") if not module_destination_path.is_file(): raise RuntimeError("Managed package patch is partial: middleware is missing") + if not observability_destination_path.is_file(): + raise RuntimeError( + "Managed package patch is partial: observability module is missing" + ) + for marker, boundary in ( + (TOOL_DISCLOSURE_PATCH_MARKER, "progressive-disclosure"), + (OBSERVABILITY_PATCH_MARKER, "observability"), + ): + if texts["agent"].count(marker) != 1: + raise RuntimeError( + f"Managed package {boundary} patch is partial in {paths['agent']}" + ) if texts["agent"].count(AGENT_PATCH.lstrip()) != 1: raise RuntimeError( f"Managed package progressive-disclosure patch is incomplete in {paths['agent']}" @@ -1028,6 +1101,11 @@ def main() -> None: "Managed package progressive-disclosure patch is partial; " "refusing mixed source state" ) + if OBSERVABILITY_PATCH_MARKER in texts["agent"]: + raise RuntimeError( + "Managed package observability patch is partial; " + "refusing mixed source state" + ) _require_functions(paths["main"], texts["main"], {"parse_args"}) _require_methods( @@ -1255,6 +1333,11 @@ def main() -> None: helper_path.write_text(managed_runtime_source, encoding="utf-8") if not module_destination_path.exists(): module_destination_path.write_text(module_source, encoding="utf-8") + if not observability_destination_path.exists(): + observability_destination_path.write_text( + observability_source, + encoding="utf-8", + ) if __name__ == "__main__": diff --git a/agents/langchain-deepagents-code/requirements.in b/agents/langchain-deepagents-code/requirements.in index 98d2090b39..f8df0784be 100644 --- a/agents/langchain-deepagents-code/requirements.in +++ b/agents/langchain-deepagents-code/requirements.in @@ -3,3 +3,4 @@ # uv==0.11.15 deepagents-code[nvidia]==0.1.30 +nemo-relay[langgraph]==0.4.0 diff --git a/agents/langchain-deepagents-code/requirements.lock b/agents/langchain-deepagents-code/requirements.lock index 088f32b204..dc3cced7bc 100644 --- a/agents/langchain-deepagents-code/requirements.lock +++ b/agents/langchain-deepagents-code/requirements.lock @@ -1173,6 +1173,7 @@ langchain==1.3.11 \ # deepagents # deepagents-code # langchain-quickjs + # nemo-relay langchain-anthropic==1.4.8 \ --hash=sha256:778e9301b6fd517824f76ec1776975ce8add97a1f6a36c50ae3c2f4b03a66f7f \ --hash=sha256:c76891b2044d56105ff13c106ed12650637b53bd598a4bdf15b4796eefa2a4ec @@ -1196,6 +1197,7 @@ langchain-core==1.4.8 \ # langgraph-checkpoint # langgraph-prebuilt # langgraph-sdk + # nemo-relay langchain-google-genai==4.2.6 \ --hash=sha256:653dc331e691ddd79784d9ff6a4082749e0f873394c6dc782c414eb4409850eb \ --hash=sha256:c40db0c2d033a5fb6db8e2cc3fb6d49c5678b89b337a64da095fb73ec9f72021 @@ -1233,6 +1235,7 @@ langgraph==1.2.6 \ # langchain-quickjs # langgraph-api # langgraph-runtime-inmem + # nemo-relay langgraph-api==0.10.0 \ --hash=sha256:f4b545bf1936c4e90bed1cc07a6852b1be37e1667007ab4736d800cc18e9f9b3 \ --hash=sha256:f594a160857e2cd7fb2352bba585111177a4a394625ad0cf5898814754fb7bc1 @@ -1460,6 +1463,13 @@ multidict==6.7.1 \ # via # aiohttp # yarl +nemo-relay==0.4.0 \ + --hash=sha256:0f92883b81540076e4b6c5e754eb7726225336634204ebd22ff730ccbcfb2723 \ + --hash=sha256:6a5a5f5dec1428085f6c41c3645f1773f9cfb154cfa47306c93d6514091a8006 \ + --hash=sha256:a5d02d9d0a2d19bf341bf9fd1faae01713bc2021a5b144ef79f91a07ec4c0bfb \ + --hash=sha256:d5f14fe5c8e5fbcc26827cc88e5b867f4ad37d6f983100cbf31705234c39f9d3 \ + --hash=sha256:dc1effd4052d1d47a1cdc553415c878a73d1e01c81a7e9f9194b30a3c88d1947 + # via -r agents/langchain-deepagents-code/requirements.in openai==2.43.0 \ --hash=sha256:65a670b54fadf2268c9e1330133373c963eb779ee969e5cbad419ec2c21dce97 \ --hash=sha256:e74d238200a26868977002190fb6631613480a93dfe0c9c982e77021ed60a017 diff --git a/agents/langchain-deepagents-code/start.sh b/agents/langchain-deepagents-code/start.sh index 65cd2a0af0..52b5e462eb 100755 --- a/agents/langchain-deepagents-code/start.sh +++ b/agents/langchain-deepagents-code/start.sh @@ -176,7 +176,21 @@ prepare_runtime_env() { mv -f "$tmp" "$target" } +prepare_observability_marker() { + local target=/tmp/nemoclaw-observability-enabled + local tmp + if [ "${NEMOCLAW_OBSERVABILITY:-}" != "1" ]; then + rm -f "$target" + return 0 + fi + tmp="$(mktemp /tmp/nemoclaw-observability-enabled.XXXXXX)" + printf '%s\n' '1' >"$tmp" + chmod 444 "$tmp" + mv -f "$tmp" "$target" +} + prepare_runtime_env +prepare_observability_marker # With no command, this invocation IS the sandbox's long-running entrypoint. # Deep Agents Code is a terminal-runtime agent invoked on demand via diff --git a/agents/langchain-deepagents-code/validate-observability.py b/agents/langchain-deepagents-code/validate-observability.py new file mode 100644 index 0000000000..3a96164c36 --- /dev/null +++ b/agents/langchain-deepagents-code/validate-observability.py @@ -0,0 +1,930 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validate managed observability against the pinned real NeMo Relay runtime.""" + +from __future__ import annotations + +import asyncio +import http.server +import importlib +import importlib.metadata +import importlib.util +import math +import os +import re +import sys +import threading +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType +from types import SimpleNamespace +from typing import Any +from typing import cast + +import nemo_relay +from langchain.agents.middleware import ModelRequest +from langchain.agents.middleware import ModelResponse +from langchain.agents.middleware.types import ToolCallRequest +from langchain_core.messages import AIMessage +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.callbacks import CallbackManager +from langchain_core.messages import HumanMessage +from langchain_core.messages import ToolMessage +from langchain_core.runnables.config import get_async_callback_manager_for_config +from langchain_core.runnables.config import get_callback_manager_for_config +from langgraph._internal._config import ensure_config as ensure_langgraph_config +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) + +_EXPECTED_RELAY_VERSION = "0.4.0" +_EXPECTED_LANGGRAPH_VERSION = "1.2.6" +_EXPECTED_PRODUCTION_ENDPOINT = "http://host.openshell.internal:4318/v1/traces" +_EXPECTED_REQUEST_COUNT = 13 +_EXPECTED_WIRE_HEADERS = { + "accept", + "content-length", + "content-type", + "host", + "user-agent", +} +_MAX_REQUEST_BODY_BYTES = 1_048_576 +_SAFE_IDENTIFIER = re.compile(r"[A-Za-z0-9_.:/-]+") + +_PROMPT_SECRET = "NEMOCLAW_PROMPT_SECRET" +_MODEL_OUTPUT_SECRET = "NEMOCLAW_MODEL_OUTPUT_SECRET" +_TOOL_ARGUMENT_SECRET = "NEMOCLAW_TOOL_ARGUMENT_SECRET" +_TOOL_RESULT_SECRET = "NEMOCLAW_TOOL_RESULT_SECRET" +_MODEL_WRAPPER_OUTPUT = "NEMOCLAW_MODEL_WRAPPER_OUTPUT" +_TOOL_MESSAGE_OUTPUT = "NEMOCLAW_TOOL_MESSAGE_OUTPUT" +_OPAQUE_ARTIFACT_SECRET = "NEMOCLAW_OPAQUE_ARTIFACT_SECRET" +_EXCEPTION_SECRET = "NEMOCLAW_EXCEPTION_SECRET" +_AMBIENT_EXPORTER_SECRET = "NEMOCLAW_AMBIENT_EXPORTER_SECRET" +_DROPPED_REQUEST_SURFACE_SECRET = "NEMOCLAW_DROPPED_REQUEST_SURFACE_SECRET" +_TRUNCATION_SENTINEL = "MUST_NOT_REACH_RELAY" +_STABLE_ERROR_CODE = "NEMOCLAW_DCODE_OPERATION_FAILED" +_CONTROL_CHARACTERS = "\r\n\t\x00\u202e" +_OVERLONG_IDENTIFIER = "x" * 200 +_HOSTILE_EXCEPTION_DISPATCHES = [0] + + +@dataclass(frozen=True) +class _CapturedRequest: + method: str + path: str + headers: dict[str, str] + body: bytes + + +class _HostileCallback(BaseCallbackHandler): + """Invocation callback that must never enter the managed graph.""" + + +class _OpaqueArtifact: + def __init__(self) -> None: + self.api_token = _OPAQUE_ARTIFACT_SECRET + + def __str__(self) -> str: + return _OPAQUE_ARTIFACT_SECRET + + +class _HostileMessage: + def __repr__(self) -> str: + raise AssertionError("observability evaluated a hostile message repr") + + def __str__(self) -> str: + raise AssertionError("observability evaluated a hostile message string") + + +class _HostileException(RuntimeError): + @property + def __traceback__(self) -> Any: + _HOSTILE_EXCEPTION_DISPATCHES[0] += 1 + raise RuntimeError(f"hostile-traceback:{_EXCEPTION_SECRET}") + + def with_traceback(self, _traceback: Any) -> Any: + _HOSTILE_EXCEPTION_DISPATCHES[0] += 1 + raise RuntimeError(f"hostile-restore:{_EXCEPTION_SECRET}") + + +class _CollectorServer(http.server.ThreadingHTTPServer): + daemon_threads = True + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _CollectorHandler) + self._capture_lock = threading.Lock() + self._requests: list[_CapturedRequest] = [] + self._failures: list[str] = [] + + def capture(self, request: _CapturedRequest) -> None: + with self._capture_lock: + self._requests.append(request) + + def fail(self, message: str) -> None: + with self._capture_lock: + self._failures.append(message) + + def snapshot(self) -> tuple[list[_CapturedRequest], list[str]]: + with self._capture_lock: + return list(self._requests), list(self._failures) + + +class _CollectorHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self) -> None: + collector = cast(_CollectorServer, self.server) + try: + content_length = int(self.headers.get("content-length", "")) + except ValueError: + collector.fail("OTLP request had an invalid content-length") + self.send_error(400) + return + if not 0 < content_length <= _MAX_REQUEST_BODY_BYTES: + collector.fail("OTLP request body exceeded the validation bound") + self.send_error(413) + return + + body = self.rfile.read(content_length) + if len(body) != content_length: + collector.fail("OTLP request body was truncated") + self.send_error(400) + return + collector.capture( + _CapturedRequest( + method="POST", + path=self.path, + headers={key.lower(): value for key, value in self.headers.items()}, + body=body, + ) + ) + self.send_response(200) + self.send_header("content-length", "0") + self.end_headers() + + def log_message(self, _format: str, *_args: Any) -> None: + pass + + +def _load_observability_module() -> ModuleType: + """Import the patched package module, or an explicit source path for local checks.""" + if len(sys.argv) == 1: + return importlib.import_module("deepagents_code.nemoclaw_observability") + if len(sys.argv) != 2: + raise SystemExit("usage: validate-observability.py [nemoclaw_observability.py]") + + path = Path(sys.argv[1]).resolve(strict=True) + spec = importlib.util.spec_from_file_location( + "nemoclaw_observability_validation", path + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load observability module from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _raw_identifier(prefix: str) -> str: + return ( + f"{prefix}{_CONTROL_CHARACTERS}{_OVERLONG_IDENTIFIER}" + f"-{_TRUNCATION_SENTINEL}" + ) + + +def _tool_request(name: str) -> ToolCallRequest: + return ToolCallRequest( + tool_call={ + "name": name, + "args": {"command": _TOOL_ARGUMENT_SECRET}, + "id": "managed-observability-validation", + }, + tool=None, + state={}, + runtime=None, + ) + + +def _assert_original_exception( + caught: BaseException, + expected: BaseException, + handler_name: str, +) -> None: + if caught is not expected: + raise AssertionError(f"Relay changed the {handler_name} exception identity") + traceback = BaseException.__traceback__.__get__(caught, BaseException) + frame_names: list[str] = [] + while traceback is not None: + frame_names.append(traceback.tb_frame.f_code.co_name) + traceback = traceback.tb_next + if handler_name not in frame_names: + raise AssertionError(f"Relay removed the {handler_name} traceback frame") + + +async def _exercise_async_boundaries( + observability: ModuleType, + middleware: Any, + raw_names: dict[str, str], +) -> None: + request = nemo_relay.LLMRequest( + {"authorization": _PROMPT_SECRET}, + { + "model": raw_names["model"], + "messages": [{"role": "user", "content": _PROMPT_SECRET}], + "model_settings": {"api_key": _DROPPED_REQUEST_SURFACE_SECRET}, + "response_format": {"schema": _DROPPED_REQUEST_SURFACE_SECRET}, + "tools": [{"description": _DROPPED_REQUEST_SURFACE_SECRET}], + }, + ) + + async def successful_model(inner_request: Any) -> dict[str, str]: + if inner_request.headers != {"authorization": _PROMPT_SECRET}: + raise AssertionError("model telemetry changed execution headers") + if inner_request.content["messages"][0]["content"] != _PROMPT_SECRET: + raise AssertionError("model telemetry changed the execution prompt") + return {"content": _MODEL_OUTPUT_SECRET} + + model_result = await middleware._llm_execute( + raw_names["model"], + request, + None, + None, + successful_model, + ) + if model_result != {"content": _MODEL_OUTPUT_SECRET}: + raise AssertionError("model telemetry changed the callback result") + + async_model_error = RuntimeError(f"async-model:{_EXCEPTION_SECRET}") + + async def failing_async_model(_request: Any) -> Any: + raise async_model_error + + try: + await middleware._llm_execute( + "failure-model", request, None, None, failing_async_model + ) + except RuntimeError as caught: + _assert_original_exception(caught, async_model_error, "failing_async_model") + else: + raise AssertionError("Relay swallowed the async model exception") + + hostile_cause = ValueError(f"hostile-cause:{_EXCEPTION_SECRET}") + hostile_error = _HostileException(f"hostile-error:{_EXCEPTION_SECRET}") + hostile_error.__cause__ = hostile_cause + + async def hostile_async_model(_request: Any) -> Any: + raise hostile_error + + try: + await middleware._llm_execute( + "hostile-model", request, None, None, hostile_async_model + ) + except RuntimeError as caught: + _assert_original_exception(caught, hostile_error, "hostile_async_model") + cause = BaseException.__cause__.__get__(caught, BaseException) + if cause is not hostile_cause: + raise AssertionError("Relay removed the hostile exception cause") + if _HOSTILE_EXCEPTION_DISPATCHES[0] != 0: + raise AssertionError("observability used hostile exception dispatch") + else: + raise AssertionError("Relay swallowed the hostile async model exception") + + async_request = _tool_request(raw_names["async_tool"]) + + async def successful_async_tool(inner_request: Any) -> dict[str, str]: + if inner_request.tool_call["args"] != {"command": _TOOL_ARGUMENT_SECRET}: + raise AssertionError("tool telemetry changed execution arguments") + return {"result": _TOOL_RESULT_SECRET} + + tool_result = await middleware.awrap_tool_call( + async_request, successful_async_tool + ) + if tool_result != {"result": _TOOL_RESULT_SECRET}: + raise AssertionError("tool telemetry changed the callback result") + + async_tool_error = RuntimeError(f"async-tool:{_EXCEPTION_SECRET}") + + async def failing_async_tool(_request: Any) -> Any: + raise async_tool_error + + try: + await middleware.awrap_tool_call(async_request, failing_async_tool) + except RuntimeError as caught: + _assert_original_exception(caught, async_tool_error, "failing_async_tool") + else: + raise AssertionError("Relay swallowed the async tool exception") + + control_flow_error = KeyboardInterrupt(f"control-flow:{_EXCEPTION_SECRET}") + + async def interrupted_model(_request: Any) -> Any: + raise control_flow_error + + try: + await middleware._llm_execute( + "interrupt-model", request, None, None, interrupted_model + ) + except KeyboardInterrupt as caught: + _assert_original_exception(caught, control_flow_error, "interrupted_model") + else: + raise AssertionError("Relay swallowed the control-flow exception") + + if not observability._lifecycle.active: + raise AssertionError("observability deactivated while handling callbacks") + + +def _exercise_sync_tool(middleware: Any, raw_tool_name: str) -> None: + request = _tool_request(raw_tool_name) + sync_tool_error = RuntimeError(f"sync-tool:{_EXCEPTION_SECRET}") + + def failing_sync_tool(inner_request: Any) -> Any: + if inner_request.tool_call["args"] != {"command": _TOOL_ARGUMENT_SECRET}: + raise AssertionError("tool telemetry changed sync execution arguments") + raise sync_tool_error + + try: + middleware.wrap_tool_call(request, failing_sync_tool) + except RuntimeError as caught: + _assert_original_exception(caught, sync_tool_error, "failing_sync_tool") + else: + raise AssertionError("Relay swallowed the sync tool exception") + + +def _exercise_framework_result_transparency(middleware: Any) -> None: + model_request = ModelRequest( + model=SimpleNamespace(model="managed-wrapper-model"), + messages=[HumanMessage(content=_PROMPT_SECRET)], + ) + expected_model_result = ModelResponse( + result=[AIMessage(content=_MODEL_WRAPPER_OUTPUT)], + structured_response={"value": float("nan")}, + ) + + def model_handler(_request: Any) -> ModelResponse[Any]: + return expected_model_result + + actual_model_result = middleware.wrap_model_call(model_request, model_handler) + if actual_model_result is not expected_model_result: + raise AssertionError("observability replaced the LangChain ModelResponse") + structured_value = actual_model_result.structured_response["value"] + if not math.isnan(structured_value): + raise AssertionError("observability mutated non-finite structured model output") + + artifact = _OpaqueArtifact() + expected_tool_result = ToolMessage( + content=_TOOL_MESSAGE_OUTPUT, + tool_call_id="managed-observability-validation", + artifact=artifact, + ) + + def tool_handler(_request: Any) -> ToolMessage: + return expected_tool_result + + actual_tool_result = middleware.wrap_tool_call( + _tool_request("framework-result-tool"), tool_handler + ) + if actual_tool_result is not expected_tool_result: + raise AssertionError("observability replaced the LangChain ToolMessage") + if actual_tool_result.artifact is not artifact: + raise AssertionError("observability mutated the ToolMessage artifact") + + +def _exercise_real_relay_json_domain(middleware: Any) -> None: + original_args = { + "huge_negative": -(10**1000), + "huge_positive": 10**1000, + "lone_surrogate": "before\ud800after", + } + expected_result = { + "huge_result": 10**1000, + "lone_surrogate_result": "before\udfffafter", + } + request = ToolCallRequest( + tool_call={ + "name": "relay-json-domain-tool", + "args": original_args, + "id": "managed-observability-json-domain", + }, + tool=None, + state={}, + runtime=None, + ) + calls = 0 + + def handler(inner_request: Any) -> dict[str, Any]: + nonlocal calls + calls += 1 + if inner_request.tool_call["args"] != original_args: + raise AssertionError("observability mutated non-Relay-safe tool arguments") + return expected_result + + actual_result = middleware.wrap_tool_call(request, handler) + if calls != 1 or actual_result is not expected_result: + raise AssertionError("Relay changed a non-Relay-safe application value") + + +def _exercise_relay_failure_transparency(middleware: Any) -> None: + request = _tool_request("relay-failure-tool") + expected_result = object() + original_execute = nemo_relay.tools.execute + + def run_case(mode: str) -> None: + calls = 0 + + def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + return expected_result + + async def injected_execute(**kwargs: Any) -> Any: + if mode == "before": + raise RuntimeError("injected Relay failure before callback") + await original_execute(**kwargs) + raise RuntimeError("injected Relay failure after callback") + + nemo_relay.tools.execute = injected_execute + try: + actual_result = middleware.wrap_tool_call(request, handler) + finally: + nemo_relay.tools.execute = original_execute + if calls != 1 or actual_result is not expected_result: + raise AssertionError( + f"Relay {mode}-callback failure changed application execution" + ) + + run_case("before") + run_case("after") + + fallback_cause = ValueError("fallback application cause") + fallback_context = LookupError("fallback application context") + fallback_error = RuntimeError("fallback application error") + fallback_error.__cause__ = fallback_cause + fallback_error.__context__ = fallback_context + fallback_calls = 0 + + def failing_fallback_handler(_request: Any) -> Any: + nonlocal fallback_calls + fallback_calls += 1 + raise fallback_error + + async def fail_before_callback(**_kwargs: Any) -> Any: + raise RuntimeError("injected Relay failure before callback") + + nemo_relay.tools.execute = fail_before_callback + try: + try: + middleware.wrap_tool_call(request, failing_fallback_handler) + except RuntimeError as caught: + if caught is not fallback_error or fallback_calls != 1: + raise AssertionError("Relay changed the fallback application error") + cause = BaseException.__cause__.__get__(caught, BaseException) + context = BaseException.__context__.__get__(caught, BaseException) + if cause is not fallback_cause or context is not fallback_context: + raise AssertionError("Relay contaminated fallback exception chaining") + else: + raise AssertionError("Relay swallowed the fallback application error") + finally: + nemo_relay.tools.execute = original_execute + + application_error = RuntimeError(f"relay-control:{_EXCEPTION_SECRET}") + control_flow = KeyboardInterrupt("operator interrupt") + control_calls = 0 + + def interrupted_handler(_request: Any) -> Any: + nonlocal control_calls + control_calls += 1 + raise application_error + + async def replace_relay_error_with_control_flow(**kwargs: Any) -> Any: + try: + return await original_execute(**kwargs) + except Exception: + raise control_flow + + nemo_relay.tools.execute = replace_relay_error_with_control_flow + try: + try: + middleware.wrap_tool_call(request, interrupted_handler) + except KeyboardInterrupt as caught: + if caught is not control_flow or control_calls != 1: + raise AssertionError("observability changed Relay control flow") + else: + raise AssertionError("observability swallowed Relay control flow") + finally: + nemo_relay.tools.execute = original_execute + + +def _assert_capture_traversal_bounds(observability: ModuleType) -> None: + shared: list[Any] = ["leaf"] + for _ in range(observability._MAX_CAPTURE_DEPTH + 1): + shared = [shared] * observability._MAX_CAPTURE_ITEMS + captured_shared = observability._bounded_capture(shared) + encoded_shared = repr(captured_shared) + if len(encoded_shared) > observability._MAX_CAPTURE_JSON_CHARS: + raise AssertionError("shared-container capture exceeded the aggregate bound") + if "shared_or_cycle" not in encoded_shared: + raise AssertionError("shared-container capture did not record reference omission") + + cyclic: list[Any] = [] + cyclic.append(cyclic) + captured_cycle = observability._bounded_capture(cyclic) + if "shared_or_cycle" not in repr(captured_cycle): + raise AssertionError("cyclic capture did not terminate with a reference marker") + + large_request = SimpleNamespace( + model=SimpleNamespace(model="bounded-message-model"), + system_message=None, + messages=[HumanMessage(content="x" * 9_000) for _ in range(100)] + + [_HostileMessage()], + ) + _, captured_request = observability._bounded_model_call_request(large_request) + encoded_request = repr(captured_request.content) + if len(encoded_request) > observability._MAX_CAPTURE_JSON_CHARS: + raise AssertionError("projected model messages exceeded the aggregate bound") + if "_truncated" not in encoded_request: + raise AssertionError("projected model messages did not record truncation") + + large_response = ModelResponse( + result=[AIMessage(content="y" * 9_000) for _ in range(100)] + ) + encoded_response = repr(observability._bounded_model_call_response(large_response)) + if len(encoded_response) > observability._MAX_CAPTURE_JSON_CHARS: + raise AssertionError("projected model response exceeded the aggregate bound") + if "_truncated" not in encoded_response: + raise AssertionError("projected model response did not record truncation") + + +def _exercise_graph(observability: ModuleType, raw_graph_name: str) -> None: + callback = observability.new_metadata_only_callback_handler() + callback.on_chain_start( + None, + {}, + run_id="managed-observability-validation", + name=raw_graph_name, + ) + callback.on_chain_error( + RuntimeError(f"graph:{_EXCEPTION_SECRET}"), + run_id="managed-observability-validation", + ) + + +def _safe_names( + observability: ModuleType, raw_names: dict[str, str] +) -> dict[str, str]: + fallbacks = { + "model": "unknown", + "sync_tool": "unknown", + "async_tool": "unknown", + "graph": "LangGraph", + } + safe_names = { + name: observability._safe_identifier(value, fallbacks[name]) + for name, value in raw_names.items() + } + for name, value in safe_names.items(): + if len(value) > 128: + raise AssertionError(f"{name} identifier exceeds the 128-character cap") + if _SAFE_IDENTIFIER.fullmatch(value) is None: + raise AssertionError(f"{name} identifier contains an unsafe character") + if _TRUNCATION_SENTINEL in value: + raise AssertionError(f"{name} identifier was not truncated") + return safe_names + + +def _assert_only_managed_handler(manager: Any, managed_handler: Any) -> None: + if manager.handlers != [managed_handler]: + raise AssertionError("an invocation callback entered the managed handler set") + if manager.inheritable_handlers != [managed_handler]: + raise AssertionError("an invocation callback became inheritable") + + +def _assert_unique_attributes(attributes: Any, location: str) -> None: + seen: set[str] = set() + for attribute in attributes: + if attribute.key in seen: + raise AssertionError(f"{location} contains duplicate OTLP attribute keys") + seen.add(attribute.key) + + +def _assert_unique_otlp_attribute_keys(body: bytes, request_index: int) -> None: + request = ExportTraceServiceRequest.FromString(body) + for resource_index, resource_spans in enumerate(request.resource_spans, 1): + resource_location = ( + f"OTLP request {request_index} resource {resource_index}" + ) + _assert_unique_attributes( + resource_spans.resource.attributes, resource_location + ) + for scope_index, scope_spans in enumerate(resource_spans.scope_spans, 1): + scope_location = f"{resource_location} scope {scope_index}" + _assert_unique_attributes( + scope_spans.scope.attributes, scope_location + ) + for span_index, span in enumerate(scope_spans.spans, 1): + span_location = f"{scope_location} span {span_index}" + _assert_unique_attributes(span.attributes, span_location) + for event_index, event in enumerate(span.events, 1): + _assert_unique_attributes( + event.attributes, + f"{span_location} event {event_index}", + ) + for link_index, link in enumerate(span.links, 1): + _assert_unique_attributes( + link.attributes, + f"{span_location} link {link_index}", + ) + + +def _assert_callback_manager_boundary(observability: ModuleType) -> None: + bound_manager = observability.new_metadata_only_callback_manager() + managed_handler = bound_manager.handlers[0] + hostile_handler = _HostileCallback() + + bound_manager.add_handler(hostile_handler) + bound_manager.set_handler(hostile_handler) + bound_manager.set_handlers([hostile_handler]) + bound_manager.remove_handler(managed_handler) + _assert_only_managed_handler(bound_manager, managed_handler) + _assert_only_managed_handler(bound_manager.copy(), managed_handler) + + hostile_manager = CallbackManager( + handlers=[hostile_handler], + inheritable_handlers=[hostile_handler], + tags=["invocation-manager-tag"], + inheritable_tags=["invocation-manager-inheritable-tag"], + metadata={"invocation_manager": "preserved"}, + inheritable_metadata={"invocation_manager_inheritable": "preserved"}, + ) + merged_manager = bound_manager.merge(hostile_manager) + _assert_only_managed_handler(merged_manager, managed_handler) + if merged_manager.tags != ["invocation-manager-tag"]: + raise AssertionError("callback-manager tags were not preserved") + if merged_manager.metadata != {"invocation_manager": "preserved"}: + raise AssertionError("callback-manager metadata was not preserved") + + # Pregel 1.2.6 invokes this as ensure_config(self.config, input_config). + list_config = ensure_langgraph_config( + { + "callbacks": bound_manager, + "tags": ["managed-tag"], + "metadata": {"managed": "preserved"}, + }, + { + "callbacks": [hostile_handler], + "tags": ["invocation-list-tag"], + "metadata": {"invocation_list": "preserved"}, + }, + ) + manager_config = ensure_langgraph_config( + {"callbacks": bound_manager}, + {"callbacks": hostile_manager}, + ) + configured_cases = ( + ( + list_config, + {"managed-tag", "invocation-list-tag"}, + {"managed": "preserved", "invocation_list": "preserved"}, + ), + ( + manager_config, + {"invocation-manager-tag"}, + {"invocation_manager": "preserved"}, + ), + ) + for config, expected_tags, expected_metadata in configured_cases: + _assert_only_managed_handler(config["callbacks"], managed_handler) + sync_manager = get_callback_manager_for_config(config) + async_manager = get_async_callback_manager_for_config(config) + for configured_manager in (sync_manager, async_manager): + _assert_only_managed_handler(configured_manager, managed_handler) + if set(configured_manager.tags) != expected_tags: + raise AssertionError("configured callback tags were not preserved") + if configured_manager.metadata != expected_metadata: + raise AssertionError("configured callback metadata was not preserved") + + if set(list_config["tags"]) != {"managed-tag", "invocation-list-tag"}: + raise AssertionError("runnable tags were not preserved") + if list_config["metadata"] != { + "managed": "preserved", + "invocation_list": "preserved", + }: + raise AssertionError("runnable metadata was not preserved") + + +def _assert_wire_requests( + requests: list[_CapturedRequest], + failures: list[str], + observability: ModuleType, + raw_names: dict[str, str], +) -> int: + if failures: + raise AssertionError(f"loopback collector failures: {failures}") + if len(requests) != _EXPECTED_REQUEST_COUNT: + raise AssertionError( + f"expected {_EXPECTED_REQUEST_COUNT} OTLP requests, received {len(requests)}" + ) + + for request_index, request in enumerate(requests, 1): + if request.method != "POST" or request.path != "/v1/traces": + raise AssertionError( + f"unexpected OTLP route: {request.method} {request.path}" + ) + header_names = set(request.headers) + if header_names != _EXPECTED_WIRE_HEADERS: + raise AssertionError( + f"unexpected OTLP wire headers: {sorted(header_names)}" + ) + if request.headers["content-type"] != "application/x-protobuf": + raise AssertionError("OTLP request is not binary protobuf") + if int(request.headers["content-length"]) != len(request.body): + raise AssertionError("OTLP content-length does not match its body") + _assert_unique_otlp_attribute_keys(request.body, request_index) + + bodies = b"".join(request.body for request in requests) + header_values = "\n".join( + value for request in requests for value in request.headers.values() + ).encode() + captured_content = ( + _PROMPT_SECRET, + _MODEL_OUTPUT_SECRET, + _TOOL_ARGUMENT_SECRET, + _TOOL_RESULT_SECRET, + _MODEL_WRAPPER_OUTPUT, + _TOOL_MESSAGE_OUTPUT, + ) + for sentinel in captured_content: + if sentinel.encode() not in bodies: + raise AssertionError(f"expected captured content {sentinel} is absent from OTLP") + relay_json_content = ( + observability._OUT_OF_RANGE_INTEGER, + "before\ufffdafter", + ) + for sentinel in relay_json_content: + if sentinel.encode() not in bodies: + raise AssertionError( + f"normalized Relay JSON content {sentinel} is absent from OTLP" + ) + + excluded = ( + _EXCEPTION_SECRET, + _AMBIENT_EXPORTER_SECRET, + _DROPPED_REQUEST_SURFACE_SECRET, + _OPAQUE_ARTIFACT_SECRET, + _TRUNCATION_SENTINEL, + ) + for sentinel in excluded: + encoded = sentinel.encode() + if encoded in bodies or encoded in header_values: + raise AssertionError(f"sensitive {sentinel} reached the OTLP request") + for sentinel in captured_content: + if sentinel.encode() in header_values: + raise AssertionError(f"captured content {sentinel} reached OTLP HTTP headers") + + stable_message = observability._REDACTED_EXCEPTION_MESSAGE.encode() + if stable_message not in bodies or _STABLE_ERROR_CODE.encode() not in bodies: + raise AssertionError("stable redacted error code is absent from OTLP") + if observability._SERVICE_NAME.encode() not in bodies: + raise AssertionError("managed service name is absent from OTLP") + + for name, safe_value in _safe_names(observability, raw_names).items(): + if safe_value.encode() not in bodies: + raise AssertionError(f"sanitized {name} identifier is absent from OTLP") + if raw_names[name].encode() in bodies: + raise AssertionError(f"raw {name} identifier reached OTLP") + + return len(bodies) + + +def _set_validation_environment(canary_endpoint: str) -> dict[str, str | None]: + values = { + "NEMOCLAW_OBSERVABILITY": "1", + "LANGCHAIN_TRACING": "false", + "LANGCHAIN_TRACING_V2": "false", + "LANGSMITH_TRACING": "false", + "LANGSMITH_TRACING_V2": "false", + "OTEL_ENABLED": "true", + "OTEL_SDK_DISABLED": "true", + "OTEL_SERVICE_NAME": _AMBIENT_EXPORTER_SECRET, + "OTEL_RESOURCE_ATTRIBUTES": ( + f"service.name={_AMBIENT_EXPORTER_SECRET},ambient.secret=" + f"{_AMBIENT_EXPORTER_SECRET}" + ), + "OTEL_TRACES_SAMPLER": "always_off", + "OTEL_EXPORTER_OTLP_ENDPOINT": canary_endpoint, + "OTEL_EXPORTER_OTLP_HEADERS": ( + f"authorization={_AMBIENT_EXPORTER_SECRET}" + ), + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": canary_endpoint, + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": ( + f"x-api-key={_AMBIENT_EXPORTER_SECRET}" + ), + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", + "OTEL_EXPORTER_OTLP_TIMEOUT": "999999", + "OTEL_EXPORTER_OTLP_CERTIFICATE": ( + f"/nonexistent/{_AMBIENT_EXPORTER_SECRET}/ca.pem" + ), + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE": ( + f"/nonexistent/{_AMBIENT_EXPORTER_SECRET}/client.pem" + ), + "OTEL_EXPORTER_OTLP_CLIENT_KEY": ( + f"/nonexistent/{_AMBIENT_EXPORTER_SECRET}/client.key" + ), + } + previous = {name: os.environ.get(name) for name in values} + os.environ.update(values) + return previous + + +def _restore_environment(previous: dict[str, str | None]) -> None: + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def main() -> None: + observability = _load_observability_module() + relay_version = importlib.metadata.version("nemo-relay") + if relay_version != _EXPECTED_RELAY_VERSION: + raise AssertionError( + f"expected nemo-relay {_EXPECTED_RELAY_VERSION}, found {relay_version}" + ) + langgraph_version = importlib.metadata.version("langgraph") + if langgraph_version != _EXPECTED_LANGGRAPH_VERSION: + raise AssertionError( + f"expected langgraph {_EXPECTED_LANGGRAPH_VERSION}, found {langgraph_version}" + ) + if observability._OTLP_ENDPOINT != _EXPECTED_PRODUCTION_ENDPOINT: + raise AssertionError( + f"unexpected production OTLP endpoint: {observability._OTLP_ENDPOINT}" + ) + + raw_names = { + "model": _raw_identifier("model"), + "sync_tool": _raw_identifier("sync-tool"), + "async_tool": _raw_identifier("async-tool"), + "graph": _raw_identifier("graph"), + } + _safe_names(observability, raw_names) + + collector = _CollectorServer() + canary = _CollectorServer() + collector_thread = threading.Thread(target=collector.serve_forever, daemon=True) + canary_thread = threading.Thread(target=canary.serve_forever, daemon=True) + collector_thread.start() + canary_thread.start() + original_endpoint = observability._OTLP_ENDPOINT + previous_environment = _set_validation_environment( + f"http://127.0.0.1:{canary.server_port}/v1/traces" + ) + initialized = False + try: + observability._OTLP_ENDPOINT = ( + f"http://127.0.0.1:{collector.server_port}/v1/traces" + ) + initialized = observability.initialize_observability() + if not initialized or observability._lifecycle.subscriber is None: + raise AssertionError("real Relay observability failed to initialize") + + _assert_callback_manager_boundary(observability) + _assert_capture_traversal_bounds(observability) + middleware = observability.new_relay_middleware() + asyncio.run( + _exercise_async_boundaries(observability, middleware, raw_names) + ) + _exercise_sync_tool(middleware, raw_names["sync_tool"]) + _exercise_framework_result_transparency(middleware) + _exercise_real_relay_json_domain(middleware) + _exercise_relay_failure_transparency(middleware) + _exercise_graph(observability, raw_names["graph"]) + + nemo_relay.subscribers.flush() + observability._lifecycle.subscriber.force_flush() + requests, failures = collector.snapshot() + canary_requests, canary_failures = canary.snapshot() + if canary_requests or canary_failures: + raise AssertionError("ambient OTLP canary received managed telemetry") + total_bytes = _assert_wire_requests( + requests, failures, observability, raw_names + ) + print( + "Validated real NeMo Relay observability: " + f"relay={relay_version} langgraph={langgraph_version} " + f"requests={len(requests)} bytes={total_bytes}" + ) + finally: + if initialized: + observability.shutdown_observability() + observability._OTLP_ENDPOINT = original_endpoint + _restore_environment(previous_environment) + collector.shutdown() + collector.server_close() + canary.shutdown() + canary.server_close() + collector_thread.join(timeout=5) + canary_thread.join(timeout=5) + if collector_thread.is_alive() or canary_thread.is_alive(): + raise RuntimeError("loopback OTLP collectors did not stop") + + +if __name__ == "__main__": + main() diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index e01270db7f..162942308f 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -11,6 +11,6 @@ "test/onboard-messaging.test.ts": 2062, "test/onboard-selection.test.ts": 4834, "test/onboard.test.ts": 4043, - "test/policies.test.ts": 2279 + "test/policies.test.ts": 2243 } } diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index 84e826f0ff..3ae6c415bd 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -3,11 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 title: "Quickstart with LangChain Deep Agents Code" sidebar-title: "Quickstart with Deep Agents" -description: "Create a NemoClaw sandbox that runs LangChain Deep Agents Code as a terminal harness." -description-agent: "Creates a NemoClaw sandbox that runs LangChain Deep Agents Code as a terminal harness. Use when testing dcode with NemoClaw-managed inference." -keywords: ["langchain deep agents code nemoclaw", "dcode openshell sandbox", "langchain coding agent"] -topics: ["get-started", "terminal-runtime", "langchain-deepagents-code"] -tags: ["deep-agents-code", "dcode", "managed-inference"] +description: "Create and operate a NemoClaw sandbox for LangChain Deep Agents Code with managed inference and backend-neutral OTLP tracing." +description-agent: "Creates and operates a NemoClaw sandbox that runs LangChain Deep Agents Code with managed inference and backend-neutral OTLP trace export through a host-side collector, including LangSmith. Use when installing or testing dcode, enabling observability, or configuring a local OpenTelemetry collector for LangSmith." +keywords: ["langchain deep agents code nemoclaw", "dcode openshell sandbox", "langchain coding agent", "dcode otlp tracing"] +topics: ["get-started", "terminal-runtime", "langchain-deepagents-code", "observability"] +tags: ["deep-agents-code", "dcode", "managed-inference", "otlp"] difficulty: "intermediate" audience: "operators" status: published @@ -109,7 +109,7 @@ dcode -n "Summarize this repository" ``` The managed `dcode`, `dcode.real`, and `deepagents-code` launchers use `/opt/venv/bin/python3 -I` to run the pinned package with an isolated import path and `HOME=/sandbox`. -They disable Deep Agents Code package update checks and the LangGraph server version check, block CLI and TUI update/install commands, and disable nested remote sandbox providers, remote async subagents, MCP commands and project auto-loading, startup commands, executable hooks, ACP mode, interpreter tool calling, shell allow-list overrides, LangSmith tracing, and OpenTelemetry export. +They disable Deep Agents Code package update checks and the LangGraph server version check, block CLI and TUI update/install commands, and disable nested remote sandbox providers, remote async subagents, MCP commands and project auto-loading, startup commands, executable hooks, ACP mode, interpreter tool calling, shell allow-list overrides, native LangSmith tracing, and ambient OpenTelemetry exporter configuration. The managed model constructor accepts only Deep Agents Code's `openai` provider path and reads its endpoint from a root-owned image file. It supplies the non-secret gateway placeholder key and ignores mutable provider classes, credentials, endpoints, and constructor parameters in Deep Agents Code config. CLI and TUI model parameter overrides and custom rubric models are blocked. @@ -167,7 +167,7 @@ Before a managed Deep Agents Code rebuild changes the sandbox, NemoClaw selects Initial failures stop before backup. After backup, NemoClaw rechecks the target, route, and retained build inputs before changing MCP state, then checks again after MCP preparation and before stopping inference or deleting the old sandbox. If the final check fails, NemoClaw restores the previous MCP state and keeps the existing sandbox intact. -Rebuild also preserves the standalone Deep Agents Code `tavily` preset and replays recorded custom policies from their exact stored content. +Rebuild also preserves the standalone Deep Agents Code `tavily` preset, the recorded observability choice unless explicitly overridden, and recorded custom policies from their exact stored content. ## Optional Tavily Egress @@ -178,8 +178,8 @@ The gateway injects it at egress instead. The managed Deep Agents Code entry points reject credential-shaped process environment values, disable project `.env` and global `/sandbox/.deepagents/.env` loading, and block upstream `/auth`, `/connect`, startup/onboarding credential prompts, model-selector credential prompts, notification-service key prompts, and ChatGPT OAuth. These controls apply to Deep Agents Code and do not sanitize arbitrary Python programs in the sandbox. Use NemoClaw-managed credential paths when support is available instead of storing service keys inside Deep Agents Code state. -NemoClaw does not enable Tavily or LangSmith by default for this harness. -The sandbox policy denies `api.tavily.com` and `api.smith.langchain.com` until you opt in. +NemoClaw does not enable Tavily or observability by default for this harness. +The sandbox policy denies `api.tavily.com` until you opt into Tavily and continues to deny direct `api.smith.langchain.com` egress when you enable observability. To allow Tavily egress for the target sandbox, apply the maintained `tavily` policy preset, register the credential with the OpenShell gateway, then rebuild the sandbox so the new provider attaches. The policy preset is a per-sandbox managed-Python opt-in, but provider registration is gateway-wide: `tavily-search` attaches to every sandbox that you build or rebuild afterward. @@ -215,14 +215,380 @@ This does not unregister the gateway-wide `tavily-search` provider; its credenti When no sandbox needs the provider, destroy those sandboxes or detach it from each one with `openshell sandbox provider detach tavily-search`, then remove it globally with `nemo-deepagents credentials reset tavily-search --yes`. OpenShell rejects provider deletion while any sandbox still has it attached. -### Tracing (LangSmith and OpenTelemetry) +## Export Traces Through a Local Collector + +NemoClaw can export Deep Agents Code traces to an OTLP/HTTP collector that you operate on the host. +The sandbox always targets one local collector address, while the collector owns the remote backend, credentials, TLS, batching, retry, and optional filtering. +Changing from LangSmith to another OTLP-compatible backend does not require a sandbox rebuild or policy change. + +The complete example below uses the primary tested NemoClaw platform, Linux with Docker, and the official OpenTelemetry Collector Contrib `0.155.0` image. +It binds the unauthenticated receiver only to the private Docker bridge address that the target sandbox resolves as `host.openshell.internal`. +Do not publish this receiver as `0.0.0.0:4318` on the host. +For general image and configuration-file mechanics, refer to [Install the Collector with Docker](https://opentelemetry.io/docs/collector/install/docker/). + +### Understand the Export Boundary + +Trace export is off by default and requires an explicit onboarding choice. +When enabled, the managed exporter can include bounded prompts, model responses, tool arguments, tool results, operation names, model and tool names, and success or error information. +Treat the resulting traces as sensitive application data even when your normal prompts do not contain secrets. + +Managed capture limits each string to 8,000 characters, each mapping or sequence to 50 items, and nesting to 8 levels. +Each captured value also has an aggregate budget of 2,048 traversed nodes and 50,000 source string characters. +Repeated or cyclic containers are replaced with a reference-omission marker instead of being expanded again. +After bounding a value, a JSON encoding longer than 50,000 characters is replaced by a constant opaque marker and a 16,000-character serialized preview. +Other opaque objects are replaced by the same constant marker without reading their class name or string representation. +It replaces binary values with their byte count and redacts recognized credential, header, cookie, password, token, checkpoint, resume, and interrupt keys. +It also replaces original exception text with a stable redacted error. +Model request traces include only the bounded messages and sanitized model identifier. +They exclude request headers, `model_settings`, `response_format`, and tool definitions or schemas. +Model and tool spans carry the bounded content used for debugging. +LangGraph node scopes export only bounded node names, a static LangGraph integration label, and success or error status. +They omit raw graph inputs and outputs, callback metadata, checkpoint payloads, and interrupt or resume values so a node span does not duplicate the full conversation and graph state. +These controls bound payload shape and remove recognized key classes, but they do not classify arbitrary text values. +A secret pasted into a prompt, model response, shell command, tool result, file body, or unrecognized field can still be exported. + +The sandbox sends OTLP/HTTP protobuf requests only to `http://host.openshell.internal:4318/v1/traces` and reports `service.name=nemoclaw-langchain-deepagents-code`. +The OTLP library adds standard transport headers such as content type and content length, but the managed exporter cannot add operator-supplied custom or authentication headers. +It cannot select a remote endpoint or receive a backend credential. +Native LangSmith tracing and ambient OpenTelemetry exporter configuration remain disabled inside the sandbox. +Do not put `LANGSMITH_API_KEY` or another backend credential in the sandbox. + +Exporter initialization, delivery, and flush failures do not stop Deep Agents Code work. +This fail-open behavior keeps tracing outages from blocking the agent, but it also means successful agent work does not prove that traces were delivered. + + +The `observability-otlp-local` preset authorizes `/opt/venv/bin/python3*`, which is the executable OpenShell observes for Deep Agents Code. +That permission is process-wide for the managed Python environment rather than limited to the `dcode` launcher. +Sandbox Python can forge spans, resource attributes, and `service.name`, so the collector must not use trace fields as authenticated tenant identity. +Any process that can reach this receiver can submit trace content without a receiver credential. +Bind the receiver only to the private sandbox bridge, use it only with trusted local sandboxes, and apply your organization's filtering or redaction requirements in the host collector before remote export. +This path is not a multi-tenant identity or data-loss-prevention boundary. + + +### Enable Trace Export + +Set the sandbox name in the host shell, then opt in during initial onboarding. -NemoClaw does not support LangSmith or OpenTelemetry tracing for this managed harness. -`start.sh` does not persist user-supplied tracing credentials, projects, or replica endpoints in the shared shell environment. -No policy preset opens `api.smith.langchain.com`, and no supported mechanism injects `LANGSMITH_API_KEY`. -The managed launch paths force the supported LangSmith and LangChain tracing enable flags, including their `V2` and `DEEPAGENTS_CODE_`-prefixed variants, to `false`. -They force `OTEL_ENABLED=false`, reject OTLP exporter endpoints and headers, remove those variables from the LangGraph child process, and reject LangSmith and LangChain replica endpoint configuration. -Adding egress endpoints alone does not enable tracing; treat it as unsupported until NemoClaw ships a maintained integration. +```bash +export SANDBOX_NAME=my-dcode +nemo-deepagents onboard --name "$SANDBOX_NAME" --observability +``` + +NemoClaw records the choice with the onboarding session and sandbox. +Resume and rebuild operations preserve it without requiring the flag again. + +To enable tracing on an existing Deep Agents Code sandbox, use the transactional rebuild opt-in. +NemoClaw backs up the declared agent state, preserves managed MCP providers and adapter state, recreates the sandbox, and restores the backup. +Finish active `dcode` tasks first because Deep Agents Code backup refuses to capture state while a task is running. + +```bash +export SANDBOX_NAME=my-dcode +nemo-deepagents "$SANDBOX_NAME" rebuild --observability --yes +``` + +The setting is part of the sandbox startup environment, so changing it cannot reuse the existing sandbox process. + +### Recover a Skipped Policy + +Balanced and Open policy tiers add the `observability-otlp-local` preset during onboarding. +The Restricted tier suppresses it. +`NEMOCLAW_POLICY_MODE=skip` skips policy application and reconciliation during non-interactive onboarding. +On a new sandbox, either choice leaves the preset inactive, so the fail-open exporter cannot reach the collector until you add it. +On an existing sandbox, skip mode leaves the current live policy unchanged. + +Inspect the effective policy first. + +```bash +nemo-deepagents "$SANDBOX_NAME" policy-list +``` + +If `observability-otlp-local` is not active, preview and apply it. + +```bash +nemo-deepagents "$SANDBOX_NAME" policy-add observability-otlp-local --dry-run +nemo-deepagents "$SANDBOX_NAME" policy-add observability-otlp-local --yes +``` + +The preset permits only `POST /v1/traces` to `host.openshell.internal:4318` from `/opt/venv/bin/python3*`. +On the Restricted tier, the next onboarding or rebuild reconciliation removes this manually added preset unless you change tiers. + +### Create LangSmith Credentials + +LangSmith is one possible downstream backend for the same backend-neutral receiver. +Create a workspace-scoped service key for the collector when your LangSmith plan supports service keys. +Otherwise, create a personal access token for the collector. +Record the target workspace ID from LangSmith settings because this configuration sends `X-Tenant-Id` explicitly. +For key types, permissions, and the workspace ID location, refer to [Create an account and API key](https://docs.langchain.com/langsmith/create-account-api-key). + +Set the credential only in the host shell that starts the collector. +The following endpoint is for the default US LangSmith Cloud deployment. + +```bash +read -rsp "LangSmith API key: " LANGSMITH_API_KEY +printf '\n' +export LANGSMITH_API_KEY +export LANGSMITH_WORKSPACE_ID='replace-with-workspace-id' +export LANGSMITH_PROJECT=nemoclaw-dcode +export LANGSMITH_OTLP_TRACES_ENDPOINT=https://api.smith.langchain.com/otel/v1/traces +``` + +Use `https://eu.api.smith.langchain.com/otel/v1/traces` for the EU deployment, `https://apac.api.smith.langchain.com/otel/v1/traces` for the GCP-hosted APAC deployment, or `https://aws.api.smith.langchain.com/otel/v1/traces` for the AWS-hosted US deployment. +For a self-hosted deployment, append `/api/v1/otel/v1/traces` to the LangSmith instance origin, for example `https://langsmith.example.com/api/v1/otel/v1/traces`. +The self-hosted OTLP base is `/api/v1/otel`, and the per-signal traces exporter adds `/v1/traces`. +The `X-Tenant-Id` header in the collector configuration selects the workspace and is required for organization-scoped service keys. +For current endpoint guidance and supported OpenTelemetry field mappings, refer to [Trace with OpenTelemetry](https://docs.langchain.com/langsmith/trace-with-opentelemetry). + +### Find the Private Host Bind Address + +Resolve `host.openshell.internal` from the target sandbox, then verify that the resulting private IPv4 address belongs to a host interface. +This recipe intentionally supports the Linux Docker topology and fails closed if the address is empty, public, or not assigned to the host. + +```bash +OTLP_BIND_IP="$( + nemo-deepagents "$SANDBOX_NAME" exec -- \ + sh -lc "getent ahostsv4 host.openshell.internal | awk 'NR == 1 { print \$1 }'" +)" + +if [ -z "$OTLP_BIND_IP" ]; then + printf '%s\n' 'Could not resolve host.openshell.internal from the sandbox.' >&2 + exit 1 +fi + +case "$OTLP_BIND_IP" in + 10.*|192.168.*|172.1[6-9].*|172.2[0-9].*|172.3[01].*) ;; + *) + printf 'Refusing non-private collector bind address: %s\n' "$OTLP_BIND_IP" >&2 + exit 1 + ;; +esac + +if ! ip -o -4 address show \ + | awk '{ sub(/\/.*/, "", $4); print $4 }' \ + | grep -Fxq "$OTLP_BIND_IP"; then + printf 'Address is not assigned to this host: %s\n' "$OTLP_BIND_IP" >&2 + exit 1 +fi + +printf 'Collector bind address: %s\n' "$OTLP_BIND_IP" +``` + +Binding to the bridge address prevents ordinary LAN exposure, but other trusted local containers can still have a route to it. +Use a host firewall or equivalent ACL if your Docker host runs containers outside your trust boundary. +NemoClaw does not support shared multi-user hosts as a security boundary. + +### Configure the Collector + +Create an owner-only directory for the collector configuration. + +```bash +export OTEL_CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/nemoclaw/otel" +install -d -m 700 "$OTEL_CONFIG_DIR" +``` + +Save the following configuration as `$OTEL_CONFIG_DIR/collector.yaml`. +The `debug` exporter uses basic verbosity, which records receipt counts in the collector log without printing complete span payloads. +This example deliberately preserves the useful trace content selected by the explicit sandbox opt-in. +Add organization-required filtering or redaction processors before `batch` if your data policy requires them. +For a LangSmith-specific example, refer to [Trace redaction through an OpenTelemetry Collector](https://docs.langchain.com/langsmith/otel-gateway-trace-redaction). + +```yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: {} + +exporters: + debug: + verbosity: basic + otlphttp/langsmith: + traces_endpoint: "${env:LANGSMITH_OTLP_TRACES_ENDPOINT}" + headers: + x-api-key: "${env:LANGSMITH_API_KEY}" + Langsmith-Project: "${env:LANGSMITH_PROJECT}" + X-Tenant-Id: "${env:LANGSMITH_WORKSPACE_ID}" + sending_queue: + enabled: true + queue_size: 1000 + retry_on_failure: + enabled: true + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [debug, otlphttp/langsmith] +``` + +Restrict the configuration file to the current host user. + +```bash +chmod 600 "$OTEL_CONFIG_DIR/collector.yaml" +``` + +The receiver uses `0.0.0.0` only inside the collector container. +The Docker command in the next section publishes it on the host's exact private bridge address rather than every host interface. + +### Start and Verify the Collector + +Pull the pinned Contrib image and validate the effective configuration before starting a long-running collector. +The Contrib distribution is used because it contains the processors and extensions in this configuration. +The commands run the collector as your current host user so it can read the owner-only bind-mounted configuration without widening its file mode. + +```bash +export COLLECTOR_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.155.0@sha256:4935caa35e9a4cb387e35732e8fb22b2b5759af8d12e7043357f03837f6e8df5 +: "${LANGSMITH_API_KEY:?Set LANGSMITH_API_KEY in this host shell.}" +: "${LANGSMITH_WORKSPACE_ID:?Set LANGSMITH_WORKSPACE_ID in this host shell.}" +: "${LANGSMITH_PROJECT:?Set LANGSMITH_PROJECT in this host shell.}" +: "${LANGSMITH_OTLP_TRACES_ENDPOINT:?Set LANGSMITH_OTLP_TRACES_ENDPOINT in this host shell.}" +: "${OTLP_BIND_IP:?Run the private bind address step first.}" +: "${OTEL_CONFIG_DIR:?Run the collector configuration step first.}" +docker pull "$COLLECTOR_IMAGE" +docker run --rm \ + --user "$(id -u):$(id -g)" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --memory 512m \ + --pids-limit 128 \ + --env LANGSMITH_API_KEY \ + --env LANGSMITH_WORKSPACE_ID \ + --env LANGSMITH_PROJECT \ + --env LANGSMITH_OTLP_TRACES_ENDPOINT \ + --mount "type=bind,src=${OTEL_CONFIG_DIR}/collector.yaml,dst=/etc/otelcol-contrib/config.yaml,readonly" \ + "$COLLECTOR_IMAGE" \ + validate --config=/etc/otelcol-contrib/config.yaml +``` + +An invalid configuration exits nonzero and prints the component or field that needs correction. +Start the collector only after validation succeeds. + +```bash +docker run --detach \ + --name nemoclaw-otel-langsmith \ + --restart unless-stopped \ + --user "$(id -u):$(id -g)" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --memory 512m \ + --pids-limit 128 \ + --log-opt max-size=10m \ + --log-opt max-file=3 \ + --publish "${OTLP_BIND_IP}:4318:4318/tcp" \ + --publish 127.0.0.1:13133:13133/tcp \ + --env LANGSMITH_API_KEY \ + --env LANGSMITH_WORKSPACE_ID \ + --env LANGSMITH_PROJECT \ + --env LANGSMITH_OTLP_TRACES_ENDPOINT \ + --mount "type=bind,src=${OTEL_CONFIG_DIR}/collector.yaml,dst=/etc/otelcol-contrib/config.yaml,readonly" \ + "$COLLECTOR_IMAGE" \ + --config=/etc/otelcol-contrib/config.yaml + +unset LANGSMITH_API_KEY +``` + +Users who can control the Docker daemon can inspect the collector process and its environment. +Use your organization's container secret injection mechanism instead of environment variables when Docker operator access is outside the credential trust boundary. + +Verify the loopback-only health endpoint and the published receiver address. + +```bash +curl -fsS http://127.0.0.1:13133/ +docker port nemoclaw-otel-langsmith 4318/tcp +docker logs --tail 30 nemoclaw-otel-langsmith +``` + +The port output must show the value of `$OTLP_BIND_IP`, not `0.0.0.0` or `[::]`. + +### Verify Traces End to End + +Run a short headless Deep Agents Code task to generate a managed trace. + +```bash +nemo-deepagents "$SANDBOX_NAME" exec -- \ + dcode -n "Reply with the single word traced." +``` + +Confirm that the collector received a trace batch and did not report a LangSmith exporter error. + +```bash +docker logs --since 5m nemoclaw-otel-langsmith 2>&1 | tail -n 100 +``` + +The basic debug exporter prints a trace count without printing the full payload. +Then open the project named by `$LANGSMITH_PROJECT` in the [LangSmith UI](https://smith.langchain.com/) and confirm that the new trace is present. +Both checks are required because the debug exporter can succeed while the remote LangSmith exporter fails. +The LangSmith trace should include bounded model inputs and outputs rather than metadata alone. +A representative task that invokes a tool should also show its bounded arguments and result on the associated tool span. +LangGraph node scopes intentionally remain operation-only as described in the export boundary above. + +### Stop or Disable Trace Export + +NemoClaw does not start, stop, upgrade, or remove this operator-owned collector container. +Stop and restart the host collector without changing the sandbox configuration. +While the collector is stopped, trace delivery fails open and Deep Agents Code continues working. + +```bash +docker stop nemoclaw-otel-langsmith +docker start nemoclaw-otel-langsmith +``` + +To revoke the sandbox's collector reachability immediately, remove the policy preset. +This does not change the recorded observability choice. +A later rebuild restores the matching preset on Balanced and Open tiers, while Restricted continues to suppress it. + +```bash +nemo-deepagents "$SANDBOX_NAME" policy-remove observability-otlp-local --yes +``` + +To disable instrumentation persistently, use the transactional rebuild negative flag. +NemoClaw performs the state-preserving recreation while preserving managed MCP providers and adapter state. +Finish active `dcode` tasks before running the command. + +```bash +nemo-deepagents "$SANDBOX_NAME" rebuild --no-observability --yes +``` + +Remove the collector only after every sandbox that uses it has been disabled or had the policy removed. + +```bash +docker rm -f nemoclaw-otel-langsmith +``` + +Recreate the collector container after changing its configuration, endpoint, project, workspace, or API key. + +### Troubleshoot Trace Export + +Use the following checks to isolate each hop in the export path. +For additional collector diagnostics, refer to [Troubleshooting the OpenTelemetry Collector](https://opentelemetry.io/docs/collector/troubleshooting/). + +| Symptom | Check and action | +| --- | --- | +| Collector exits at startup | Run the validation command again, then inspect `docker logs nemoclaw-otel-langsmith`. Confirm that the image is the Contrib `0.155.0` image and that all four `LANGSMITH_*` variables were set when the container was created. | +| Port `4318` is already allocated | Run `ss -ltnp 'sport = :4318'` and stop the conflicting listener. The managed sandbox endpoint is fixed, so changing the collector port does not work. | +| Collector is healthy but logs no trace count | Run `policy-list` and add `observability-otlp-local` if it is absent. Confirm that `docker port` shows the current `$OTLP_BIND_IP`. Run `nemo-deepagents rebuild --observability --yes` if the existing sandbox was originally started without the opt-in. | +| Collector logs `401` | Replace an invalid or expired LangSmith API key, then recreate the collector container. | +| Collector logs `403` | Confirm the service key can write to the target workspace and that `LANGSMITH_WORKSPACE_ID` matches that workspace. Organization-scoped service keys require `X-Tenant-Id`. | +| Collector logs `404` | Confirm that `LANGSMITH_OTLP_TRACES_ENDPOINT` uses the correct US, EU, GCP-hosted APAC, AWS-hosted US, or self-hosted API base and ends in `/otel/v1/traces`. | +| Collector logs `429` | Review LangSmith ingestion and plan limits, then allow the configured sending queue and retry policy to drain. | +| Debug exporter logs traces but the LangSmith project is empty | Inspect the same collector log for remote exporter errors, then verify the endpoint, project, workspace ID, and API key. | +| Agent succeeds while every trace check fails | This is expected fail-open behavior. Troubleshoot the policy, receiver bind, collector health, and remote exporter independently rather than using the agent exit status as delivery evidence. | ## Troubleshooting @@ -249,5 +615,7 @@ There is no dashboard port or long-running gateway process for this harness. - [Inference Options](../inference/inference-options) explains how to choose a provider and model. - [Workspace Files](../manage-sandboxes/workspace-files) explains `/sandbox/.deepagents`, memory, skills, and what NemoClaw preserves. - [Backup and Restore](../manage-sandboxes/backup-restore) explains snapshot and rebuild preservation. +- [Network Policies](../reference/network-policies#local-otlp-trace-export) explains the local collector egress preset. +- [Troubleshooting](../reference/troubleshooting) covers common setup and runtime issues. - [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers) explains managed MCP configuration for Deep Agents sandboxes. - [Deep Agents Code overview](https://docs.langchain.com/oss/python/deepagents/code/overview) explains upstream `dcode` capabilities and commands. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 82d599bb40..ef8ca993bd 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -168,7 +168,7 @@ The wizard creates an OpenShell gateway, registers inference providers, builds t Use this command for new installs and for recreating a sandbox after changes to policy or configuration. ```bash -$$nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--control-ui-port ] [--yes | -y] [--no-ollama-autostart] [--yes-i-accept-third-party-software] +$$nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--observability | --no-observability] [--control-ui-port ] [--yes | -y] [--no-ollama-autostart] [--yes-i-accept-third-party-software] ``` @@ -198,7 +198,7 @@ For example, `nemohermes` resolves to `hermes`, while `dcode`, `deepagents`, `de #### `--resume` and `--fresh` NemoClaw records onboarding progress so interrupted runs can continue. -Use `--resume` to continue a resumable onboarding session with the provider, model, sandbox name, agent, and custom Dockerfile path recorded by the original run. +Use `--resume` to continue a resumable onboarding session with the provider, model, sandbox name, agent, observability choice, and custom Dockerfile path recorded by the original run. Completed onboarding sessions are not resumable. Use `--resume` only for interrupted `in_progress` sessions, not to change provider, model, agent, or sandbox recreation settings after onboarding has completed. During resume, NemoClaw reruns preflight, gateway, provider, and sandbox repair checks even when the saved session has already reached a later nonterminal onboarding phase. @@ -233,6 +233,47 @@ $$nemoclaw onboard --name my-assistant --recreate-sandbox --tool-disclosure dire Without an explicit flag or environment value, recreation preserves the recorded setting and only falls back to `progressive` for legacy state. Resuming an interrupted session with a different explicit setting fails with a conflict instead of changing behavior mid-session. + + +#### `--observability` and `--no-observability` + +Enable backend-neutral trace export for a LangChain Deep Agents Code sandbox. +During initial onboarding, pass `--observability` with the Deep Agents alias. +When you use the generic `nemoclaw` entry point, combine it with `--agent langchain-deepagents-code`. +NemoClaw rejects the positive flag for OpenClaw and Hermes sandboxes. +Use `--no-observability` when you need to clear a recorded Deep Agents Code choice before switching the resumed session to another agent. + +```bash +$$nemoclaw onboard --observability +nemoclaw onboard --agent langchain-deepagents-code --observability +``` + +The flag is off by default. +When enabled, NemoClaw records the choice with the onboarding session and sandbox, adds the `observability-otlp-local` policy preset on supported policy tiers, and preserves the choice across resume and rebuild operations. +An explicit `--observability` or `--no-observability` choice updates a resumed onboarding session. +The Restricted tier suppresses automatic application of the preset. +An operator can add it manually after reviewing the additional egress, but the next Restricted onboarding or rebuild reconciliation removes it. + +The explicit opt-in can export bounded prompts, responses, tool arguments, tool results, and operational metadata. +Treat trace payloads as sensitive application data. +The managed capture applies size, depth, item-count, recognized-key, and exception-text safeguards, but it does not detect secrets embedded in ordinary content values. +Deep Agents Code sends OTLP/HTTP protobuf traces to the fixed local endpoint `http://host.openshell.internal:4318/v1/traces`. +The OTLP library adds standard transport headers, but the sandbox cannot configure operator-supplied custom or authentication headers, a remote endpoint, backend credentials, or a backend. + +Changing this setting on an existing sandbox requires a new sandbox process so the startup environment matches the recorded choice. +Use the transactional rebuild flags so NemoClaw backs up declared agent state, preserves managed MCP providers and adapter state, recreates the sandbox, and restores the backup. + +```bash +$$nemoclaw my-dcode rebuild --observability --yes +$$nemoclaw my-dcode rebuild --no-observability --yes +``` + +Removing the `observability-otlp-local` policy stops delivery immediately but does not clear the recorded opt-in. +A later rebuild restores the preset on Balanced and Open tiers, while Restricted continues to suppress it. +For the complete collector setup, privacy boundary, policy recovery, verification, and a host-side LangSmith exporter example, refer to [Quickstart with LangChain Deep Agents Code](/user-guide/deepagents/get-started/quickstart#export-traces-through-a-local-collector). + + + When Docker exposes the required identity metadata, NemoClaw records the base-image resolution on managed sandbox images. During a warm recreate or rebuild, it validates the local image identity and platform, plus the exact repository digest for a published image and any active OpenShell ABI requirement, before reusing it. A valid match avoids candidate discovery and a network pull. @@ -2216,11 +2257,12 @@ Policy presets applied to the old sandbox are reapplied to the new one so your e The replacement uses the recorded compatible-endpoint reasoning mode and web search selection instead of ambient shell values. The recorded sandbox GPU mode is preserved across rebuild. A rebuild preserves the recorded tool-disclosure mode unless `--tool-disclosure` explicitly changes it; it ignores an ambient `NEMOCLAW_TOOL_DISCLOSURE` value while recreating the sandbox. +A rebuild preserves the recorded Deep Agents Code observability choice and matching local OTLP policy state unless `--observability` or `--no-observability` explicitly changes them. A sandbox onboarded with an explicit GPU opt-out (stored as `sandboxGpuMode: "0"`, plus legacy registry entries that only record `gpuEnabled: false`) is recreated with the same opt-out, so the inner `onboard --resume` skips the Docker CDI GPU preflight on hosts without an NVIDIA GPU. Auto-mode sandboxes remain auto. ```bash -$$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] +$$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] [--observability|--no-observability] ``` | Flag | Description | @@ -2228,6 +2270,7 @@ $$nemoclaw my-assistant rebuild [--yes|-y|--force] [--verbose|-v] [--tool-disclo | `--yes`, `-y`, `--force` | Skip the confirmation prompt | | `--verbose`, `-v` | Log SSH commands, exit codes, and session state (also enabled by `NEMOCLAW_REBUILD_VERBOSE=1`) | | `--tool-disclosure ` | Change the model-visible tool catalog during this transactional rebuild. Use this path for sandboxes with managed MCP servers so their providers and adapter state are preserved. | +| `--observability`, `--no-observability` | Enable or disable managed trace export for a LangChain Deep Agents Code sandbox during the transactional rebuild. This path preserves managed MCP providers and adapter state. | If another terminal has an active SSH session to the sandbox, `rebuild` prints an active-session warning and requires confirmation before destroying the sandbox. Pass `--yes`, `-y`, or `--force` to skip the prompt in scripted workflows. @@ -2676,7 +2719,7 @@ The `$$nemoclaw setup` command is deprecated. Use `$$nemoclaw onboard` instead. -This command remains as a compatibility alias to `$$nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--sandbox-gpu` / `--no-sandbox-gpu`, `--sandbox-gpu-device`, `--agent`, `--agents `, `--tool-disclosure `, `--control-ui-port`, `--yes` / `-y`, `--no-ollama-autostart`, `--yes-i-accept-third-party-software`. +This command remains as a compatibility alias to `$$nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--sandbox-gpu` / `--no-sandbox-gpu`, `--sandbox-gpu-device`, `--agent`, `--agents `, `--tool-disclosure `, `--observability` / `--no-observability`, `--control-ui-port`, `--yes` / `-y`, `--no-ollama-autostart`, `--yes-i-accept-third-party-software`. ```bash $$nemoclaw setup @@ -2689,7 +2732,7 @@ The `$$nemoclaw setup-spark` command is deprecated. Use the standard installer and run `$$nemoclaw onboard` instead, because current OpenShell releases handle the older DGX Spark cgroup behavior. -This command remains as a compatibility alias to `$$nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--sandbox-gpu` / `--no-sandbox-gpu`, `--sandbox-gpu-device`, `--agent`, `--agents `, `--tool-disclosure `, `--control-ui-port`, `--yes` / `-y`, `--no-ollama-autostart`, `--yes-i-accept-third-party-software`. +This command remains as a compatibility alias to `$$nemoclaw onboard` and accepts the same flags: `--non-interactive`, `--resume`, `--fresh`, `--recreate-sandbox`, `--gpu` / `--no-gpu`, `--from`, `--name`, `--sandbox-gpu` / `--no-sandbox-gpu`, `--sandbox-gpu-device`, `--agent`, `--agents `, `--tool-disclosure `, `--observability` / `--no-observability`, `--control-ui-port`, `--yes` / `-y`, `--no-ollama-autostart`, `--yes-i-accept-third-party-software`. ```bash $$nemoclaw setup-spark @@ -3183,6 +3226,26 @@ NEMOCLAW_TRACE_FILE=/tmp/nemoclaw-onboard-trace.json $$nemoclaw onboard Trace artifacts include onboard phase timing, sandbox and service readiness waits, policy application, inference validation probes, curl probe results, and sandbox build progress events. Secret-like metadata such as API keys, bearer tokens, cookies, and credentials is redacted before the file is written. + + +### Deep Agents Code OTLP Traces + +Pass `--observability` during Deep Agents onboarding to enable backend-neutral runtime traces for Deep Agents Code. +This feature is separate from `NEMOCLAW_TRACE`, which records NemoClaw onboarding phases, and from the OpenClaw diagnostics plugin. + +The sandbox sends OTLP/HTTP protobuf requests only to `http://host.openshell.internal:4318/v1/traces`. +The managed exporter uses standard OTLP transport headers but does not accept operator-supplied custom or authentication headers. +A host operator must run the receiver on port `4318` and configure any Jaeger, Phoenix, LangSmith, or other backend exporter on the collector side. +Changing the host collector's exporter does not require a sandbox rebuild or policy change. +Collector and exporter failures are non-fatal to agent work. + +Native LangSmith tracing and ambient OTLP configuration remain disabled in the sandbox. +The explicit opt-in can export bounded prompts, responses, tool arguments, tool results, and operational metadata, so operators must treat trace payloads as sensitive application data. +The collector must enforce the operator's filtering and redaction requirements before remote forwarding because the local policy applies to the managed Python interpreter and does not provide authenticated tenant identity. +For the complete receiver contract and a runnable LangSmith collector setup, refer to [Quickstart with LangChain Deep Agents Code](/user-guide/deepagents/get-started/quickstart#export-traces-through-a-local-collector). + + + ### OpenClaw Conversation OTEL Diagnostics diff --git a/docs/reference/enterprise-readiness.mdx b/docs/reference/enterprise-readiness.mdx index c12c175fa0..7f9f388330 100644 --- a/docs/reference/enterprise-readiness.mdx +++ b/docs/reference/enterprise-readiness.mdx @@ -53,9 +53,9 @@ Knowing who enforces each boundary prevents misattributing a limitation to NemoC | Host CLI and onboarding | NemoClaw | Onboarding, provider validation, blueprint resolution, sandbox lifecycle commands, and credential handling on the host. | | Blueprint and policy presets | NemoClaw | Versioned blueprint, baseline network policy, filesystem and process defaults, and integration presets. | | Gateway, sandbox runtime, and egress enforcement | OpenShell | Network namespace isolation, the CONNECT proxy, policy enforcement, inference routing, TLS termination, and structured platform logging. | -| Agent behavior | OpenClaw or Hermes | The agent loop, tools, skills, and in-sandbox configuration. | +| Agent behavior | OpenClaw, Hermes, or Deep Agents Code | The agent loop, tools, skills, and in-sandbox configuration. | | Model inference and data handling | Inference provider | Model execution, per-token cost, rate limits, and provider-side data policies. | -| Operator decisions | You | Endpoint approvals, policy widening, posture choices, provider selection, and credential rotation. | +| Operator decisions | You | Endpoint approvals, policy widening, posture choices, provider selection, credential rotation, and host observability collector configuration. | For the architecture behind these boundaries, refer to [How It Works](../about/how-it-works) and [Architecture Details](architecture). @@ -72,7 +72,7 @@ Each row links to deeper documentation and, when a concrete fix is in progress, | Model and provider switching | Supported | Switch the active provider or model with the NemoClaw inference commands. Some changes rebuild the sandbox image. Refer to [Switch Inference Providers](../inference/switch-inference-providers) and [Inference Options](../inference/inference-options). | | Multi-agent and multi-sandbox usage | Supported with caveats | Side-by-side sandboxes run on distinct names and dashboard ports, and each name maps to exactly one agent type. Known multi-instance issues include gateway-port collisions ([#5359](https://github.com/NVIDIA/NemoClaw/issues/5359)) and parallel inference routing fallback ([#5343](https://github.com/NVIDIA/NemoClaw/issues/5343)). A declarative multi-agent manifest is roadmap ([#2853](https://github.com/NVIDIA/NemoClaw/issues/2853)). | | Monitoring and health | Supported | Use `$$nemoclaw status`, `$$nemoclaw logs --follow`, and `openshell term`. Refer to [Monitor Sandbox Activity](../monitoring/monitor-sandbox-activity). | -| External telemetry and observability export | Roadmap-only | NemoClaw has no built-in metrics or trace export to external observability backends. An observability adapter plugin is tracked in [#3915](https://github.com/NVIDIA/NemoClaw/issues/3915). OpenShell emits structured platform logs (platform-owned). | +| External telemetry and observability export | Supported with caveats | OpenClaw can emit conversation traces through its diagnostics plugin, and LangChain Deep Agents Code can explicitly opt into bounded-content OTLP/HTTP traces with `--observability`. Both paths require an operator-run host collector. NemoClaw does not manage the collector, remote exporter credentials, fleet-wide routing, metrics or logs export, or authenticated tenant identity. Treat exported prompts, responses, tool inputs, and tool results as sensitive application data. Refer to [Deep Agents Code OTLP Traces](commands#deep-agents-code-otlp-traces). Broader observability adapter work remains tracked in [#3915](https://github.com/NVIDIA/NemoClaw/issues/3915). | | Audit and session records | Supported with caveats | OpenClaw stores per-session JSONL event logs you can export for audit or compliance review; Hermes stores its own runtime state. Export is manual per sandbox. Refer to [Inspect Agent Session State](../monitoring/monitor-sandbox-activity#inspect-agent-session-state). | | Resource quotas | Supported with caveats | The entrypoint applies best-effort process and file-descriptor limits (`ulimit -u 512`, `ulimit -n 65536`). Set hard limits through the container runtime for fail-closed enforcement. Refer to [Process Controls](../security/best-practices#process-controls). | | Cost and spend controls | Platform or partner-owned | Deny-by-default egress and routed inference reduce exfiltration and stray endpoints, but NemoClaw does not enforce per-token spend budgets. Set spend limits with your inference provider and monitor unattended agents. | @@ -101,7 +101,7 @@ The following table classifies each admin and control-plane expectation by curre | Role-based access control for operators | Out of scope | NemoClaw assumes a single trusted operator per host. There is no operator RBAC layer. | | Enterprise identity integration (SSO, OIDC, SAML) | Roadmap-only | Gateway access uses device pairing for the OpenClaw dashboard or bearer-token auth for the Hermes API, not enterprise identity providers. | | Multi-tenant isolation | Out of scope | Isolation is per-sandbox at the container level. NemoClaw does not provide tenant separation for multiple untrusted users on one host. | -| Centralized audit export and SIEM integration | Manual or admin-only | Export per-session JSONL logs by hand for audit review. External telemetry forwarding is roadmap ([#3915](https://github.com/NVIDIA/NemoClaw/issues/3915)). | +| Centralized audit export and SIEM integration | Manual or admin-only | Export per-session JSONL logs by hand for audit review. Deep Agents Code traces can reach an operator-managed backend, but that local collector path is not centralized fleet audit, authenticated tenant identity, or a managed SIEM integration. Broader forwarding work remains tracked in [#3915](https://github.com/NVIDIA/NemoClaw/issues/3915). | | Usage quotas, cost budgets, and billing | Platform or partner-owned | Set token and rate limits with your inference provider. NemoClaw does not meter or cap spend. | | Credential and secrets management | Supported with caveats | Provider credentials live on the host with restricted permissions and redaction. Integration with an external secrets manager is manual. Refer to [Credential Storage](../security/credential-storage). | | Policy as code distributed across a fleet | Manual or admin-only | Baseline policy and presets are versioned in the blueprint and applied per sandbox. There is no fleet-wide policy distribution service. | diff --git a/docs/reference/network-policies.mdx b/docs/reference/network-policies.mdx index 7a1627637f..90dca5c021 100644 --- a/docs/reference/network-policies.mdx +++ b/docs/reference/network-policies.mdx @@ -138,7 +138,7 @@ The baseline policy is always applied regardless of the selected tier. | Tier | Presets included | Description | |------|------------------|-------------| -| Restricted | No tier defaults | Starts from the baseline policy. Web search or messaging integrations selected earlier can still suggest their required presets; deselect them during policy review for baseline-only access. Restricted suppresses other agent-required additions; reapply them later with `policy-add` if needed. | +| Restricted | No tier defaults | Starts from the baseline policy. Web search or messaging integrations selected earlier can still suggest their required presets; deselect them during policy review for baseline-only access. Restricted suppresses other agent-required additions; reapply them later with `policy-add` only after reviewing the additional egress. | | Balanced (default) | `npm`, `pypi`, `huggingface`, `brew`, selected `brave` or `tavily` web search preset | Full dev tooling and web search when you select a provider the active agent supports. No messaging platform access. Apply the `weather` preset explicitly if your agent needs read-only weather lookups. | | Open | `npm`, `pypi`, `huggingface`, `brew`, selected `brave` or `tavily` web search preset, `weather`, `public-reference`, `slack`, `discord`, `telegram`, `wechat` (experimental), `whatsapp` (experimental), `jira`, `outlook` | Broad access across third-party services including messaging, productivity, weather, and public-reference APIs. | @@ -164,6 +164,11 @@ When Hermes uses Tavily, NemoClaw removes `nous-web` from the effective managed- OpenClaw onboarding also adds the `openclaw-pricing` preset on top of tier defaults so session-cost records can populate from LiteLLM and OpenRouter without manual configuration. When the OpenClaw OTEL diagnostics feature is enabled with a local endpoint, NemoClaw adds the `openclaw-diagnostics-otel-local` preset on the same basis. + +When LangChain Deep Agents Code is onboarded with `--observability`, NemoClaw adds the `observability-otlp-local` preset on Balanced and Open tiers. +The Restricted tier suppresses this agent-required preset during onboarding and rebuild reconciliation. +An operator can add it manually after reviewing the additional egress, but the next Restricted reconciliation removes it. + The applied set therefore reflects the chosen tier *plus* any agent-required presets, so `policy-list` may show one or more presets that do not appear in the tier table above. The `policy-list` provenance tags are inferred from the current tier YAML and the active agent at display time and are not persisted per preset. A preset whose name matches an entry in the sandbox's current tier definition is labelled `[from tier]` even when an operator added it manually with `policy-add` after onboarding; agent-specific preset names are only labelled `[from agent]` when the active agent matches. @@ -188,6 +193,32 @@ Interactive onboarding ignores an invalid environment value and shows the normal The baseline policy allows only the `local` inference route. External inference providers are reached through the OpenShell gateway, not by direct sandbox egress. + + +### Local OTLP Trace Export + +The `observability-otlp-local` preset supports the opt-in LangChain Deep Agents Code trace path. +It is not a general remote observability policy. + +| Preset | Destination | Binary | Rules | +|---|---|---|---| +| `observability-otlp-local` | `host.openshell.internal:4318` | `/opt/venv/bin/python3*` | Exact `POST /v1/traces` only | + +The sandbox sends OTLP/HTTP protobuf traces to a collector that the operator runs on the host. +The managed exporter uses standard OTLP transport headers but does not accept operator-supplied custom or authentication headers. +Remote backend endpoints and credentials stay in that collector. +The policy does not allow direct LangSmith, Jaeger, Phoenix, or other backend egress from Deep Agents Code. +Changing the collector's downstream exporter requires no sandbox policy change. + +OpenShell observes Deep Agents Code export as the managed Python interpreter, so this permission is process-wide for `/opt/venv/bin/python3*` rather than limited to the `dcode` launcher. +Sandbox Python can forge spans and resource attributes. +The explicit `--observability` opt-in can export bounded prompts, responses, tool arguments, tool results, and operational metadata. +Managed size and recognized-key redaction do not detect secrets embedded in ordinary content values. +The collector must enforce the operator's filtering and redaction requirements before forwarding traces, and it must not treat span fields such as `service.name` as authenticated tenant identity. +For a safe host binding, policy recovery commands, a runnable collector, and end-to-end verification, refer to [Export Traces Through a Local Collector](../get-started/quickstart#export-traces-through-a-local-collector). + + + ## Operator Approval Flow When the agent attempts to reach an endpoint not listed in the policy, OpenShell intercepts the request and presents it in the TUI for operator review. diff --git a/docs/security/credential-storage.mdx b/docs/security/credential-storage.mdx index b70c1b9309..11c3e75397 100644 --- a/docs/security/credential-storage.mdx +++ b/docs/security/credential-storage.mdx @@ -87,8 +87,10 @@ Use a dedicated low-scope search key and keep the matching `brave` or `tavily` p Rerun onboarding when you change providers because the provider selection and credential attachment are part of the sandbox image. -NemoClaw does not support LangSmith or OpenTelemetry tracing for the managed Deep Agents harness. -No supported path injects `LANGSMITH_API_KEY`, OTLP exporter headers, or tracing endpoint credentials into `dcode`; adding egress endpoints alone does not enable tracing. +NemoClaw supports opt-in, backend-neutral OTLP tracing for the managed Deep Agents harness through an operator-run host collector. +The sandbox sends traces only to the fixed local receiver and does not receive `LANGSMITH_API_KEY`, remote OTLP exporter headers, or backend credentials. +Native LangSmith tracing and ambient OpenTelemetry exporter configuration remain disabled inside `dcode`. +Keep backend credentials in the host collector, and refer to [Export Traces Through a Local Collector](../get-started/quickstart#export-traces-through-a-local-collector) for the supported setup. NemoClaw still keeps non-secret operational state under `~/.nemoclaw/` (such as the sandbox registry). diff --git a/nemoclaw-blueprint/policies/presets/observability-otlp-local.yaml b/nemoclaw-blueprint/policies/presets/observability-otlp-local.yaml new file mode 100644 index 0000000000..14ad674083 --- /dev/null +++ b/nemoclaw-blueprint/policies/presets/observability-otlp-local.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Local OpenTelemetry trace export for managed agent observability. +# The sandbox never receives a remote endpoint, exporter header, or backend credential. +preset: + name: observability-otlp-local + description: "OTLP/HTTP trace export to a local host collector" + +network_policies: + observability-otlp-local: + name: observability-otlp-local + endpoints: + - host: host.openshell.internal + port: 4318 + protocol: rest + enforcement: enforce + # Docker/Podman bridge gateway addresses vary by host and network. + # OpenShell's SSRF guard rejects their private resolutions unless the + # policy allowlists them. The exact hostname, port, method, path, and + # managed-Python binary remain independently enforced below. + allowed_ips: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + rules: + - allow: { method: POST, path: "/v1/traces" } + binaries: + - { path: /opt/venv/bin/python3* } diff --git a/scripts/checks/openshell-policy-mutation-read.ts b/scripts/checks/openshell-policy-mutation-read.ts index 31f2d37e35..8a7df8dfaf 100644 --- a/scripts/checks/openshell-policy-mutation-read.ts +++ b/scripts/checks/openshell-policy-mutation-read.ts @@ -36,7 +36,7 @@ interface AuditedMutationRead { export const MUTATION_READS: readonly AuditedMutationRead[] = [ { relativePath: "src/lib/policy/index.ts", - expectedReadCalls: 5, + expectedReadCalls: 6, baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), { ignoreError: true })", fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName), { ignoreError: true })", diff --git a/src/commands/sandbox/exec.test.ts b/src/commands/sandbox/exec.test.ts index 3b5c449364..ccc8e6aeab 100644 --- a/src/commands/sandbox/exec.test.ts +++ b/src/commands/sandbox/exec.test.ts @@ -29,6 +29,56 @@ describe("SandboxExecCommand oclif parse path", () => { ); }); + it("preserves repeated flag/value pairs after -- in their original order", async () => { + await SandboxExecCommand.run( + [ + "alpha", + "--", + "env", + "-u", + "ALL_PROXY", + "-u", + "HTTPS_PROXY", + "-u", + "HTTP_PROXY", + "-u", + "all_proxy", + "-u", + "https_proxy", + "-u", + "http_proxy", + "/opt/venv/bin/python3", + "-I", + "-c", + "pass", + ], + rootDir, + ); + expect(execSandboxMock).toHaveBeenCalledWith( + "alpha", + [ + "env", + "-u", + "ALL_PROXY", + "-u", + "HTTPS_PROXY", + "-u", + "HTTP_PROXY", + "-u", + "all_proxy", + "-u", + "https_proxy", + "-u", + "http_proxy", + "/opt/venv/bin/python3", + "-I", + "-c", + "pass", + ], + { workdir: undefined, tty: null, timeoutSeconds: undefined }, + ); + }); + it("parses --workdir before -- and keeps the inner command intact", async () => { await SandboxExecCommand.run( ["alpha", "--workdir", "/sandbox/workspace", "--", "ls", "-la"], diff --git a/src/commands/sandbox/exec.ts b/src/commands/sandbox/exec.ts index e2ce003880..f79eb5165e 100644 --- a/src/commands/sandbox/exec.ts +++ b/src/commands/sandbox/exec.ts @@ -40,8 +40,17 @@ export default class SandboxExecCommand extends NemoClawCommand { }; public async run(): Promise { + const originalArgv = [...this.argv]; const { args, flags, argv } = await this.parse(SandboxExecCommand); - const cmd = argv.slice(1) as string[]; + const separatorIndex = originalArgv.indexOf("--"); + // oclif's non-strict parser preserves ordinary inner flags, but sorts + // repeated unknown flags by their first input position. That turns a + // command such as `env -u A -u B` into `env -u -u A B`. Once the caller + // used the documented `--` boundary, take the command from oclif's + // original argv instead of its reconstructed parser output. + const cmd = ( + separatorIndex === -1 ? argv.slice(1) : originalArgv.slice(separatorIndex + 1) + ) as string[]; await execSandbox(args.sandboxName, cmd, { workdir: flags.workdir, tty: typeof flags.tty === "boolean" ? flags.tty : null, diff --git a/src/commands/sandbox/oclif-command-adapters.test.ts b/src/commands/sandbox/oclif-command-adapters.test.ts index 1d90c43c86..fe5d77d1ef 100644 --- a/src/commands/sandbox/oclif-command-adapters.test.ts +++ b/src/commands/sandbox/oclif-command-adapters.test.ts @@ -122,6 +122,7 @@ describe("sandbox oclif command adapters", () => { ["alpha", "--force", "--verbose", "--tool-disclosure", "direct"], rootDir, ); + await RebuildCliCommand.run(["dcode", "--yes", "--no-observability"], rootDir); await GatewayRestartCliCommand.run(["alpha", "--quiet"], rootDir); expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { probeOnly: true }); @@ -132,6 +133,13 @@ describe("sandbox oclif command adapters", () => { verbose: true, yes: false, }); + expect(mocks.rebuildSandbox).toHaveBeenCalledWith("dcode", { + force: false, + observabilityEnabled: false, + toolDisclosure: undefined, + verbose: false, + yes: true, + }); expect(mocks.restartSandboxGateway).toHaveBeenCalledWith("alpha", { quiet: true }); } finally { if (originalCleanupGatewayEnv === undefined) { @@ -230,6 +238,7 @@ describe("sandbox oclif command adapters", () => { expect(RebuildCliCommand.id).toBe("sandbox:rebuild"); expect(usage(RebuildCliCommand)).toContain("[--yes|-y|--force]"); expect(usage(RebuildCliCommand)).toContain("[--tool-disclosure ]"); + expect(usage(RebuildCliCommand)).toContain("[--observability|--no-observability]"); expect(SandboxPolicyListCommand.id).toBe("sandbox:policy:list"); expect(SandboxChannelsListCommand.id).toBe("sandbox:channels:list"); expect(SandboxConfigGetCommand.id).toBe("sandbox:config:get"); diff --git a/src/commands/sandbox/rebuild.ts b/src/commands/sandbox/rebuild.ts index 55db741e0e..01224aaa53 100644 --- a/src/commands/sandbox/rebuild.ts +++ b/src/commands/sandbox/rebuild.ts @@ -14,12 +14,13 @@ export default class RebuildCliCommand extends NemoClawCommand { static summary = "Upgrade sandbox to current agent version"; static description = "Back up, recreate, and restore a sandbox using the current agent image."; static usage = [ - " [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ]", + " [--yes|-y|--force] [--verbose|-v] [--tool-disclosure ] [--observability|--no-observability]", ]; static examples = [ "<%= config.bin %> sandbox rebuild alpha", "<%= config.bin %> sandbox rebuild alpha --yes --verbose", "<%= config.bin %> sandbox rebuild alpha --yes --tool-disclosure direct", + "<%= config.bin %> sandbox rebuild my-dcode --yes --observability", ]; static args = { sandboxName: Args.string({ name: "sandbox", description: "Sandbox name", required: true }), @@ -32,12 +33,17 @@ export default class RebuildCliCommand extends NemoClawCommand { description: "Change the sandbox tool-disclosure mode during the transactional rebuild", options: [...TOOL_DISCLOSURE_VALUES], }), + observability: Flags.boolean({ + allowNo: true, + description: "Change managed Deep Agents Code trace export during the transactional rebuild", + }), }; public async run(): Promise { const { args, flags } = await this.parse(RebuildCliCommand); await rebuildSandbox(args.sandboxName, { force: flags.force === true, + ...(flags.observability === undefined ? {} : { observabilityEnabled: flags.observability }), toolDisclosure: (flags["tool-disclosure"] as ToolDisclosure | undefined) ?? undefined, verbose: flags.verbose === true, yes: flags.yes === true, diff --git a/src/lib/actions/sandbox/exec-policy-hint-detection.test.ts b/src/lib/actions/sandbox/exec-policy-hint-detection.test.ts index f8e2d21fee..c76742caaa 100644 --- a/src/lib/actions/sandbox/exec-policy-hint-detection.test.ts +++ b/src/lib/actions/sandbox/exec-policy-hint-detection.test.ts @@ -40,6 +40,81 @@ describe("isPolicyDenialLine (#5978)", () => { true, ], ["proxy JSON policy_denied body", PROXY_JSON_LINE, true], + [ + "forward proxy host denial body", + '{"error":"policy_denied","detail":"POST example.com:4318/v1/traces not permitted by policy"}', + true, + ], + [ + "forward proxy unmatched endpoint path body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/not-traces did not match an L7 endpoint path"}', + true, + ], + [ + "forward proxy L7 method denial body", + '{"error":"policy_denied","detail":"GET host.openshell.internal:4318/v1/traces denied by L7 policy: GET /v1/traces not permitted by policy"}', + true, + ], + [ + "forward proxy L7 path denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/not-traces denied by L7 policy: POST /not-traces not permitted by policy"}', + true, + ], + [ + "forward proxy explicit deny-rule body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/v1/traces denied by L7 policy: POST /v1/traces blocked by deny rule"}', + true, + ], + [ + "forward proxy GraphQL policy denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/graphql denied by L7 policy: GraphQL operation blocked by endpoint policy"}', + true, + ], + [ + "forward proxy unregistered GraphQL persisted-query denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/graphql denied by L7 policy: GraphQL persisted query is not registered"}', + true, + ], + [ + "forward proxy GraphQL allow-policy denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/graphql denied by L7 policy: GraphQL operation not permitted by policy"}', + true, + ], + [ + "forward proxy GraphQL parse denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/graphql denied by L7 policy: GraphQL request rejected: missing operation document"}', + true, + ], + [ + "forward proxy JSON-RPC parse denial body", + `{"error":"policy_denied","detail":"POST host.openshell.internal:4318/rpc denied by L7 policy: JSON-RPC request rejected: missing or non-string 'jsonrpc' field"}`, + true, + ], + [ + "forward proxy JSON-RPC response-frame denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/rpc denied by L7 policy: JSON-RPC response frames are not permitted from client to server"}', + true, + ], + [ + "forward proxy policy-engine fallback denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/rpc denied by L7 policy: request denied by policy"}', + true, + ], + [ + "forward proxy policy-evaluation failure body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4318/rpc denied by L7 policy: L7 evaluation error: policy engine unavailable"}', + true, + ], + [ + "forward proxy extension method denial body", + '{"error":"policy_denied","detail":"PROPFIND host.openshell.internal:4318/resource not permitted by policy"}', + true, + ], + [ + "forward proxy port denial body", + '{"error":"policy_denied","detail":"POST host.openshell.internal:4319/v1/traces not permitted by policy"}', + true, + ], [ "timestamp-prefixed proxy JSON policy_denied body", `[1783046573.602] [gateway] ${PROXY_JSON_LINE}`, @@ -68,10 +143,68 @@ describe("isPolicyDenialLine (#5978)", () => { false, ], [ - "exact JSON error code without the structured CONNECT denial detail", + "exact JSON error code without a structured proxy denial detail", '{"detail":"policy_denied is configured here","error":"policy_denied"}', false, ], + [ + "forward proxy detail with an invalid endpoint", + '{"error":"policy_denied","detail":"POST bad/host:4318/v1/traces not permitted by policy"}', + false, + ], + [ + "forward proxy detail with a lowercase method", + '{"error":"policy_denied","detail":"post example.com:4318/v1/traces not permitted by policy"}', + false, + ], + [ + "forward proxy detail with an unsupported suffix", + '{"error":"policy_denied","detail":"POST example.com:4318/v1/traces access denied"}', + false, + ], + [ + "forward proxy L7 detail whose reason does not match its method", + '{"error":"policy_denied","detail":"GET example.com:4318/v1/traces denied by L7 policy: POST /v1/traces not permitted by policy"}', + false, + ], + [ + "forward proxy L7 detail whose reason does not match its path", + '{"error":"policy_denied","detail":"GET example.com:4318/v1/traces denied by L7 policy: GET /other not permitted by policy"}', + false, + ], + [ + "forward proxy L7 detail with an unknown policy reason", + '{"error":"policy_denied","detail":"POST example.com:4318/v1/traces denied by L7 policy: arbitrary denial prose"}', + false, + ], + [ + "forward proxy L7 evaluation detail with an empty error", + '{"error":"policy_denied","detail":"POST example.com:4318/v1/traces denied by L7 policy: L7 evaluation error: "}', + false, + ], + [ + "forward proxy L7 detail with control text in a dynamic reason", + JSON.stringify({ + detail: + "POST example.com:4318/graphql denied by L7 policy: GraphQL request rejected: bad\noperation", + error: "policy_denied", + }), + false, + ], + [ + "proxy denial body with extra JSON fields", + '{"error":"policy_denied","detail":"POST example.com:4318/v1/traces not permitted by policy","extra":true}', + false, + ], + ["oversized structured proxy line", `${"x".repeat(4097)}${PROXY_JSON_LINE}`, false], + [ + "oversized structured proxy detail", + JSON.stringify({ + detail: `POST example.com:4318/${"a".repeat(1100)} not permitted by policy`, + error: "policy_denied", + }), + false, + ], ["unrelated log line", "[123.0] [sandbox] [INFO ] flushed activity summary", false], ["empty line", "", false], ])("classifies %s", (_label, line, expected) => { diff --git a/src/lib/actions/sandbox/exec-policy-hint-detection.ts b/src/lib/actions/sandbox/exec-policy-hint-detection.ts index 3fd67ffac3..3a84cbba20 100644 --- a/src/lib/actions/sandbox/exec-policy-hint-detection.ts +++ b/src/lib/actions/sandbox/exec-policy-hint-detection.ts @@ -27,27 +27,84 @@ function isSafeEndpoint(candidate: string): boolean { const OCSF_NETWORK_DENIAL_RE = /\bNET:OPEN\b\]?(?:\s+\[[^\]\r\n]*\])*\s+DENIED(?=\s|$)/; const PROXY_DENIAL_DETAIL_RE = /^CONNECT\s+(\[[^\]\s]+\]:\d{1,5}|[^\s:]+:\d{1,5})\s+not\s+(?:allowed|permitted)\s+by\s+(?:any\s+)?policy$/i; +const PROXY_HTTP_DENIAL_DETAIL_RE = + /^([A-Z][A-Z0-9!#$%&'*+.^_`|~-]{0,31}) (\[[^\]\s]+\]:\d{1,5}|[^\s:]+:\d{1,5})(\/\S{0,2047}) (.+)$/; +const SAFE_PROXY_PATH_RE = /^\/[A-Za-z0-9._~!$&'()*+,;=:@%/?-]{0,2047}$/; +const SAFE_DYNAMIC_PROXY_REASON_RE = /^[\x20-\x7e]{1,512}$/; +const MAX_STRUCTURED_PROXY_LINE_LENGTH = 4096; +const MAX_STRUCTURED_PROXY_DETAIL_LENGTH = 1024; + +function isStructuredL7PolicyDenialReason(method: string, path: string, reason: string): boolean { + if ( + reason === `${method} ${path} not permitted by policy` || + reason === `${method} ${path} blocked by deny rule` + ) { + return true; + } + if ( + reason === "GraphQL persisted query is not registered" || + reason === "GraphQL operation blocked by endpoint policy" || + reason === "GraphQL operation not permitted by policy" || + reason === "JSON-RPC response frames are not permitted from client to server" || + reason === "request denied by policy" + ) { + return true; + } + const dynamicPrefixes = [ + "GraphQL request rejected: ", + "JSON-RPC request rejected: ", + "L7 evaluation error: ", + ]; + const prefix = dynamicPrefixes.find((candidate) => reason.startsWith(candidate)); + return Boolean(prefix && SAFE_DYNAMIC_PROXY_REASON_RE.test(reason.slice(prefix.length))); +} + +function isStructuredForwardProxyPolicyDenial(detail: string): boolean { + const forward = detail.match(PROXY_HTTP_DENIAL_DETAIL_RE); + if (!forward) return false; + const [, method, endpoint, path, suffix] = forward; + if (!isSafeEndpoint(endpoint) || !SAFE_PROXY_PATH_RE.test(path)) { + return false; + } + if (suffix === "not permitted by policy" || suffix === "did not match an L7 endpoint path") { + return true; + } + const reasonPrefix = "denied by L7 policy: "; + if (!suffix.startsWith(reasonPrefix)) return false; + return isStructuredL7PolicyDenialReason(method, path, suffix.slice(reasonPrefix.length)); +} + +function isStructuredProxyPolicyDenialDetail(detail: string): boolean { + if (detail.length > MAX_STRUCTURED_PROXY_DETAIL_LENGTH) return false; + const connect = detail.match(PROXY_DENIAL_DETAIL_RE); + if (connect) return isSafeEndpoint(connect[1]); + return isStructuredForwardProxyPolicyDenial(detail); +} // Source-of-truth for structured proxy JSON: -// - Invalid state: OpenShell reports the policy refusal only in its CONNECT 403 -// JSON while the child tool receives opaque protocol text. +// - Invalid state: OpenShell reports some policy refusals only in structured +// CONNECT or forward-HTTP 403 JSON while the child tool receives opaque text. // - Source boundary/fix constraint: the payload is emitted by the external // OpenShell proxy, so NemoClaw can only translate it after exec returns. // - Regression coverage: prefixed, unprefixed, malformed, and near-miss JSON // payloads live in exec-policy-hint-detection.test.ts. // - Removal condition: delete this fallback when OpenShell provides a typed // exec-denial result. Until then, require both the exact error code and the -// complete safely bounded CONNECT detail so unrelated JSON cannot match. +// complete safely bounded CONNECT or forward-HTTP detail so unrelated JSON +// cannot match. The forward forms mirror OpenShell v0.0.72's endpoint, path, +// and L7 policy denial messages. function isStructuredJsonPolicyDenial(line: string): boolean { + if (line.length > MAX_STRUCTURED_PROXY_LINE_LENGTH) return false; const jsonStart = line.indexOf("{"); if (jsonStart === -1) return false; try { const parsed: unknown = JSON.parse(line.slice(jsonStart)); if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return false; const payload = parsed as Record; + const keys = Object.keys(payload); + if (keys.length !== 2 || !keys.includes("detail") || !keys.includes("error")) return false; if (payload.error !== "policy_denied" || typeof payload.detail !== "string") return false; - const detail = payload.detail.match(PROXY_DENIAL_DETAIL_RE); - return Boolean(detail && isSafeEndpoint(detail[1])); + return isStructuredProxyPolicyDenialDetail(payload.detail); } catch { return false; } diff --git a/src/lib/actions/sandbox/rebuild-backup-phase.test.ts b/src/lib/actions/sandbox/rebuild-backup-phase.test.ts index 36c5f2efcf..ee02cff951 100644 --- a/src/lib/actions/sandbox/rebuild-backup-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-backup-phase.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import { + normalizeRebuildObservabilityPolicyPresets, + normalizeRebuildTargetPolicyPresets, normalizeRebuildWebSearchPolicyPresets, runRebuildBackupPhase, } from "./rebuild-backup-phase"; @@ -84,4 +86,76 @@ describe("rebuild web-search policy normalization", () => { expect(result?.policyPresets).toEqual([]); expect(result?.sessionPolicyPresets).toEqual([]); }); + + it("removes stale built-in observability egress from disabled and restricted rebuild targets", () => { + expect( + normalizeRebuildObservabilityPolicyPresets(["npm", "observability-otlp-local"], { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + }), + ).toEqual(["npm"]); + expect( + normalizeRebuildObservabilityPolicyPresets(["npm", "observability-otlp-local"], { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: "restricted", + }), + ).toEqual(["npm"]); + expect( + normalizeRebuildObservabilityPolicyPresets(["npm"], { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: "balanced", + }), + ).toEqual(["npm", "observability-otlp-local"]); + }); + + it("leaves a same-name custom observability policy for exact custom replay", () => { + expect( + normalizeRebuildObservabilityPolicyPresets(["npm", "observability-otlp-local"], { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "restricted", + customPolicies: [{ name: "observability-otlp-local", content: "network_policies: {}" }], + }), + ).toEqual(["npm"]); + }); + + it("does not add built-in observability when a differently named custom policy owns its key", () => { + expect( + normalizeRebuildObservabilityPolicyPresets(["npm", "observability-otlp-local"], { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: "balanced", + customPolicies: [ + { + name: "corp-otel", + content: + "network_policies:\n observability-otlp-local:\n endpoints:\n - host: collector.corp.example\n", + }, + ], + }), + ).toEqual(["npm"]); + }); + + it("keeps fresh agent-required additions while suppressing stale restricted observability", () => { + expect( + normalizeRebuildTargetPolicyPresets( + ["npm", "future-agent-required", "observability-otlp-local"], + { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: " Restricted ", + }, + null, + ), + ).toEqual(["npm", "future-agent-required"]); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-backup-phase.ts b/src/lib/actions/sandbox/rebuild-backup-phase.ts index 6117e55d06..510b6bcadb 100644 --- a/src/lib/actions/sandbox/rebuild-backup-phase.ts +++ b/src/lib/actions/sandbox/rebuild-backup-phase.ts @@ -4,8 +4,16 @@ import { type WebSearchConfig, webSearchProviderForConfig } from "../../inference/web-search"; import type { SandboxMessagingPlan } from "../../messaging"; import { mergeRebuildMessagingPolicyPresets } from "../../onboard/messaging-policy-presets"; +import { + isDcodeAgent, + isInactiveObservabilityPolicyPreset, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + requiredObservabilityPolicyPresets, +} from "../../onboard/observability-policy-presets"; import { resolveRecreatePolicyPresets } from "../../onboard/policy-preset-persistence"; import { isStaleBuiltinWebSearchPolicyPreset } from "../../onboard/policy-selection"; +import { filterSuppressedAgentRequiredPresets } from "../../onboard/policy-tier-suppression"; +import { parsePresetPolicyKeys } from "../../policy"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import { backupSandboxStateForRebuild, type RebuildSandboxEntry } from "./rebuild-flow-helpers"; @@ -64,6 +72,62 @@ export function normalizeRebuildWebSearchPolicyPresets( return [...new Set(normalized)]; } +/** Align built-in observability egress with the durable opt-in and policy tier. */ +export function normalizeRebuildObservabilityPolicyPresets( + presets: readonly string[], + sandboxEntry: RebuildSandboxEntry, +): string[] { + const customPresetNames = new Set( + (sandboxEntry.customPolicies ?? []).map((policy) => policy.name.trim().toLowerCase()), + ); + const customOwnsObservabilityPolicy = (sandboxEntry.customPolicies ?? []).some((policy) => + parsePresetPolicyKeys(policy.content).includes(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET), + ); + const customOwnsObservability = + customPresetNames.has(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET) || customOwnsObservabilityPolicy; + const activePresets = presets.filter((name) => { + const normalizedName = name.trim().toLowerCase(); + if (normalizedName !== OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET) return true; + // Custom content is replayed separately from the captured manifest. Its + // registry name may differ from the network-policy key it owns, so neither + // form may be substituted with the built-in preset. + if (customOwnsObservability) return false; + return ( + isDcodeAgent(sandboxEntry.agent) && + !isInactiveObservabilityPolicyPreset(name, { + agent: sandboxEntry.agent, + observabilityEnabled: sandboxEntry.observabilityEnabled, + customPresetNames, + }) + ); + }); + if (!customOwnsObservability) { + for (const requiredPreset of requiredObservabilityPolicyPresets( + sandboxEntry.agent, + sandboxEntry.observabilityEnabled, + )) { + if (!activePresets.includes(requiredPreset)) activePresets.push(requiredPreset); + } + } + return filterSuppressedAgentRequiredPresets( + [...new Set(activePresets)], + sandboxEntry.policyTier, + sandboxEntry.agent, + ); +} + +/** Normalize the complete replacement target, including fresh inner-onboard additions. */ +export function normalizeRebuildTargetPolicyPresets( + presets: readonly string[], + sandboxEntry: RebuildSandboxEntry, + webSearchConfig: WebSearchConfig | null, +): string[] { + return normalizeRebuildObservabilityPolicyPresets( + normalizeRebuildWebSearchPolicyPresets([...new Set(presets)], sandboxEntry, webSearchConfig), + sandboxEntry, + ); +} + export function runRebuildBackupPhase( input: RebuildBackupPhaseInput, ): RebuildBackupPhaseResult | null { @@ -94,7 +158,7 @@ export function runRebuildBackupPhase( enabledChannelIds, disabledChannels, ); - const policyPresets = normalizeRebuildWebSearchPolicyPresets( + const policyPresets = normalizeRebuildTargetPolicyPresets( mergedPolicyPresets, input.sandboxEntry, input.webSearchConfig, diff --git a/src/lib/actions/sandbox/rebuild-dcode-recovery.test.ts b/src/lib/actions/sandbox/rebuild-dcode-recovery.test.ts index 7efc3c817b..36c72dedaa 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-recovery.test.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-recovery.test.ts @@ -87,4 +87,314 @@ describe("rebuildSandbox DCode flow: recovery", () => { expect.objectContaining({ policies: [], policyPresetsFinalized: true }), ); }); + + it("removes transient observability egress after rebuilding a restricted DCode sandbox", async () => { + let policyTierSeenDuringOnboard: string | undefined; + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + applyPreset: () => true, + backupPolicyPresets: ["npm", "observability-otlp-local"], + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets: ["observability-otlp-local"], + sandboxEntry: { + ...makeDcodeSandboxEntry(), + observabilityEnabled: true, + policies: ["npm", "observability-otlp-local"], + policyPresetsFinalized: true, + policyTier: " Restricted ", + }, + onboard: () => { + policyTierSeenDuringOnboard = process.env.NEMOCLAW_POLICY_TIER; + }, + }); + configureDcodeSession(harness); + harness.session.observabilityEnabled = true; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(policyTierSeenDuringOnboard).toBe("restricted"); + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ observabilityRequestedExplicitly: false }), + ); + expect(harness.session.observabilityRequestedExplicitly).toBe(false); + expect(harness.applyPresetSpy).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { + agentVersion: "0.2.0", + policies: ["npm"], + policyTier: "restricted", + policyPresetsFinalized: true, + }); + }); + + it("restores the required observability preset on a balanced DCode rebuild", async () => { + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + applyPreset: () => true, + backupPolicyPresets: ["npm"], + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets: [], + sandboxEntry: { + ...makeDcodeSandboxEntry(), + observabilityEnabled: true, + policies: ["npm"], + policyPresetsFinalized: true, + policyTier: "balanced", + }, + }); + configureDcodeSession(harness); + harness.session.observabilityEnabled = true; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + policies: ["npm", "observability-otlp-local"], + policyTier: "balanced", + policyPresetsFinalized: true, + }), + ); + }); + + it.each([ + { + label: "enables", + flag: "--observability", + before: false, + expected: true, + expectedObservabilityApplyCalls: [["alpha", "observability-otlp-local"]] as const, + backupPresets: [] as string[], + gatewayPresets: [] as string[], + }, + { + label: "disables", + flag: "--no-observability", + before: true, + expected: false, + expectedObservabilityApplyCalls: [] as const, + backupPresets: ["observability-otlp-local"], + gatewayPresets: ["observability-otlp-local"], + }, + ])("$label observability transactionally while preserving managed MCP state", async ({ + flag, + before, + expected, + expectedObservabilityApplyCalls, + backupPresets, + gatewayPresets, + }) => { + const mcpEntry = { server: "search", providerName: "mcp-search" }; + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + applyPreset: () => true, + backupPolicyPresets: backupPresets, + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets, + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [], + }, + sandboxEntry: { + ...makeDcodeSandboxEntry(), + observabilityEnabled: before, + policies: backupPresets, + policyPresetsFinalized: true, + policyTier: "balanced", + mcp: { + bridges: { search: mcpEntry }, + managedServerNames: ["search"], + }, + }, + }); + configureDcodeSession(harness); + harness.session.observabilityEnabled = before; + + await expect( + harness.rebuildSandbox("alpha", ["--yes", flag], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ + observabilityEnabled: expected, + observabilityRequestedExplicitly: true, + }), + ); + expect(harness.session.observabilityEnabled).toBe(expected); + expect(harness.session.observabilityRequestedExplicitly).toBe(true); + const observabilityApplyCalls = harness.applyPresetSpy.mock.calls.filter( + ([sandboxName, presetName]) => + sandboxName === "alpha" && presetName === "observability-otlp-local", + ); + expect(observabilityApplyCalls).toEqual(expectedObservabilityApplyCalls); + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + policies: expected ? ["observability-otlp-local"] : [], + policyTier: "balanced", + policyPresetsFinalized: true, + }), + ); + }); + + it("preserves a fresh agent-required preset introduced by inner onboard", async () => { + const freshRequiredPreset = "future-dcode-required"; + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + applyPreset: () => true, + backupPolicyPresets: ["npm"], + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets: [freshRequiredPreset], + onboard: (session) => { + session.policyPresets = ["npm", freshRequiredPreset]; + }, + sandboxEntry: { + ...makeDcodeSandboxEntry(), + policies: ["npm"], + policyPresetsFinalized: true, + policyTier: "balanced", + }, + }); + configureDcodeSession(harness); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + policies: ["npm", freshRequiredPreset], + policyPresetsFinalized: true, + }), + ); + }); + + it("never removes or persists DCode base-policy keys detected as broad presets", async () => { + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + backupPolicyPresets: [], + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets: ["github", "pypi"], + sandboxEntry: { + ...makeDcodeSandboxEntry(), + observabilityEnabled: false, + policies: [], + policyPresetsFinalized: true, + policyTier: "balanced", + }, + }); + configureDcodeSession(harness); + harness.session.observabilityEnabled = false; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.removePresetSpy).not.toHaveBeenCalled(); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ policies: [], policyPresetsFinalized: true }), + ); + }); + + it("does not narrow a differently named custom policy owning observability egress", async () => { + const customPolicy = { + name: "corp-otel", + content: + "network_policies:\n observability-otlp-local:\n endpoints:\n - host: collector.corp.example\n", + sourcePath: "/tmp/corp-otel.yaml", + }; + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + backupPolicyPresets: [], + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets: ["observability-otlp-local"], + sandboxEntry: { + ...makeDcodeSandboxEntry(), + customPolicies: [customPolicy], + observabilityEnabled: false, + policies: [], + policyPresetsFinalized: true, + policyTier: "balanced", + }, + }); + configureDcodeSession(harness); + harness.session.observabilityEnabled = false; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.applyPresetContentSpy).toHaveBeenCalledWith( + "alpha", + customPolicy.name, + customPolicy.content, + { custom: { sourcePath: customPolicy.sourcePath } }, + ); + expect(harness.removePresetSpy).not.toHaveBeenCalled(); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ policies: [], policyPresetsFinalized: true }), + ); + }); + + it("fails after recording recovery state when restricted egress removal cannot be verified", async () => { + const harness = createRebuildFlowHarness({ + agentName: "langchain-deepagents-code", + applyPreset: () => true, + backupPolicyPresets: ["npm", "observability-otlp-local"], + dcodeRouteResults: [{ ok: true }, { ok: true }, { ok: true }, { ok: true }], + gatewayPresets: ["observability-otlp-local"], + verificationUnavailableAfterPresetRemoval: true, + sandboxEntry: { + ...makeDcodeSandboxEntry(), + observabilityEnabled: true, + policies: ["npm", "observability-otlp-local"], + policyPresetsFinalized: true, + policyTier: "restricted", + }, + }); + configureDcodeSession(harness); + harness.session.observabilityEnabled = true; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Rebuild completed with unverified live policy reconciliation for 'alpha'."); + + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + policies: ["npm", "observability-otlp-local"], + policyTier: "restricted", + policyPresetsFinalized: undefined, + }), + ); + expect(harness.relockSpy).toHaveBeenCalled(); + }); + + it("rejects an observability override for a non-DCode sandbox before mutation", async () => { + const harness = createRebuildFlowHarness({ + agentName: "openclaw", + sandboxEntry: { + name: "alpha", + agent: "openclaw", + nemoclawVersion: "0.1.0", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes", "--observability"], { throwOnError: true }), + ).rejects.toThrow("Unsupported rebuild observability override"); + + expect(harness.openShieldsSpy).not.toHaveBeenCalled(); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts index 859b0135ec..88f5f6e380 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts @@ -172,6 +172,8 @@ describe("buildRebuildRecreateOnboardOpts", () => { sandboxGpuDevice: null, autoYes: true, toolDisclosure: "progressive", + observabilityEnabled: false, + observabilityRequestedExplicitly: false, }); }); @@ -184,6 +186,54 @@ describe("buildRebuildRecreateOnboardOpts", () => { expect(opts.toolDisclosure).toBe("direct"); }); + it("carries durable observability intent into inner onboard", () => { + const enabled = buildRebuildRecreateOnboardOpts({ + ...baseArgs, + rebuildAgent: "langchain-deepagents-code", + sb: { ...dashboard, observabilityEnabled: true }, + }); + const legacy = buildRebuildRecreateOnboardOpts({ ...baseArgs, sb: dashboard }); + + expect(enabled.observabilityEnabled).toBe(true); + expect(legacy.observabilityEnabled).toBe(false); + }); + + it("carries the authoritative restricted tier with observability into inner onboard", () => { + const opts = buildRebuildRecreateOnboardOpts({ + ...baseArgs, + rebuildAgent: "langchain-deepagents-code", + sb: { + observabilityEnabled: true, + policyTier: "restricted", + }, + }); + + expect(opts.policyTier).toBe("restricted"); + expect(opts.observabilityEnabled).toBe(true); + }); + + it("rejects an invalid recorded policy tier before destructive recreate work", () => { + expect(() => + buildRebuildRecreateOnboardOpts({ + ...baseArgs, + sb: { ...dashboard, policyTier: "unknown-tier" }, + }), + ).toThrow("Invalid recorded policy tier 'unknown-tier'."); + }); + + it.each([ + "openclaw", + "hermes", + ])("rejects malformed %s observability state before recreate onboarding", (rebuildAgent) => { + expect(() => + buildRebuildRecreateOnboardOpts({ + ...baseArgs, + rebuildAgent, + sb: { ...dashboard, observabilityEnabled: true }, + }), + ).toThrow("Recorded observability state is valid only for agent 'langchain-deepagents-code'."); + }); + it("forwards noGpu:true for legacy entries with gpuEnabled:false and no sandboxGpuMode", () => { const opts = buildRebuildRecreateOnboardOpts({ ...baseArgs, diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index 68c69c83e0..90e4a602f2 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -7,6 +7,7 @@ import { resolveGatewayPortFromName, resolveSandboxGatewayName, } from "../../onboard/gateway-binding"; +import { isDcodeAgent } from "../../onboard/observability-policy-presets"; import type { PreparedDcodeRebuildHandoff, PreparedImageRebuildHandoff, @@ -16,6 +17,7 @@ import type { RebuildRouteHandoff, } from "../../onboard/rebuild-route-handoff"; import { normalizeSandboxGpuMode } from "../../onboard/sandbox-gpu-mode"; +import { getTier } from "../../policy/tiers"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import { type ToolDisclosure, toolDisclosureOrDefault } from "../../tool-disclosure"; @@ -28,6 +30,8 @@ export type RebuildGpuOptOutEntry = { gatewayName?: string | null; gatewayPort?: number | null; toolDisclosure?: ToolDisclosure; + observabilityEnabled?: boolean; + policyTier?: string | null; }; // Modern source of truth is the persisted `sandboxGpuMode` string ("0" / "1" / @@ -100,6 +104,10 @@ export type RebuildRecreateOnboardOpts = { preparedImageRebuild?: PreparedImageRebuildHandoff; autoYes: boolean; toolDisclosure: ToolDisclosure; + observabilityEnabled: boolean; + /** Whether the rebuild command explicitly overrode the recorded observability state. */ + observabilityRequestedExplicitly: boolean; + policyTier: string | null; baseImageResolutionHint: SandboxBaseImageResolutionMetadata | null; noGpu?: true; }; @@ -113,7 +121,16 @@ export function buildRebuildRecreateOnboardOpts(args: { baseImageResolutionHint?: SandboxBaseImageResolutionMetadata | null; usageNoticeAccepted: true; }): RebuildRecreateOnboardOpts { + if (args.sb?.observabilityEnabled === true && !isDcodeAgent(args.rebuildAgent)) { + throw new Error( + "Recorded observability state is valid only for agent 'langchain-deepagents-code'.", + ); + } const gpuOverrides = getRebuildSandboxGpuOverrides(args.sb); + const rawPolicyTier = args.sb?.policyTier?.trim().toLowerCase() || null; + if (rawPolicyTier && !getTier(rawPolicyTier)) { + throw new Error(`Invalid recorded policy tier '${String(args.sb?.policyTier)}'.`); + } const targetGatewayName = resolveSandboxGatewayName(args.sb); const targetGatewayPort = resolveGatewayPortFromName(targetGatewayName); if (targetGatewayPort === null) { @@ -152,6 +169,9 @@ export function buildRebuildRecreateOnboardOpts(args: { ...(args.preparedDcodeRebuild ? { preparedDcodeRebuild: args.preparedDcodeRebuild } : {}), autoYes: args.autoYes, toolDisclosure: toolDisclosureOrDefault(args.sb?.toolDisclosure), + observabilityEnabled: args.sb?.observabilityEnabled === true, + observabilityRequestedExplicitly: false, + policyTier: rawPolicyTier, baseImageResolutionHint: args.baseImageResolutionHint ?? null, ...(rebuildShouldOptOutGpu(args.sb) ? { noGpu: true as const } : {}), }; diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts new file mode 100644 index 0000000000..cc1ed1f271 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.test.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { printMcpRebuildRetryCommand } from "./rebuild-mcp-phase"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("MCP rebuild retry guidance", () => { + it.each([ + [true, "--observability"], + [false, "--no-observability"], + ])("preserves an explicit observability=%s override", (enabled, expectedFlag) => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + printMcpRebuildRetryCommand("alpha", [{} as never], "progressive", { + enabled, + requestedExplicitly: true, + }); + + const output = error.mock.calls.flat().join("\n"); + expect(output).toContain( + `nemoclaw alpha rebuild --yes --tool-disclosure progressive ${expectedFlag}`, + ); + }); + + it("preserves an explicit opt-out on the resume retry form", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + printMcpRebuildRetryCommand("alpha", [], "direct", { + enabled: false, + requestedExplicitly: true, + }); + + expect(error.mock.calls.flat().join("\n")).toContain( + "nemoclaw onboard --resume --tool-disclosure direct --no-observability", + ); + }); + + it("does not turn inherited observability state into an explicit retry override", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + printMcpRebuildRetryCommand("alpha", [{} as never], "progressive", { + enabled: true, + requestedExplicitly: false, + }); + + const command = error.mock.calls.flat().find((line) => line.includes("rebuild --yes")); + expect(command).not.toContain("--observability"); + expect(command).not.toContain("--no-observability"); + }); + + it("keeps inherited observability state implicit on the resume retry form", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + printMcpRebuildRetryCommand("alpha", [], "progressive", { + enabled: false, + requestedExplicitly: false, + }); + + const command = error.mock.calls.flat().find((line) => line.includes("onboard --resume")); + expect(command).not.toContain("--observability"); + expect(command).not.toContain("--no-observability"); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.ts index 3e3018b945..74b5844d4f 100644 --- a/src/lib/actions/sandbox/rebuild-mcp-phase.ts +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.ts @@ -3,6 +3,7 @@ import { CLI_NAME } from "../../cli/branding"; import { G, R, YW } from "../../cli/terminal-style"; +import { explicitObservabilityFlag } from "../../onboard/observability-command-flag"; import * as registry from "../../state/registry"; import type { ToolDisclosure } from "../../tool-disclosure"; import { @@ -71,17 +72,24 @@ export function printMcpRebuildRetryCommand( sandboxName: string, entries: McpRebuildPreparation["entries"], toolDisclosure?: ToolDisclosure, + observability?: { enabled: boolean; requestedExplicitly: boolean }, ): void { + const observabilityFlag = observability + ? explicitObservabilityFlag(observability.enabled, observability.requestedExplicitly) + : null; + const observabilityArg = observabilityFlag ? ` ${observabilityFlag}` : ""; if (entries.length > 0) { const disclosureArg = toolDisclosure ? ` --tool-disclosure ${toolDisclosure}` : ""; - console.error(` 2. Run: ${CLI_NAME} ${sandboxName} rebuild --yes${disclosureArg}`); + console.error( + ` 2. Run: ${CLI_NAME} ${sandboxName} rebuild --yes${disclosureArg}${observabilityArg}`, + ); console.error( ` This will recreate sandbox '${sandboxName}' and restore its MCP bridges.`, ); return; } const disclosureArg = toolDisclosure ? ` --tool-disclosure ${toolDisclosure}` : ""; - console.error(` 2. Run: ${CLI_NAME} onboard --resume${disclosureArg}`); + console.error(` 2. Run: ${CLI_NAME} onboard --resume${disclosureArg}${observabilityArg}`); console.error(` This will recreate sandbox '${sandboxName}'.`); } diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 093e96183a..c982e54d72 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -8,9 +8,11 @@ import { MESSAGING_CHANNEL_CONFIG_ENV_KEYS } from "../../messaging-channel-confi import { hydrateCredentialEnv } from "../../onboard/credential-env"; import { DOCKER_GPU_PATCH_NETWORK_ENV } from "../../onboard/docker-gpu-patch"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; +import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; -import { runRebuildBackupPhase } from "./rebuild-backup-phase"; +import { normalizeRebuildTargetPolicyPresets, runRebuildBackupPhase } from "./rebuild-backup-phase"; import { buildRefreshMutableOpenClawConfigHashCommand } from "./rebuild-config-hash"; +import { DCODE_AGENT_NAME } from "./rebuild-dcode-target"; import { runRebuildDestroyPhase } from "./rebuild-destroy-phase"; import { REBUILD_HERMES_DASHBOARD_ENV_KEYS } from "./rebuild-durable-config"; import { stageMessagingManifestPlanForRebuild } from "./rebuild-messaging-phase"; @@ -148,7 +150,13 @@ async function rebuildSandboxUnlocked( const backup = runRebuildBackupPhase({ sandboxName, - sandboxEntry, + // The requested observability bit is replacement intent, not a + // preflight mutation of the old registry row. Use a copy only for + // target policy normalization; replacement registration commits it. + sandboxEntry: { + ...sandboxEntry, + observabilityEnabled: recreateOptions.observabilityEnabled, + }, staleRecovery, preparedRecoveryManifest: recoveryManifest, messagingPlan, @@ -269,25 +277,42 @@ async function rebuildSandboxUnlocked( } if (!recreated) return; + const completedInnerSession = onboardSession.loadSession(); + const freshInnerOnboardPolicyPresets = + completedInnerSession?.sandboxName === sandboxName && + Array.isArray(completedInnerSession.policyPresets) + ? completedInnerSession.policyPresets + : []; + const targetPolicyPresets = normalizeRebuildTargetPolicyPresets( + [...backup.policyPresets, ...freshInnerOnboardPolicyPresets], + { + ...sandboxEntry, + observabilityEnabled: recreateOptions.observabilityEnabled, + }, + durableConfig.webSearchConfig, + ); + const restored = runRebuildRestorePhase({ sandboxName, backupManifest: backup.backupManifest, - policyPresets: backup.policyPresets, + policyPresets: targetPolicyPresets, customPolicies: backup.backupManifest?.customPolicies?.map((entry) => ({ ...entry })) ?? preservedCustomPolicies, + reconcileManagedDcodeObservability: rebuildAgent === DCODE_AGENT_NAME, log, }); await runRebuildPostRestorePhase({ sandboxName, sandboxEntry, - preservedCustomPolicies, messagingPlan, backupManifest: backup.backupManifest, mcpEntries: mcpPreparation.entries, restoreSucceeded: restored.restoreSucceeded, - restoredPresets: restored.restoredPresets, failedPresets: restored.failedPresets, + finalBuiltinPresets: restored.finalBuiltinPresets, + failedPresetRemovals: restored.failedPresetRemovals, + policyPresetReconciliationVerified: restored.policyPresetReconciliationVerified, staleRecovery, recoveryRecreate, preparedBackupRecovery, diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts index c11d54f1d2..de3e727793 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts @@ -6,6 +6,7 @@ import * as agentRuntime from "../../agent/runtime"; import { CLI_NAME } from "../../cli/branding"; import { D, G, R, YW } from "../../cli/terminal-style"; import type { SandboxMessagingPlan } from "../../messaging"; +import { normalizePolicyTierName } from "../../onboard/policy-tier-suppression"; import type * as sandboxVersion from "../../sandbox/version"; import * as shields from "../../shields"; import * as registry from "../../state/registry"; @@ -26,13 +27,14 @@ import { reapplyMessagingManifestAfterOpenClawDoctor } from "./rebuild-messaging export interface RebuildPostRestorePhaseInput { sandboxName: string; sandboxEntry: RebuildSandboxEntry; - preservedCustomPolicies: NonNullable; messagingPlan: SandboxMessagingPlan | null; backupManifest: RebuildBackupManifest; mcpEntries: McpRebuildPreparation["entries"]; restoreSucceeded: boolean; - restoredPresets: string[]; failedPresets: string[]; + finalBuiltinPresets: string[]; + failedPresetRemovals: string[]; + policyPresetReconciliationVerified: boolean; staleRecovery: boolean; recoveryRecreate: boolean; preparedBackupRecovery: boolean; @@ -44,15 +46,19 @@ export interface RebuildPostRestorePhaseInput { } export function resolveRestoredPolicyRegistryState( - sandboxEntry: Pick, - restoredPresets: readonly string[], + sandboxEntry: Pick, + restoredBuiltinPresets: readonly string[], failedPresets: readonly string[], + policyPresetReconciliationVerified = true, ): { policies: string[]; policyPresetsFinalized: true | undefined } { - const customPolicyNames = new Set((sandboxEntry.customPolicies ?? []).map((entry) => entry.name)); return { - policies: restoredPresets.filter((name) => !customPolicyNames.has(name)), + policies: [...new Set(restoredBuiltinPresets)], policyPresetsFinalized: - sandboxEntry.policyPresetsFinalized === true && failedPresets.length === 0 ? true : undefined, + sandboxEntry.policyPresetsFinalized === true && + failedPresets.length === 0 && + policyPresetReconciliationVerified + ? true + : undefined, }; } @@ -67,13 +73,14 @@ export async function runRebuildPostRestorePhase( const { sandboxName, sandboxEntry: sb, - preservedCustomPolicies, messagingPlan, backupManifest, mcpEntries, restoreSucceeded, - restoredPresets, failedPresets, + finalBuiltinPresets, + failedPresetRemovals, + policyPresetReconciliationVerified, staleRecovery, recoveryRecreate, preparedBackupRecovery, @@ -89,7 +96,10 @@ export async function runRebuildPostRestorePhase( let mutablePermsRepairUnverified = false; let mutableConfigHashRefreshUnverified = false; let messagingHostForwardUnverified = false; - const policyPresetRestoreIncomplete = failedPresets.length > 0; + const policyPresetRestoreIncomplete = + failedPresets.length > 0 || + failedPresetRemovals.length > 0 || + !policyPresetReconciliationVerified; if (agentDef.name === "openclaw") { log("Running openclaw doctor --fix inside sandbox for post-upgrade structure repair"); @@ -146,16 +156,16 @@ export async function runRebuildPostRestorePhase( const { policies: restoredBuiltinPresets, policyPresetsFinalized } = resolveRestoredPolicyRegistryState( { - customPolicies: backupManifest?.customPolicies ?? preservedCustomPolicies, policyPresetsFinalized: sb.policyPresetsFinalized, }, - restoredPresets, + finalBuiltinPresets, failedPresets, + policyPresetReconciliationVerified, ); registry.updateSandbox(sandboxName, { agentVersion: agentDef.expectedVersion || null, policies: restoredBuiltinPresets, - policyTier: sb.policyTier ?? null, + policyTier: normalizePolicyTierName(sb.policyTier), policyPresetsFinalized, }); log( @@ -215,9 +225,16 @@ export async function runRebuildPostRestorePhase( } printMcpRestoreRecovery(sandboxName, mcpBridgeRestoreUnverified); if (policyPresetRestoreIncomplete) { - console.log( - ` Policy presets failed to reapply: ${failedPresets.join(", ")} \u2014 re-apply manually with \`${CLI_NAME} ${sandboxName} policy-add\``, - ); + if (failedPresets.length > 0) { + console.log( + ` Policy presets failed to reapply: ${failedPresets.join(", ")} \u2014 re-apply manually with \`${CLI_NAME} ${sandboxName} policy-add\``, + ); + } + if (failedPresetRemovals.length > 0 || !policyPresetReconciliationVerified) { + console.log( + ` Exact live policy reconciliation was incomplete${failedPresetRemovals.length > 0 ? `; remove failed: ${failedPresetRemovals.join(", ")}` : ""} \u2014 reconcile manually with \`${CLI_NAME} ${sandboxName} policy-add\` or \`${CLI_NAME} ${sandboxName} policy-remove\``, + ); + } } } if (recoveryRecreate && staleSandboxWasLocked) { @@ -225,6 +242,10 @@ export async function runRebuildPostRestorePhase( ` ${YW}\u26a0${R} Shields were previously enabled but the recreated sandbox starts unlocked \u2014 run \`${CLI_NAME} ${sandboxName} shields up\` to restore lockdown.`, ); } + if (failedPresetRemovals.length > 0 || !policyPresetReconciliationVerified) { + bail(`Rebuild completed with unverified live policy reconciliation for '${sandboxName}'.`); + return; + } if (preparedBackupRecovery && !postRestoreComplete) { bail( `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, diff --git a/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts b/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts index ca551e78eb..d81cfc7a10 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts @@ -29,6 +29,7 @@ export function createRebuildCommandContext( bail: RebuildBail; log: RebuildLog; requestedToolDisclosure: ToolDisclosure | undefined; + requestedObservabilityEnabled: boolean | undefined; skipConfirm: boolean; } { const normalized = normalizeRebuildSandboxOptions(options); @@ -39,6 +40,7 @@ export function createRebuildCommandContext( console.error(` ${D}[rebuild ${new Date().toISOString()}] ${redact(message)}${R}`) : () => {}, requestedToolDisclosure: normalized.toolDisclosure, + requestedObservabilityEnabled: normalized.observabilityEnabled, skipConfirm: normalized.yes === true || normalized.force === true, bail: opts.throwOnError ? (message: string) => { diff --git a/src/lib/actions/sandbox/rebuild-preflight-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-phase.ts index 0b54620e08..bfb7f87b2d 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-phase.ts @@ -30,6 +30,7 @@ import { getRebuildAgentDisplayName, type RebuildVersionCheck, } from "./rebuild-preflight-confirmation"; +import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; import { acquireRebuildOnboardLock, assertRebuildEntryUnchanged, @@ -76,10 +77,8 @@ export async function runRebuildPreflightPhase( options: string[] | RebuildSandboxOptions = {}, opts: RebuildSandboxExecutionOptions = {}, ): Promise { - const { log, bail, requestedToolDisclosure, skipConfirm } = createRebuildCommandContext( - options, - opts, - ); + const { log, bail, requestedToolDisclosure, requestedObservabilityEnabled, skipConfirm } = + createRebuildCommandContext(options, opts); const activeSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); const sandboxEntry = getRebuildSandboxEntryOrBail(sandboxName, bail); if (!sandboxEntry) return null; @@ -96,6 +95,15 @@ export async function runRebuildPreflightPhase( if (!isSingleAgentRebuildSupported(sandboxEntry, bail)) return null; const rebuildAgent = sandboxEntry.agent || null; + if (requestedObservabilityEnabled !== undefined && !isDcodeRebuildAgent(rebuildAgent)) { + printRebuildPreflightFailure( + "the observability override is supported only for managed LangChain Deep Agents Code sandboxes.", + "Remove --observability/--no-observability or select a managed Deep Agents Code sandbox.", + "Unsupported rebuild observability override", + bail, + ); + return null; + } const agentName = getRebuildAgentDisplayName(sandboxName); const dcodePreflight = createDcodeRebuildOrchestrator({ sandboxName, @@ -138,6 +146,7 @@ export async function runRebuildPreflightPhase( // succeeded, matching the previous `skipConfirm || confirmed` contract. autoYes: true, requestedToolDisclosure, + requestedObservabilityEnabled, allowLegacyManagedImageRecovery, // A validated prepared backup is the only path allowed to reconstruct // a missing gateway provider and route during recreate. The exact diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index e82321d17f..b5e778517e 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -50,6 +50,7 @@ export async function prepareRebuildTargetPreflights(args: { rebuildAgent: string | null; autoYes: boolean; requestedToolDisclosure?: ToolDisclosure; + requestedObservabilityEnabled?: boolean; allowLegacyManagedImageRecovery?: boolean; preparedBackupRecovery?: boolean; log: RebuildLog; @@ -61,6 +62,7 @@ export async function prepareRebuildTargetPreflights(args: { rebuildAgent, autoYes, requestedToolDisclosure, + requestedObservabilityEnabled, allowLegacyManagedImageRecovery, preparedBackupRecovery, log, @@ -98,6 +100,9 @@ export async function prepareRebuildTargetPreflights(args: { // session. Use that authoritative value for both preflight and inner onboard, // never the raw registry fallback used while constructing generic options. recreateOptions.toolDisclosure = durableConfig.toolDisclosure; + recreateOptions.observabilityEnabled = + requestedObservabilityEnabled ?? recreateOptions.observabilityEnabled; + recreateOptions.observabilityRequestedExplicitly = requestedObservabilityEnabled !== undefined; if ( !stageRebuildHermesDashboardConfig( rebuildAgent, diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts new file mode 100644 index 0000000000..426d5fbc98 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; +import type { Session } from "../../state/onboard-session"; +import * as onboardSession from "../../state/onboard-session"; +import type { RebuildDurableConfig } from "./rebuild-durable-config"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { rebuildOnboardDependencies } from "./rebuild-onboard-dependencies"; +import { type RebuildRecreatePhaseInput, runRebuildRecreatePhase } from "./rebuild-recreate-phase"; +import type { RebuildResumeConfig } from "./rebuild-resume-config"; + +const DCODE_AGENT = "langchain-deepagents-code"; + +const durableConfig: RebuildDurableConfig = { + fromDockerfile: null, + fromDockerfileError: null, + hermesAuthMethod: null, + hermesAuthMethodError: null, + webSearchConfig: null, + webSearchError: null, + toolDisclosure: "progressive", + toolDisclosureError: null, +}; + +const resumeConfig: RebuildResumeConfig = { + agent: DCODE_AGENT, + provider: "nvidia", + model: "nvidia/llama-3.3-nemotron-super-49b-v1.5", + nimContainer: null, + credentialEnv: "NVIDIA_API_KEY", + preferredInferenceApi: null, + compatibleEndpointReasoning: null, + pinEndpoint: true, + endpointUrl: "https://integrate.api.nvidia.com/v1", + registryInferenceRoute: null, + ambient: { presentVars: [], agentMismatch: null }, +}; + +const recreateOptions: RebuildRecreateOnboardOpts = { + resume: true, + nonInteractive: true, + recreateSandbox: true, + authoritativeResumeConfig: true, + acceptThirdPartySoftware: true, + agent: DCODE_AGENT, + fromDockerfile: null, + sandboxGpu: null, + sandboxGpuDevice: null, + controlUiPort: null, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + onboardLockAlreadyHeld: true, + autoYes: true, + toolDisclosure: "progressive", + observabilityEnabled: true, + observabilityRequestedExplicitly: true, + policyTier: "restricted", + baseImageResolutionHint: null, +}; + +function makeInput(overrides: Partial = {}): RebuildRecreatePhaseInput { + return { + sandboxName: "alpha", + sandboxEntry: { + name: "alpha", + agent: DCODE_AGENT, + observabilityEnabled: true, + policyTier: "restricted", + }, + sessionSnapshot: onboardSession.createSession({ + sandboxName: "alpha", + observabilityEnabled: false, + }), + sessionMatchesSandbox: true, + durableConfig, + resumeConfig, + recreateOptions, + fromDockerfile: null, + rebuildAgent: DCODE_AGENT, + messagingPlan: null, + rebuildsHermesSandbox: false, + hermesToolGateways: [], + hasHermesToolGateways: false, + sessionPolicyPresets: ["observability-otlp-local"], + credentialEnv: "NVIDIA_API_KEY", + baseImagePreflight: { ok: true, imageRef: null, overrideEnvVar: null }, + recoveryRecreate: false, + registryRollback: { recordRemoval: vi.fn(), restoreForRetry: vi.fn() }, + backupManifest: null, + mcpEntries: [], + rebuildShieldsWindow: { relocked: false, wasLocked: false }, + relockShieldsIfNeeded: vi.fn(() => true), + onCreated: vi.fn(), + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(`bail: ${message}`); + }), + ...overrides, + }; +} + +describe("runRebuildRecreatePhase observability handoff", () => { + let session: Session; + + beforeEach(() => { + session = onboardSession.createSession({ + sandboxName: "alpha", + observabilityEnabled: false, + }); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { + session = mutator(session) ?? session; + return session; + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("persists enabled observability before inner onboard and through successful recreate", async () => { + const observedAtOnboard: boolean[] = []; + const observedAtCreated: boolean[] = []; + vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async (options) => { + observedAtOnboard.push(onboardSession.loadSession()?.observabilityEnabled === true); + expect(options.observabilityEnabled).toBe(true); + }); + const input = makeInput({ + onCreated: vi.fn(() => { + observedAtCreated.push(onboardSession.loadSession()?.observabilityEnabled === true); + }), + }); + + await expect(runRebuildRecreatePhase(input)).resolves.toBe(true); + + expect(observedAtOnboard).toEqual([true]); + expect(observedAtCreated).toEqual([true]); + expect(onboardSession.loadSession()?.observabilityEnabled).toBe(true); + expect(onboardSession.loadSession()?.observabilityRequestedExplicitly).toBe(true); + expect(input.onCreated).toHaveBeenCalledOnce(); + expect(input.registryRollback.restoreForRetry).not.toHaveBeenCalled(); + expect(input.bail).not.toHaveBeenCalled(); + }); + + it("retains inherited observability provenance through inner onboard handoff", async () => { + vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async (options) => { + expect(options.observabilityEnabled).toBe(true); + expect(options.observabilityRequestedExplicitly).toBe(false); + expect(onboardSession.loadSession()?.observabilityRequestedExplicitly).toBe(false); + }); + + await expect( + runRebuildRecreatePhase( + makeInput({ + recreateOptions: { + ...recreateOptions, + observabilityRequestedExplicitly: false, + }, + }), + ), + ).resolves.toBe(true); + + expect(onboardSession.loadSession()?.observabilityEnabled).toBe(true); + expect(onboardSession.loadSession()?.observabilityRequestedExplicitly).toBe(false); + }); + + it("pins the authoritative restricted tier during recreate and restores ambient policy input", async () => { + const previousPolicyTier = process.env.NEMOCLAW_POLICY_TIER; + process.env.NEMOCLAW_POLICY_TIER = "open"; + try { + let observedTier: string | undefined; + vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async () => { + observedTier = process.env.NEMOCLAW_POLICY_TIER; + }); + + await expect(runRebuildRecreatePhase(makeInput())).resolves.toBe(true); + + expect(observedTier).toBe("restricted"); + expect(process.env.NEMOCLAW_POLICY_TIER).toBe("open"); + } finally { + restoreEnv("NEMOCLAW_POLICY_TIER", previousPolicyTier); + } + }); + + it("retains enabled observability through inner onboard failure, recovery, and bail", async () => { + const checkpoints: Array<[string, boolean]> = []; + vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async (options) => { + checkpoints.push([ + "onboard", + options.observabilityEnabled === true && + onboardSession.loadSession()?.observabilityEnabled === true, + ]); + throw new Error("inner onboard failed"); + }); + const input = makeInput({ + recoveryRecreate: true, + registryRollback: { + recordRemoval: vi.fn(), + restoreForRetry: vi.fn(() => { + checkpoints.push([ + "rollback", + onboardSession.loadSession()?.observabilityEnabled === true, + ]); + }), + }, + relockShieldsIfNeeded: vi.fn(() => { + checkpoints.push(["relock", onboardSession.loadSession()?.observabilityEnabled === true]); + return true; + }), + bail: vi.fn((message: string): never => { + checkpoints.push(["bail", onboardSession.loadSession()?.observabilityEnabled === true]); + throw new Error(`bail: ${message}`); + }), + }); + + await expect(runRebuildRecreatePhase(input)).rejects.toThrow( + "bail: Recreate failed (stale-sandbox recovery).", + ); + + expect(checkpoints).toEqual([ + ["onboard", true], + ["rollback", true], + ["relock", true], + ["bail", true], + ]); + expect(onboardSession.loadSession()?.observabilityEnabled).toBe(true); + expect(input.registryRollback.restoreForRetry).toHaveBeenCalledOnce(); + expect(input.relockShieldsIfNeeded).toHaveBeenCalledWith(false); + expect(input.onCreated).not.toHaveBeenCalled(); + expect(input.bail).toHaveBeenCalledWith("Recreate failed (stale-sandbox recovery).", 1); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index 8506d7bffb..c81eaa7475 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -110,6 +110,8 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): hermesAuthMethod: rebuildDurableConfig.hermesAuthMethod, webSearchConfig: rebuildDurableConfig.webSearchConfig, toolDisclosure: rebuildDurableConfig.toolDisclosure, + observabilityEnabled: recreateOptions.observabilityEnabled, + observabilityRequestedExplicitly: recreateOptions.observabilityRequestedExplicitly, telegramConfig: sessionMatchesSandbox ? sessionBefore?.telegramConfig : null, wechatConfig: sessionMatchesSandbox ? sessionBefore?.wechatConfig : null, migratedLegacyValueHashes: sessionMatchesSandbox @@ -149,6 +151,8 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): s.compatibleEndpointReasoning = resumeConfig.compatibleEndpointReasoning; s.endpointUrl = resumeConfig.endpointUrl; s.toolDisclosure = rebuildDurableConfig.toolDisclosure; + s.observabilityEnabled = recreateOptions.observabilityEnabled; + s.observabilityRequestedExplicitly = recreateOptions.observabilityRequestedExplicitly; return s; }); const sessionAfter = onboardSession.loadSession(); @@ -178,6 +182,9 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): const restoreAmbientRecreateEnv = isolateAmbientRecreateEnv(); const previousSandboxName = process.env.NEMOCLAW_SANDBOX_NAME; process.env.NEMOCLAW_SANDBOX_NAME = sandboxName; + if (recreateOptions.policyTier) { + process.env.NEMOCLAW_POLICY_TIER = recreateOptions.policyTier; + } const restoreRebuildBaseImageOverride = pinRebuildAgentBaseImageForRecreate(rebuildBaseImagePreflight); try { @@ -224,6 +231,10 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): sandboxName, rebuildMcpEntries, rebuildDurableConfig.toolDisclosure, + { + enabled: recreateOptions.observabilityEnabled, + requestedExplicitly: recreateOptions.observabilityRequestedExplicitly, + }, ); if (backupManifest) { console.error(" 3. Then restore your workspace state:"); diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts index c552fb6bbb..99d28c8714 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as policies from "../../policy"; import * as sandboxState from "../../state/sandbox"; @@ -9,7 +9,17 @@ import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; import { resolveRestoredPolicyRegistryState } from "./rebuild-post-restore-phase"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; +const BUILTIN_OBSERVABILITY_CONTENT = + "network_policies:\n observability-otlp-local:\n name: observability-otlp-local\n"; + describe("rebuild policy restore fidelity", () => { + beforeEach(() => { + vi.spyOn(policies, "loadPresetForSandbox").mockImplementation((_sandboxName, presetName) => + presetName === "observability-otlp-local" ? BUILTIN_OBSERVABILITY_CONTENT : null, + ); + vi.spyOn(policies, "getPresetContentGatewayState").mockReturnValue("absent"); + }); + afterEach(() => { vi.restoreAllMocks(); }); @@ -17,6 +27,7 @@ describe("rebuild policy restore fidelity", () => { it("replays custom web-policy names from exact content instead of same-name built-ins", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); + const parsePresetPolicyKeys = vi.spyOn(policies, "parsePresetPolicyKeys"); vi.spyOn(sandboxState, "restoreSandboxState").mockReturnValue({ success: true, restoredDirs: [], @@ -31,7 +42,6 @@ describe("rebuild policy restore fidelity", () => { content: `network_policies:\n ${name}-custom:\n name: ${name}-custom\n`, sourcePath: `/tmp/${name}.yaml`, })); - const result = runRebuildRestorePhase({ sandboxName: "alpha", backupManifest: { @@ -40,6 +50,7 @@ describe("rebuild policy restore fidelity", () => { } as never, policyPresets: ["npm", "brave", "tavily", "nous-web"], customPolicies, + reconcileManagedDcodeObservability: false, log: vi.fn(), }); @@ -52,6 +63,10 @@ describe("rebuild policy restore fidelity", () => { } expect(result.restoredPresets).toEqual(["npm", "brave", "tavily", "nous-web"]); expect(result.failedPresets).toEqual([]); + expect(result.finalPresets).toEqual(["npm", "brave", "tavily", "nous-web"]); + expect(result.policyPresetReconciliationVerified).toBe(true); + expect(policies.loadPresetForSandbox).not.toHaveBeenCalled(); + expect(parsePresetPolicyKeys).not.toHaveBeenCalled(); }); it("replays captured registry custom policies during stale recovery without a backup", () => { @@ -71,12 +86,12 @@ describe("rebuild policy restore fidelity", () => { sourcePath: "/tmp/custom-egress.yaml", }, ]; - const result = runRebuildRestorePhase({ sandboxName: "alpha", backupManifest: null, policyPresets: [], customPolicies, + reconcileManagedDcodeObservability: false, log: vi.fn(), }); @@ -87,6 +102,7 @@ describe("rebuild policy restore fidelity", () => { { custom: { sourcePath: "/tmp/custom-egress.yaml" } }, ); expect(result.restoredPresets).toEqual(["custom-egress"]); + expect(result.finalPresets).toEqual(["custom-egress"]); }); it("leaves generated MCP policy replay exclusively to MCP restoration", () => { @@ -103,12 +119,12 @@ describe("rebuild policy restore fidelity", () => { "network_policies:\n mcp-bridge-search:\n endpoints:\n - host: mcp.example.com\n allowed_ips: [203.0.113.10]\n", sourcePath: MCP_BRIDGE_POLICY_SOURCE, }; - const result = runRebuildRestorePhase({ sandboxName: "alpha", backupManifest: null, policyPresets: [], customPolicies: [genuineCustomPolicy, generatedMcpPolicy], + reconcileManagedDcodeObservability: false, log: vi.fn(), }); @@ -123,26 +139,336 @@ describe("rebuild policy restore fidelity", () => { expect(result.failedPresets).toEqual([]); }); - it("keeps finalized custom-only policy state empty after exact replay", () => { + it("removes an observability preset introduced while rebuilding a restricted sandbox", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(policies, "applyPreset").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState") + .mockReturnValueOnce("match") + .mockReturnValueOnce("absent"); + const removePreset = vi.spyOn(policies, "removePreset").mockReturnValue(true); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["npm"], + customPolicies: [], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(removePreset).toHaveBeenCalledWith("alpha", "observability-otlp-local", { + nonFatal: true, + }); + expect(result.finalPresets).toEqual(["npm"]); + expect(result.failedPresetRemovals).toEqual([]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("retains an observed exact built-in when post-removal verification is unavailable", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(policies, "applyPreset").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState") + .mockReturnValueOnce("match") + .mockReturnValueOnce(null); + vi.spyOn(policies, "removePreset").mockReturnValue(true); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["npm"], + customPolicies: [], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(result.finalPresets).toEqual(["npm", "observability-otlp-local"]); + expect(result.policyPresetReconciliationVerified).toBe(false); + }); + + it("accounts for known failed additions without treating a narrower live set as unverified", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(policies, "applyPreset") + .mockImplementationOnce((_name, presetName) => { + expect(presetName).toBe("npm"); + return true; + }) + .mockImplementationOnce((_name, presetName) => { + expect(presetName).toBe("bad"); + return false; + }) + .mockImplementationOnce((_name, presetName) => { + expect(presetName).toBe("throw"); + throw new Error("apply failed"); + }); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["npm", "bad", "throw"], + customPolicies: [], + reconcileManagedDcodeObservability: false, + log: vi.fn(), + }); + + expect(result.restoredPresets).toEqual(["npm"]); + expect(result.failedPresets).toEqual(["bad", "throw"]); + expect(result.finalPresets).toEqual(["npm"]); + expect(result.failedPresetRemovals).toEqual([]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("keeps reconciliation unverified when a reported successful addition is missing live", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(policies, "applyPreset").mockReturnValue(true); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["observability-otlp-local"], + customPolicies: [], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(result.restoredPresets).toEqual(["observability-otlp-local"]); + expect(result.failedPresets).toEqual([]); + expect(result.finalPresets).toEqual([]); + expect(result.policyPresetReconciliationVerified).toBe(false); + }); + + it("retains target built-in attribution when exact post-apply verification is unavailable", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(policies, "applyPreset").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState").mockReturnValue(null); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["observability-otlp-local"], + customPolicies: [], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(result.finalBuiltinPresets).toEqual(["observability-otlp-local"]); + expect(result.policyPresetReconciliationVerified).toBe(false); + }); + + it("does not remove or persist DCode base-policy keys detected as broad presets", () => { + const removePreset = vi.spyOn(policies, "removePreset"); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: [], + customPolicies: [], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(removePreset).not.toHaveBeenCalled(); + expect(result.finalPresets).toEqual([]); + expect(result.failedPresetRemovals).toEqual([]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("leaves a same-name custom observability policy outside built-in narrowing", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const applyPresetContent = vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + const removePreset = vi.spyOn(policies, "removePreset"); + const customPolicy = { + name: "observability-otlp-local", + content: "network_policies:\n operator-collector: {}\n", + sourcePath: "/tmp/operator-collector.yaml", + }; + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: [], + customPolicies: [customPolicy], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(applyPresetContent).toHaveBeenCalledWith( + "alpha", + customPolicy.name, + customPolicy.content, + { custom: { sourcePath: customPolicy.sourcePath } }, + ); + expect(removePreset).not.toHaveBeenCalled(); + expect(result.finalPresets).toEqual([customPolicy.name]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("leaves a differently named custom policy owning observability egress outside built-in narrowing", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const applyPresetContent = vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + const exactState = vi.spyOn(policies, "getPresetContentGatewayState").mockReturnValue("match"); + const removePreset = vi.spyOn(policies, "removePreset"); + const customPolicy = { + name: "corp-otel", + content: + "network_policies:\n observability-otlp-local:\n endpoints:\n - host: collector.corp.example\n", + sourcePath: "/tmp/corp-otel.yaml", + }; + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: [], + customPolicies: [customPolicy], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(applyPresetContent).toHaveBeenCalledWith( + "alpha", + customPolicy.name, + customPolicy.content, + { custom: { sourcePath: customPolicy.sourcePath } }, + ); + expect(exactState).toHaveBeenCalledWith("alpha", customPolicy.content); + expect(removePreset).not.toHaveBeenCalled(); + expect(result.finalPresets).toEqual([customPolicy.name]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("removes an exact inner built-in behind a same-name custom with a different key", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const customPolicy = { + name: "observability-otlp-local", + content: "network_policies:\n operator-collector: {}\n", + sourcePath: "/tmp/operator-collector.yaml", + }; + vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState") + .mockReturnValueOnce("match") + .mockReturnValueOnce("absent"); + const removePreset = vi.spyOn(policies, "removePreset").mockReturnValue(true); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["observability-otlp-local"], + customPolicies: [customPolicy], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(removePreset).toHaveBeenCalledWith("alpha", "observability-otlp-local", { + nonFatal: true, + }); + expect(result.finalPresets).toEqual([customPolicy.name]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("removes an exact inner built-in when overlapping custom replay fails", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const customPolicy = { + name: "corp-otel", + content: "network_policies:\n observability-otlp-local: {}\n", + sourcePath: "/tmp/corp-otel.yaml", + }; + vi.spyOn(policies, "applyPresetContent").mockReturnValue(false); + vi.spyOn(policies, "getPresetContentGatewayState") + .mockReturnValueOnce("match") + .mockReturnValueOnce("absent"); + const removePreset = vi.spyOn(policies, "removePreset").mockReturnValue(true); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: [], + customPolicies: [customPolicy], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(result.failedPresets).toEqual([customPolicy.name]); + expect(removePreset).toHaveBeenCalledWith("alpha", "observability-otlp-local", { + nonFatal: true, + }); + expect(result.finalPresets).toEqual([]); + expect(result.policyPresetReconciliationVerified).toBe(true); + }); + + it("leaves drift untouched and unverified when successful custom ownership is not exact", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const customPolicy = { + name: "corp-otel", + content: "network_policies:\n observability-otlp-local: {}\n", + sourcePath: "/tmp/corp-otel.yaml", + }; + vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState").mockReturnValue("drift"); + const removePreset = vi.spyOn(policies, "removePreset"); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: [], + customPolicies: [customPolicy], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(removePreset).not.toHaveBeenCalled(); + expect(result.finalPresets).toEqual([customPolicy.name]); + expect(result.policyPresetReconciliationVerified).toBe(false); + }); + + it("retains separate built-in attribution when same-name custom removal is unverified", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const customPolicy = { + name: "observability-otlp-local", + content: "network_policies:\n operator-collector: {}\n", + sourcePath: "/tmp/operator-collector.yaml", + }; + vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState") + .mockReturnValueOnce("match") + .mockReturnValueOnce(null); + vi.spyOn(policies, "removePreset").mockReturnValue(true); + + const result = runRebuildRestorePhase({ + sandboxName: "alpha", + backupManifest: null, + policyPresets: ["observability-otlp-local"], + customPolicies: [customPolicy], + reconcileManagedDcodeObservability: true, + log: vi.fn(), + }); + + expect(result.finalPresets).toEqual(["observability-otlp-local"]); + expect(result.finalBuiltinPresets).toEqual(["observability-otlp-local"]); + expect(result.policyPresetReconciliationVerified).toBe(false); expect( resolveRestoredPolicyRegistryState( - { - customPolicies: [{ name: "tavily", content: "allow: []" }], - policyPresetsFinalized: true, - }, - ["tavily"], - [], + { policyPresetsFinalized: true }, + result.finalBuiltinPresets, + result.failedPresets, + result.policyPresetReconciliationVerified, ), - ).toEqual({ policies: [], policyPresetsFinalized: true }); + ).toEqual({ + policies: ["observability-otlp-local"], + policyPresetsFinalized: undefined, + }); + }); + + it("keeps finalized custom-only policy state empty after exact replay", () => { + expect(resolveRestoredPolicyRegistryState({ policyPresetsFinalized: true }, [], [])).toEqual({ + policies: [], + policyPresetsFinalized: true, + }); expect( - resolveRestoredPolicyRegistryState( - { - customPolicies: [{ name: "tavily", content: "allow: []" }], - policyPresetsFinalized: true, - }, - [], - ["tavily"], - ).policyPresetsFinalized, + resolveRestoredPolicyRegistryState({ policyPresetsFinalized: true }, [], ["tavily"]) + .policyPresetsFinalized, ).toBeUndefined(); }); }); diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.ts b/src/lib/actions/sandbox/rebuild-restore-phase.ts index 93925529e6..c658496843 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.ts @@ -3,6 +3,10 @@ import { CLI_NAME } from "../../cli/branding"; import { G, R, YW } from "../../cli/terminal-style"; +import { + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + OBSERVABILITY_POLICY_BINDING, +} from "../../onboard/observability-policy-presets"; import * as policies from "../../policy"; import * as sandboxState from "../../state/sandbox"; import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; @@ -15,6 +19,7 @@ export interface RebuildRestorePhaseInput { backupManifest: RebuildBackupManifest; policyPresets: string[]; customPolicies: NonNullable; + reconcileManagedDcodeObservability: boolean; log: RebuildLog; } @@ -22,6 +27,145 @@ export interface RebuildRestorePhaseResult { restoreSucceeded: boolean; restoredPresets: string[]; failedPresets: string[]; + finalPresets: string[]; + finalBuiltinPresets: string[]; + failedPresetRemovals: string[]; + policyPresetReconciliationVerified: boolean; +} + +function uniquePresetNames(names: readonly string[]): string[] { + return [...new Set(names)]; +} + +function isManagedObservabilityPreset(name: string): boolean { + return OBSERVABILITY_POLICY_BINDING.matchesPreset(name); +} + +function finalRestoredPresetState( + restoredBuiltinPresets: readonly string[], + restoredCustomPresets: readonly string[], + includeManagedObservability: boolean, +): Pick { + const finalBuiltinPresets = uniquePresetNames( + OBSERVABILITY_POLICY_BINDING.setAttribution( + restoredBuiltinPresets, + includeManagedObservability, + ), + ); + return { + finalBuiltinPresets, + finalPresets: uniquePresetNames([...finalBuiltinPresets, ...restoredCustomPresets]), + }; +} + +function reconcileFinalManagedObservability( + sandboxName: string, + targetManagedObservability: boolean, + restoredBuiltinPresets: readonly string[], + restoredCustomPresets: readonly string[], + failedBuiltinPresets: readonly string[], + successfulCustomObservabilityContents: readonly string[], + log: RebuildLog, +): Pick< + RebuildRestorePhaseResult, + | "finalPresets" + | "finalBuiltinPresets" + | "failedPresetRemovals" + | "policyPresetReconciliationVerified" +> { + const customObservabilityStates = successfulCustomObservabilityContents.map((content) => + OBSERVABILITY_POLICY_BINDING.inspectContent(sandboxName, content, policies), + ); + const customObservabilityExpected = successfulCustomObservabilityContents.length > 0; + const customObservabilityVerified = customObservabilityStates.includes("match"); + if (!targetManagedObservability && customObservabilityVerified) { + return { + ...finalRestoredPresetState(restoredBuiltinPresets, restoredCustomPresets, false), + failedPresetRemovals: [], + policyPresetReconciliationVerified: true, + }; + } + + const loadedBinding = OBSERVABILITY_POLICY_BINDING.load(sandboxName, policies); + const builtinContent = loadedBinding.content; + if (!builtinContent) { + log("Could not load managed observability preset content after rebuild restore"); + console.error( + ` ${YW}\u26a0${R} Could not verify managed observability policy content after restore.`, + ); + return { + ...finalRestoredPresetState( + restoredBuiltinPresets, + restoredCustomPresets, + targetManagedObservability, + ), + failedPresetRemovals: [], + policyPresetReconciliationVerified: false, + }; + } + + const liveBefore = loadedBinding.state; + const failedPresetRemovals: string[] = []; + let liveAfter = liveBefore; + if (!targetManagedObservability && liveBefore === "match") { + log(`Removing unexpected live preset: ${OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET}`); + const removal = OBSERVABILITY_POLICY_BINDING.removeExact( + sandboxName, + builtinContent, + policies, + { + knownBefore: liveBefore, + removeOptions: { nonFatal: true }, + }, + ); + if (removal.reportedSuccess !== true) { + failedPresetRemovals.push(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET); + } + if (removal.errorMessage) { + log( + `Failed to remove unexpected live preset '${OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET}': ${removal.errorMessage}`, + ); + } + liveAfter = removal.after; + } + + const failedManagedAddition = failedBuiltinPresets.some(isManagedObservabilityPreset); + const customReplayVerified = !customObservabilityExpected || customObservabilityVerified; + const managedTargetVerified = targetManagedObservability + ? liveAfter === "match" || (liveAfter === "absent" && failedManagedAddition) + : liveAfter === "absent"; + const policyPresetReconciliationVerified = + failedPresetRemovals.length === 0 && customReplayVerified && managedTargetVerified; + // Until an exact post-removal read proves absence, preserve attribution for + // the exact built-in content observed before mutation. Reconciliation stays + // unverified, but recovery does not forget policy that may still be live. + const includeManagedObservability = + liveAfter === "match" || + (targetManagedObservability && liveAfter !== "absent") || + (!targetManagedObservability && liveBefore === "match" && liveAfter !== "absent"); + const finalPresetState = finalRestoredPresetState( + restoredBuiltinPresets, + restoredCustomPresets, + includeManagedObservability, + ); + if (!policyPresetReconciliationVerified) { + const details = [ + ...(!customReplayVerified ? ["custom observability content not verified live"] : []), + ...(!managedTargetVerified + ? [`managed observability state ${liveAfter ?? "unavailable"}`] + : []), + ...(failedPresetRemovals.length > 0 + ? [`remove failed ${failedPresetRemovals.join(", ")}`] + : []), + ]; + console.error( + ` ${YW}\u26a0${R} Final live policy preset reconciliation is incomplete: ${details.join("; ")}.`, + ); + } + log( + `Final managed observability state: ${liveAfter ?? "unavailable"}; customOwned=${String(customObservabilityVerified)}; builtins=[${finalPresetState.finalBuiltinPresets.join(",")}]; presets=[${finalPresetState.finalPresets.join(",")}]; verified=${String(policyPresetReconciliationVerified)}`, + ); + return { ...finalPresetState, failedPresetRemovals, policyPresetReconciliationVerified }; } /** @@ -30,7 +174,14 @@ export interface RebuildRestorePhaseResult { * stale recovery, successful presets, and incomplete preset recovery reporting. */ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): RebuildRestorePhaseResult { - const { sandboxName, backupManifest, policyPresets, customPolicies, log } = input; + const { + sandboxName, + backupManifest, + policyPresets, + customPolicies, + reconcileManagedDcodeObservability, + log, + } = input; let restoreSucceeded = true; if (backupManifest) { console.log(""); @@ -55,13 +206,17 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild } } - const restoredPresets: string[] = []; + const restoredBuiltinPresets: string[] = []; + const restoredCustomPresets: string[] = []; const failedPresets: string[] = []; + const failedBuiltinPresets: string[] = []; + const successfulCustomObservabilityContents: string[] = []; const customPolicyNames = new Set(customPolicies.map((entry) => entry.name)); const replayableCustomPolicies = customPolicies.filter( (entry) => entry.sourcePath !== MCP_BRIDGE_POLICY_SOURCE, ); const builtinPolicyPresets = policyPresets.filter((name) => !customPolicyNames.has(name)); + const targetManagedObservability = builtinPolicyPresets.some(isManagedObservabilityPreset); if (builtinPolicyPresets.length > 0 || replayableCustomPolicies.length > 0) { console.log(""); console.log(" Restoring policy presets..."); @@ -70,11 +225,16 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild try { log(`Applying preset: ${presetName}`); const applied = policies.applyPreset(sandboxName, presetName); - if (applied) restoredPresets.push(presetName); - else failedPresets.push(presetName); + if (applied) { + restoredBuiltinPresets.push(presetName); + } else { + failedBuiltinPresets.push(presetName); + failedPresets.push(presetName); + } } catch (error) { const message = error instanceof Error ? error.message : String(error); log(`Failed to apply preset '${presetName}': ${message}`); + failedBuiltinPresets.push(presetName); failedPresets.push(presetName); } } @@ -84,14 +244,27 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild const applied = policies.applyPresetContent(sandboxName, entry.name, entry.content, { custom: { sourcePath: entry.sourcePath }, }); - if (applied) restoredPresets.push(entry.name); - else failedPresets.push(entry.name); + if (applied) { + restoredCustomPresets.push(entry.name); + if ( + reconcileManagedDcodeObservability && + OBSERVABILITY_POLICY_BINDING.ownsContent(entry.content) + ) { + successfulCustomObservabilityContents.push(entry.content); + } + } else { + failedPresets.push(entry.name); + } } catch (error) { const message = error instanceof Error ? error.message : String(error); log(`Failed to apply custom preset '${entry.name}': ${message}`); failedPresets.push(entry.name); } } + const restoredPresets = uniquePresetNames([ + ...restoredBuiltinPresets, + ...restoredCustomPresets, + ]); if (restoredPresets.length > 0) { console.log(` ${G}\u2713${R} Policy presets restored: ${restoredPresets.join(", ")}`); } @@ -101,5 +274,22 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild } } - return { restoreSucceeded, restoredPresets, failedPresets }; + const restoredPresets = uniquePresetNames([...restoredBuiltinPresets, ...restoredCustomPresets]); + const finalPolicyState = reconcileManagedDcodeObservability + ? reconcileFinalManagedObservability( + sandboxName, + targetManagedObservability, + restoredBuiltinPresets, + restoredCustomPresets, + failedBuiltinPresets, + successfulCustomObservabilityContents, + log, + ) + : { + finalBuiltinPresets: uniquePresetNames(restoredBuiltinPresets), + finalPresets: restoredPresets, + failedPresetRemovals: [], + policyPresetReconciliationVerified: true, + }; + return { restoreSucceeded, restoredPresets, failedPresets, ...finalPolicyState }; } diff --git a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts index a19923f3f0..e4dce6b54a 100644 --- a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts +++ b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts @@ -38,6 +38,7 @@ describe("rebuild shields relock guard", () => { phaseMocks.runPreflight.mockResolvedValue({ sandboxEntry: { name: "alpha", customPolicies: [] }, targetConfig: { durableConfig: { webSearchConfig: null } }, + recreateOptions: { observabilityEnabled: false }, liveState: { staleRecovery: false, staleRegistrySnapshot: null }, recoveryManifest: null, dcodePreflight: { cleanup: cleanupDcodePreflight }, diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index d57c5f461b..aa2d0a1499 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -22,6 +22,7 @@ type SandboxRecord = { gatewayName?: string | null; imageTag?: string | null; openshellDriver?: string | null; + observabilityEnabled?: boolean; provider?: string | null; model?: string | null; }; @@ -119,25 +120,40 @@ const applyPresetContentMock = vi.fn( (_sandbox: string, _name: string, _content: string, _options?: unknown) => true, ); const removePresetMock = vi.fn((_sandbox: string, _preset: string) => true); +const getPresetContentGatewayStateMock = vi.fn< + (_sandbox: string, _content: string, _policyKey?: string) => "match" | "absent" | "drift" | null +>(() => "absent"); +const builtinObservabilityPolicy = + "network_policies:\n observability-otlp-local:\n endpoints:\n - host: host.openshell.internal\n"; +const loadPresetForSandboxMock = vi.fn((_sandbox: string, preset: string) => + preset === "observability-otlp-local" ? builtinObservabilityPolicy : null, +); const getSandboxMock = vi.fn<(name?: string) => SandboxRecord | null>(() => null); const isGatewayHealthyMock = vi.fn(() => true); const listBackupsMock = vi.fn<() => Array>>(() => []); const parseLiveSandboxNamesMock = vi.fn(() => new Set(["alpha"])); const registerSandboxMock = vi.fn(); +const updateSandboxMock = vi.fn(); const restoreSandboxStateMock = vi.fn(); const runOpenshellMock = vi.fn((args: string[]) => { args[0] === "sandbox" && args[1] === "delete" && lifecycleMock.events.push("delete"); return { status: 0, output: "" }; }); -const streamSandboxCreateMock = vi.fn(async () => ({ - status: 0, - output: "", - forcedReady: false, -})); +const streamSandboxCreateMock = vi.fn( + async (_command: string, _env: NodeJS.ProcessEnv, _options?: Record) => ({ + status: 0, + output: "", + forcedReady: false, + }), +); const dcodeSandboxEntry = { name: "alpha", agent: "langchain-deepagents-code", }; +const latestBackupFixture = { + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", +}; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), @@ -167,6 +183,8 @@ vi.mock("../../policy", () => ({ applyPreset: applyPresetMock, applyPresetContent: applyPresetContentMock, getAppliedPresets: getAppliedPresetsMock, + getPresetContentGatewayState: getPresetContentGatewayStateMock, + loadPresetForSandbox: loadPresetForSandboxMock, removePreset: removePresetMock, })); @@ -217,6 +235,7 @@ vi.mock("../../state/registry", () => ({ }), registerSandbox: registerSandboxMock, removeSandbox: vi.fn(), + updateSandbox: updateSandboxMock, })); vi.mock("../../state/sandbox", () => ({ @@ -249,10 +268,15 @@ describe("runSandboxSnapshot", () => { applyPresetMock.mockReturnValue(true); applyPresetContentMock.mockReturnValue(true); removePresetMock.mockReturnValue(true); + getPresetContentGatewayStateMock.mockReturnValue("absent"); + loadPresetForSandboxMock.mockImplementation((_sandbox, preset) => + preset === "observability-otlp-local" ? builtinObservabilityPolicy : null, + ); getSandboxMock.mockReturnValue(null); isGatewayHealthyMock.mockReturnValue(true); listBackupsMock.mockReturnValue([]); registerSandboxMock.mockReset(); + updateSandboxMock.mockReset(); restoreSandboxStateMock.mockReturnValue({ success: true, restoredDirs: [], @@ -265,6 +289,7 @@ describe("runSandboxSnapshot", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); function mockDcodeProbe(state: DcodeProbeState, output = "") { @@ -859,10 +884,7 @@ describe("runSandboxSnapshot", () => { "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, }), ); - getLatestBackupMock.mockReturnValue({ - timestamp: "2026-06-15T00:00:00.000Z", - backupPath: "/tmp/backup-alpha", - }); + getLatestBackupMock.mockReturnValue({ ...latestBackupFixture }); restoreSandboxStateMock.mockReturnValue({ success: true, restoredDirs: ["workspace"], @@ -911,10 +933,7 @@ describe("runSandboxSnapshot", () => { "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, }), ); - getLatestBackupMock.mockReturnValue({ - timestamp: "2026-06-15T00:00:00.000Z", - backupPath: "/tmp/backup-alpha", - }); + getLatestBackupMock.mockReturnValue({ ...latestBackupFixture }); const { runSandboxSnapshot } = await import("./snapshot"); await expect( @@ -932,6 +951,444 @@ describe("runSandboxSnapshot", () => { expect(registerSandboxMock).not.toHaveBeenCalled(); }); + it.each([ + { enabled: true, assignmentPresent: true }, + { enabled: false, assignmentPresent: false }, + ])("starts a snapshot clone with the authoritative source observability state when enabled=$enabled", async ({ + enabled, + assignmentPresent, + }) => { + let registeredClone: SandboxRecord | null = null; + registerSandboxMock.mockImplementation((entry) => (registeredClone = entry as SandboxRecord)); + vi.stubEnv("NEMOCLAW_OBSERVABILITY", "1"); + getSandboxMock.mockImplementation((name) => + name === "alpha" + ? { + name: "alpha", + agent: "langchain-deepagents-code", + imageTag: "nemoclaw-alpha:test", + openshellDriver: "docker", + observabilityEnabled: enabled, + provider: "nvidia-nim", + model: "nvidia/model-a", + } + : registeredClone, + ); + captureOpenshellMock.mockImplementation((args) => + openshellResponses(args, { + "sandbox exec": { status: 0, output: dcodeProbeOutput("idle") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + getLatestBackupMock.mockReturnValue({ ...latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }); + const [createCommandValue, createEnv] = streamSandboxCreateMock.mock.calls[0] ?? []; + const createCommand = String(createCommandValue ?? ""); + expect(createCommand.includes("'NEMOCLAW_OBSERVABILITY=1'")).toBe(assignmentPresent); + expect(createEnv?.NEMOCLAW_OBSERVABILITY).toBeUndefined(); + expect(registerSandboxMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: "beta", + observabilityEnabled: enabled, + }), + ); + expect(applyPresetMock).toHaveBeenCalledTimes(enabled ? 1 : 0); + }); + + it.each([ + { label: "recorded", policyPresets: ["npm"] }, + { label: "legacy", policyPresets: undefined }, + ])("adds built-in OTLP egress for a $label snapshot", async ({ policyPresets }) => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: "balanced", + } as never); + getLatestBackupMock.mockReturnValue({ ...latestBackupFixture, policyPresets }); + getAppliedPresetsMock.mockReturnValue(["npm"]); + const { runSandboxSnapshot } = await import("./snapshot"); + await runSandboxSnapshot("alpha", { kind: "restore" }); + expect(applyPresetMock).toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(removePresetMock).not.toHaveBeenCalled(); + }); + + it("removes historical built-in OTLP egress when observability was disabled after the snapshot", async () => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: ["npm", "observability-otlp-local"], + }); + getAppliedPresetsMock.mockReturnValue(["npm", "observability-otlp-local"]); + getPresetContentGatewayStateMock.mockReturnValueOnce("match").mockReturnValueOnce("absent"); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(removePresetMock).toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(applyPresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + }); + + it("removes an exact unrecorded built-in OTLP policy when observability is disabled", async () => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + policies: [], + } as never); + getLatestBackupMock.mockReturnValue({ ...latestBackupFixture, policyPresets: [] }); + getAppliedPresetsMock.mockReturnValue([]); + getPresetContentGatewayStateMock.mockReturnValueOnce("match").mockReturnValueOnce("absent"); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(getPresetContentGatewayStateMock).toHaveBeenCalledWith( + "alpha", + builtinObservabilityPolicy, + ); + expect(removePresetMock).toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(updateSandboxMock).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "returns false", + configureRemoval: () => removePresetMock.mockReturnValue(false), + }, + { + label: "throws", + configureRemoval: () => + removePresetMock.mockImplementation(() => { + throw new Error("remove exploded"); + }), + }, + { + label: "claims success without removing", + configureRemoval: () => removePresetMock.mockReturnValue(true), + }, + ])("retains built-in OTLP attribution when removal $label", async ({ configureRemoval }) => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + policies: [], + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: [], + }); + getAppliedPresetsMock.mockReturnValue([]); + getPresetContentGatewayStateMock.mockReturnValue("match"); + configureRemoval(); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(getPresetContentGatewayStateMock).toHaveBeenCalledTimes(2); + expect(updateSandboxMock).toHaveBeenCalledWith("alpha", { + policies: ["observability-otlp-local"], + }); + expect(consoleWarn.mock.calls.flat().join("\n")).toContain( + "exact content still live after remove", + ); + }); + + it("does not resurrect an earlier removed preset while restoring unverified OTLP attribution", async () => { + let registryEntry = { + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + policies: ["github", "observability-otlp-local"], + }; + getSandboxMock.mockImplementation(() => registryEntry as never); + updateSandboxMock.mockImplementation((_sandboxName, update) => { + registryEntry = { ...registryEntry, ...(update as Partial) }; + }); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: [], + }); + getAppliedPresetsMock.mockReturnValue(["github", "observability-otlp-local"]); + getPresetContentGatewayStateMock.mockReturnValue("match"); + removePresetMock + .mockImplementationOnce((_sandboxName, presetName) => { + expect(presetName).toBe("github"); + registryEntry = { + ...registryEntry, + policies: registryEntry.policies.filter((name) => name !== "github"), + }; + return true; + }) + .mockReturnValue(true); + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(removePresetMock.mock.calls.map((call) => call[1])).toEqual([ + "github", + "observability-otlp-local", + ]); + expect(updateSandboxMock).toHaveBeenLastCalledWith("alpha", { + policies: ["observability-otlp-local"], + }); + expect(registryEntry.policies).toEqual(["observability-otlp-local"]); + }); + + it.each([ + { + label: "records an exact live enabled policy", + observabilityEnabled: true, + liveState: "match" as const, + policies: ["npm"], + expectedPolicies: ["npm", "observability-otlp-local"], + }, + { + label: "prunes an exact absent disabled policy", + observabilityEnabled: false, + liveState: "absent" as const, + policies: ["npm", "observability-otlp-local"], + expectedPolicies: ["npm"], + }, + ])("repairs stale OTLP registry state: $label", async ({ + observabilityEnabled, + liveState, + policies: recordedPolicies, + expectedPolicies, + }) => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled, + policyTier: "balanced", + policies: recordedPolicies, + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: ["npm"], + }); + getAppliedPresetsMock.mockReturnValue(recordedPolicies); + getPresetContentGatewayStateMock.mockReturnValue(liveState); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(updateSandboxMock).toHaveBeenCalledWith("alpha", { policies: expectedPolicies }); + expect(applyPresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(removePresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + }); + + it("does not let a same-name, different-key custom replay suppress stale built-in OTLP cleanup", async () => { + const customPolicy = { + name: "observability-otlp-local", + content: "network_policies:\n operator-collector: {}\n", + sourcePath: "/policies/operator-collector.yaml", + }; + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: [customPolicy.name], + customPolicies: [customPolicy], + }); + getCustomPoliciesMock.mockReturnValueOnce([]).mockReturnValue([customPolicy]); + getAppliedPresetsMock.mockReturnValue(["observability-otlp-local"]); + getPresetContentGatewayStateMock.mockReturnValueOnce("match").mockReturnValueOnce("absent"); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(applyPresetContentMock).toHaveBeenCalledWith( + "alpha", + customPolicy.name, + customPolicy.content, + { custom: { sourcePath: customPolicy.sourcePath } }, + ); + expect(removePresetMock).toHaveBeenCalledTimes(1); + expect(removePresetMock).toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(applyPresetMock).not.toHaveBeenCalledWith("alpha", customPolicy.name); + expect(updateSandboxMock).not.toHaveBeenCalled(); + }); + + it("lets successfully replayed corp-otel content own its exact live OTLP key", async () => { + const customPolicy = { + name: "corp-otel", + content: + "network_policies:\n observability-otlp-local:\n endpoints:\n - host: collector.corp.example\n", + sourcePath: "/policies/corp-otel.yaml", + }; + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + policies: ["npm", "observability-otlp-local"], + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: ["npm", "observability-otlp-local"], + customPolicies: [customPolicy], + }); + getCustomPoliciesMock.mockReturnValueOnce([]).mockReturnValue([customPolicy]); + getAppliedPresetsMock.mockReturnValue(["npm", "corp-otel", "observability-otlp-local"]); + getPresetContentGatewayStateMock.mockImplementation((_sandbox, content) => + content === customPolicy.content ? "match" : "drift", + ); + const { runSandboxSnapshot } = await import("./snapshot"); + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(applyPresetContentMock).toHaveBeenCalledWith( + "alpha", + customPolicy.name, + customPolicy.content, + { custom: { sourcePath: customPolicy.sourcePath } }, + ); + expect(applyPresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(removePresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(removePresetMock).not.toHaveBeenCalledWith("alpha", customPolicy.name); + expect(updateSandboxMock).toHaveBeenCalledWith("alpha", { policies: ["npm"] }); + expect(getPresetContentGatewayStateMock).toHaveBeenCalledTimes(1); + expect(getPresetContentGatewayStateMock.mock.calls[0]?.[1]).toBe(customPolicy.content); + expect(getPresetContentGatewayStateMock.mock.calls[0]?.[2]).toBe("observability-otlp-local"); + }); + + it("does not let a failed corp-otel replay suppress stale built-in OTLP cleanup", async () => { + const customPolicy = { + name: "corp-otel", + content: + "network_policies:\n observability-otlp-local:\n endpoints:\n - host: collector.corp.example\n", + sourcePath: "/policies/corp-otel.yaml", + }; + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + policies: ["npm", "observability-otlp-local"], + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: ["npm", "observability-otlp-local"], + customPolicies: [customPolicy], + }); + getAppliedPresetsMock.mockReturnValue(["npm", "observability-otlp-local"]); + applyPresetContentMock.mockReturnValue(false); + getPresetContentGatewayStateMock.mockReturnValueOnce("match").mockReturnValueOnce("absent"); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(consoleWarn.mock.calls.flat().join("\n")).toContain("corp-otel (apply failed)"); + expect(removePresetMock).toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(getPresetContentGatewayStateMock).toHaveBeenCalledTimes(2); + expect(getPresetContentGatewayStateMock).toHaveBeenCalledWith( + "alpha", + builtinObservabilityPolicy, + ); + }); + + it("aborts preset reconciliation when custom OTLP ownership is unreadable", async () => { + const currentCustomPolicy = { + name: "corp-otel", + content: "network_policies:\n observability-otlp-local: {}\n", + sourcePath: "/policies/old-collector.yaml", + }; + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: "balanced", + } as never); + getLatestBackupMock.mockReturnValue({ + ...latestBackupFixture, + policyPresets: [], + customPolicies: [], + }); + getCustomPoliciesMock.mockReturnValue([currentCustomPolicy]); + removePresetMock.mockReturnValue(false); + getPresetContentGatewayStateMock.mockImplementation((_sandbox, content) => + content === currentCustomPolicy.content ? null : "absent", + ); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + await runSandboxSnapshot("alpha", { kind: "restore" }); + expect(removePresetMock).toHaveBeenCalledWith("alpha", currentCustomPolicy.name); + expect(applyPresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + expect(consoleWarn.mock.calls.flat().join("\n")).toContain( + "leaving live policy presets unchanged", + ); + }); + it.each([ + "drift", + null, + ] as const)("does not remove built-in OTLP when its exact live content state is %s", async (gatewayState) => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: false, + policyTier: "balanced", + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: ["observability-otlp-local"], + }); + getAppliedPresetsMock.mockReturnValue(["observability-otlp-local"]); + getPresetContentGatewayStateMock.mockReturnValue(gatewayState); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(removePresetMock).not.toHaveBeenCalled(); + expect(consoleWarn.mock.calls.flat().join("\n")).toContain( + "leaving its live policy content unchanged", + ); + }); + + it("normalizes a legacy restricted tier before deciding built-in OTLP egress", async () => { + getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + policyTier: " Restricted ", + } as never); + getLatestBackupMock.mockReturnValue({ + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + policyPresets: [], + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(applyPresetMock).not.toHaveBeenCalledWith("alpha", "observability-otlp-local"); + }); + it("refuses snapshot creation before backup when the sandbox is not live", async () => { parseLiveSandboxNamesMock.mockReturnValue(new Set(["beta"])); const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); @@ -978,7 +1435,7 @@ describe("runSandboxSnapshot", () => { customPolicies: [ { name: "team-egress", - content: "allow team.example", + content: "network_policies:\n team-egress: {}\n", sourcePath: "/policies/team.yaml", }, ], @@ -994,10 +1451,10 @@ describe("runSandboxSnapshot", () => { getCustomPoliciesMock.mockReturnValue([ { name: "team-egress", - content: "allow team.example", + content: "network_policies:\n team-egress: {}\n", sourcePath: "/policies/team.yaml", }, - { name: "old-custom", content: "allow old.example", sourcePath: "/old.yaml" }, + { name: "old-custom", content: "network_policies:\n old: {}\n", sourcePath: "/old.yaml" }, ]); removePresetMock.mockImplementation((_sandbox, preset) => preset !== "old-custom"); const { runSandboxSnapshot } = await import("./snapshot"); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 98faa98288..6e79e4c38e 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -21,6 +21,12 @@ import { withGatewayRouteMutationLock } from "../../inference/gateway-route-muta import * as nim from "../../inference/nim"; import { listMessagingProviderSuffixes } from "../../messaging/channels"; import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + isDcodeAgent, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + OBSERVABILITY_POLICY_BINDING, +} from "../../onboard/observability-policy-presets"; +import { normalizePolicyTierName } from "../../onboard/policy-tier-suppression"; import * as policies from "../../policy"; import { ROOT, run, shellQuote, validateName } from "../../runner"; import { parseLiveSandboxNames } from "../../runtime-recovery"; @@ -224,6 +230,13 @@ async function autoCreateSandboxFromSource( ): Promise { const basePolicy = path.join(ROOT, "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"); const openshellBin = getOpenshellBinary(); + const sourceObservabilityEnabled = + (srcEntry as { observabilityEnabled?: boolean }).observabilityEnabled === true; + const startupCommand = sourceObservabilityEnabled + ? ["env", "NEMOCLAW_OBSERVABILITY=1", "nemoclaw-start"] + : ["nemoclaw-start"]; + const createEnv = { ...process.env }; + delete createEnv.NEMOCLAW_OBSERVABILITY; const cmdParts = [ openshellBin, @@ -237,13 +250,13 @@ async function autoCreateSandboxFromSource( basePolicy, "--auto-providers", "--", - "nemoclaw-start", + ...startupCommand, ].map((p) => shellQuote(p)); const command = `${cmdParts.join(" ")} 2>&1`; console.log(` '${dstName}' does not exist. Creating from '${srcName}' image (${fromImage})...`); - const createResult = await streamSandboxCreate(command, process.env, { + const createResult = await streamSandboxCreate(command, createEnv, { // Use a pre-built image, so skip build+push and jump to pod creation. initialPhase: "create", // Wait until the sandbox actually reaches Ready state, not just appears in the list. @@ -288,6 +301,7 @@ async function autoCreateSandboxFromSource( name: dstName, createdAt: new Date().toISOString(), policies: [], + observabilityEnabled: sourceObservabilityEnabled, // dst has its own lifecycle; don't inherit src's local NIM container // reference, or destroying dst would stop src's NIM. nimContainer: null, @@ -567,17 +581,117 @@ function reconcileSnapshotPolicyPresets( targetSandbox: string, resolvedSnapshot: ReturnType, ): void { - if (!resolvedSnapshot || !Array.isArray(resolvedSnapshot.policyPresets)) return; - const snapshotPresets = resolvedSnapshot.policyPresets; + if (!resolvedSnapshot) return; + const snapshotPolicyPresets = Array.isArray(resolvedSnapshot.policyPresets) + ? resolvedSnapshot.policyPresets + : null; + const hasSnapshotPresetMetadata = snapshotPolicyPresets !== null; + const snapshotCustomPolicies = Array.isArray(resolvedSnapshot.customPolicies) + ? resolvedSnapshot.customPolicies + : []; + const snapshotCustomPolicyNames = new Set( + snapshotCustomPolicies.map((entry) => entry.name.trim().toLowerCase()), + ); + const snapshotPresets = + snapshotPolicyPresets?.filter( + (preset) => !snapshotCustomPolicyNames.has(preset.trim().toLowerCase()), + ) ?? []; + const targetEntry = registry.getSandbox(targetSandbox); + // Custom reconciliation runs before this function. Only the registry state + // that remains after that reconciliation can participate in ownership. + const currentCustomPolicies = registry.getCustomPolicies(targetSandbox); + const currentCustomPolicyNames = new Set( + currentCustomPolicies.map((preset) => preset.name.trim().toLowerCase()), + ); + const customPolicyNames = new Set([...snapshotCustomPolicyNames, ...currentCustomPolicyNames]); + let customOwnsObservability: boolean; + try { + customOwnsObservability = OBSERVABILITY_POLICY_BINDING.hasLiveCustomOwner( + targetSandbox, + currentCustomPolicies.map((entry) => entry.content), + policies, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.warn( + ` Warning: could not verify custom ownership of '${OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET}' (${detail}); leaving live policy presets unchanged.`, + ); + return; + } + const withoutBuiltinObservability = snapshotPresets.filter( + (preset) => !OBSERVABILITY_POLICY_BINDING.matchesPreset(preset), + ); + const shouldEnableBuiltinObservability = + !customOwnsObservability && + isDcodeAgent(targetEntry?.agent) && + targetEntry?.observabilityEnabled === true && + normalizePolicyTierName(targetEntry.policyTier) !== "restricted"; // getAppliedPresets includes custom-policy names for display/CLI parity. // Built-in preset reconciliation must not remove those; custom policy content // is reconciled separately below from registry.getCustomPolicies(). - const customPolicyNames = new Set(registry.getCustomPolicies(targetSandbox).map((p) => p.name)); - const currentPresets = policies - .getAppliedPresets(targetSandbox) - .filter((preset: string) => !customPolicyNames.has(preset)); - const toRemove = currentPresets.filter((p: string) => !snapshotPresets.includes(p)); - const toAdd = snapshotPresets.filter((p: string) => !currentPresets.includes(p)); + const currentPresets = hasSnapshotPresetMetadata + ? [...new Set(policies.getAppliedPresets(targetSandbox))].filter((preset: string) => { + const normalized = preset.trim().toLowerCase(); + return ( + !OBSERVABILITY_POLICY_BINDING.matchesPreset(normalized) && + !customPolicyNames.has(normalized) + ); + }) + : []; + const recordedBuiltinObservability = (targetEntry?.policies ?? []).some((preset) => + OBSERVABILITY_POLICY_BINDING.matchesPreset(preset), + ); + const setRecordedBuiltinObservability = (enabled: boolean, force = false): void => { + const currentEntry = registry.getSandbox(targetSandbox); + if (!currentEntry) return; + const currentPolicies = currentEntry.policies ?? []; + const currentlyRecorded = currentPolicies.some((preset) => + OBSERVABILITY_POLICY_BINDING.matchesPreset(preset), + ); + if (!force && enabled === currentlyRecorded) return; + registry.updateSandbox(targetSandbox, { + policies: OBSERVABILITY_POLICY_BINDING.setAttribution(currentPolicies, enabled), + }); + }; + if (customOwnsObservability) { + setRecordedBuiltinObservability(false); + } + // Legacy snapshots predate generic preset metadata. Leave those unrelated + // presets untouched, while still reconciling the managed observability + // binding below from the target registry's authoritative enablement state. + const toRemove = hasSnapshotPresetMetadata + ? currentPresets.filter((preset: string) => !withoutBuiltinObservability.includes(preset)) + : []; + const toAdd = hasSnapshotPresetMetadata + ? withoutBuiltinObservability.filter((preset: string) => !currentPresets.includes(preset)) + : []; + + // A same-name custom policy does not own the built-in OTLP entry unless its + // exact, overlapping content is both registered after custom reconciliation + // and live in the gateway. Reconcile the built-in from exact content state, + // never from a name/key-only match that could delete drifted operator policy. + let builtinObservabilityContent: string | null = null; + let builtinObservabilityState: "match" | "absent" | "drift" | null = null; + if (!customOwnsObservability) { + const loadedBinding = OBSERVABILITY_POLICY_BINDING.load(targetSandbox, policies); + builtinObservabilityContent = loadedBinding.content; + builtinObservabilityState = loadedBinding.state; + const builtinState = builtinObservabilityState; + if (builtinState === "absent" && shouldEnableBuiltinObservability) { + toAdd.push(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET); + } else if (builtinState === "absent" && recordedBuiltinObservability) { + setRecordedBuiltinObservability(false); + } else if (builtinState === "match" && !shouldEnableBuiltinObservability) { + toRemove.push(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET); + } else if (builtinState === "match" && !recordedBuiltinObservability) { + setRecordedBuiltinObservability(true); + } else if (builtinState === "drift" || builtinState === null) { + const reason = builtinState === "drift" ? "has drifted" : "could not be inspected"; + console.warn( + ` Warning: built-in preset '${OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET}' ${reason}; leaving its live policy content unchanged.`, + ); + } + } if (toRemove.length === 0 && toAdd.length === 0) return; const summary: string[] = []; @@ -587,6 +701,25 @@ function reconcileSnapshotPolicyPresets( const failed: string[] = []; for (const preset of toRemove) { + if (OBSERVABILITY_POLICY_BINDING.matchesPreset(preset) && builtinObservabilityContent) { + const removal = OBSERVABILITY_POLICY_BINDING.removeExact( + targetSandbox, + builtinObservabilityContent, + policies, + { knownBefore: builtinObservabilityState }, + ); + builtinObservabilityState = removal.after; + if (removal.verifiedAbsent) { + setRecordedBuiltinObservability(false); + } else { + // removePreset updates the registry on a reported success. Restore + // attribution whenever exact absence was not proven so recovery does + // not forget built-in policy that may still be live. + setRecordedBuiltinObservability(true, true); + } + if (removal.failureDetail) failed.push(`${preset} (${removal.failureDetail})`); + continue; + } try { if (!policies.removePreset(targetSandbox, preset)) failed.push(`${preset} (remove failed)`); } catch (err) { @@ -885,12 +1018,15 @@ async function runSnapshotRestoreUnlocked( // #5027/#4538: openclaw.json restores via the generic copy strategy, which // lands it at 0640. Repair the mutable config contract when needed. repairRestoredOpenClawConfigPerms(targetSandbox, result); - // Reconcile the target's policy presets to match the snapshot manifest - // exactly. Skip legacy snapshots that predate the `policyPresets` field. - reconcileSnapshotPolicyPresets(targetSandbox, resolvedSnapshot); // Reconcile custom policy presets (applied via --from-file/--from-dir). // Skipped for legacy snapshots that predate the `customPolicies` field. reconcileSnapshotCustomPolicies(targetSandbox, resolvedSnapshot); + // Reconcile built-in presets after custom content so same-name custom + // policies are never transiently substituted with a built-in. The current + // target observability bit and tier override historical built-in OTLP state. + // Legacy snapshots skip unrelated generic presets but still reconcile the + // managed observability binding from current target state. + reconcileSnapshotPolicyPresets(targetSandbox, resolvedSnapshot); }); } diff --git a/src/lib/cli/public-display-defaults.ts b/src/lib/cli/public-display-defaults.ts index 51d00d4bec..57443e5d43 100644 --- a/src/lib/cli/public-display-defaults.ts +++ b/src/lib/cli/public-display-defaults.ts @@ -375,7 +375,7 @@ const PUBLIC_DISPLAY_LAYOUT: Record = { { group: "Sandbox Management", order: 13, - flags: "[--yes|-y|--force] [--verbose|-v]", + flags: "[--yes|-y|--force] [--verbose|-v] [--observability|--no-observability]", }, ], "sandbox:recover": [ diff --git a/src/lib/domain/lifecycle/options.test.ts b/src/lib/domain/lifecycle/options.test.ts index 89a25d9597..da488ac2d0 100644 --- a/src/lib/domain/lifecycle/options.test.ts +++ b/src/lib/domain/lifecycle/options.test.ts @@ -136,6 +136,16 @@ describe("lifecycle option normalization", () => { expect(normalizeRebuildSandboxOptions(["--tool-disclosure=direct"]).toolDisclosure).toBe( "direct", ); + expect(normalizeRebuildSandboxOptions(["--observability"]).observabilityEnabled).toBe(true); + expect(normalizeRebuildSandboxOptions(["--no-observability"]).observabilityEnabled).toBe(false); + expect( + normalizeRebuildSandboxOptions(["--observability", "--no-observability"]) + .observabilityEnabled, + ).toBe(false); + expect( + normalizeRebuildSandboxOptions(["--no-observability", "--observability"]) + .observabilityEnabled, + ).toBe(true); expect(() => normalizeRebuildSandboxOptions(["--tool-disclosure", "sometimes"])).toThrow( /progressive, direct/, ); diff --git a/src/lib/domain/lifecycle/options.ts b/src/lib/domain/lifecycle/options.ts index 7069b69826..db94f243c0 100644 --- a/src/lib/domain/lifecycle/options.ts +++ b/src/lib/domain/lifecycle/options.ts @@ -33,6 +33,7 @@ function readCleanupGatewayEnv(): boolean | undefined { export interface RebuildSandboxOptions { force?: boolean; + observabilityEnabled?: boolean; toolDisclosure?: ToolDisclosure; verbose?: boolean; yes?: boolean; @@ -78,6 +79,12 @@ export function normalizeRebuildSandboxOptions( ): RebuildSandboxOptions { let rawToolDisclosure: unknown; if (Array.isArray(options)) { + const observabilityIndex = options.lastIndexOf("--observability"); + const noObservabilityIndex = options.lastIndexOf("--no-observability"); + const observabilityEnabled = + observabilityIndex === -1 && noObservabilityIndex === -1 + ? undefined + : observabilityIndex > noObservabilityIndex; const splitIndex = options.lastIndexOf("--tool-disclosure"); const inline = [...options].reverse().find((value) => value.startsWith("--tool-disclosure=")); const toolDisclosureFlagProvided = splitIndex >= 0 || inline !== undefined; @@ -89,6 +96,7 @@ export function normalizeRebuildSandboxOptions( } return { force: options.includes("--force"), + ...(observabilityEnabled === undefined ? {} : { observabilityEnabled }), ...(toolDisclosure ? { toolDisclosure } : {}), verbose: options.includes("--verbose") || options.includes("-v"), yes: options.includes("--yes"), diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index db1482ceb9..6f7da9ffc6 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -33,6 +33,9 @@ const setupNimOllama: typeof import("./onboard/setup-nim-ollama") = require("./o const inferenceInputCapability = require("./onboard/inference-input-capability"); const reasoningMode: typeof import("./onboard/reasoning-mode") = require("./onboard/reasoning-mode"); const toolDisclosureFlow: typeof import("./onboard/tool-disclosure-flow") = require("./onboard/tool-disclosure-flow"); +const runtimeControlFlow: typeof import("./onboard/runtime-control-flow") = require("./onboard/runtime-control-flow"); +const observabilityPolicy: typeof import("./onboard/observability-policy-presets") = require("./onboard/observability-policy-presets"); +const observabilityCommandFlag: typeof import("./onboard/observability-command-flag") = require("./onboard/observability-command-flag"); const inferenceRouteHelpers: typeof import("./onboard/inference-route") = require("./onboard/inference-route"); const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); const { @@ -136,15 +139,9 @@ const { getRegistrySandboxMessagingPlan, MessagingHostStateApplier, } = require("./onboard/messaging-channel-setup") as typeof import("./onboard/messaging-channel-setup"); -const { - clearAgentScopedResumeState, -}: typeof import("./onboard/agent-resume-state") = require("./onboard/agent-resume-state"); const { repairResumeMachineSnapshot, }: typeof import("./onboard/resume-machine-repair") = require("./onboard/resume-machine-repair"); -const { - stopTrackedModelRouterForAgentChange, -}: typeof import("./onboard/model-router-process") = require("./onboard/model-router-process"); const bedrockRuntimeOnboard: typeof import("./onboard/bedrock-runtime") = require("./onboard/bedrock-runtime"); const { @@ -2392,6 +2389,8 @@ async function createSandboxWithBaseImageResolution( ); process.exit(1); } + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const observabilityDrift = observabilityPolicy.hasRegisteredDcodeObservabilityDrift(liveExists, isManagedDcodeAgent, existingEntry, createIntent?.observabilityEnabled); // #4614: capture default AFTER prune so a stale registry row isn't read as a live sandbox. const sandboxWasLiveDefault = liveExists && wasSandboxDefault(registry.getDefault(), sandboxName); @@ -2485,7 +2484,8 @@ async function createSandboxWithBaseImageResolution( !credentialRotation.changed && !hermesToolGatewayDrift && !hermesDashboardDrift && - !toolDisclosureMigrationNeeded + !toolDisclosureMigrationNeeded && + !observabilityDrift ) { // Guard against reusing a CPU-only sandbox when GPU passthrough is enabled. // Placed before the non-interactive / interactive split so all reuse @@ -2638,6 +2638,8 @@ async function createSandboxWithBaseImageResolution( note(` Sandbox '${sandboxName}' exists — recreating to apply Hermes managed-tool changes.`); } else if (hermesDashboardDrift) { note(` Sandbox '${sandboxName}' exists — recreating to apply Hermes dashboard settings.`); + } else if (observabilityDrift) { + note(` Sandbox '${sandboxName}' exists — recreating to apply observability settings.`); } else if (toolDisclosureMigrationNote) { note(toolDisclosureMigrationNote); } else if (credentialRotation.changed) { @@ -2649,11 +2651,13 @@ async function createSandboxWithBaseImageResolution( } if (preservedMcpState) { + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const explicitObservability = observabilityCommandFlag.explicitObservabilityFlag(createIntent?.observabilityEnabled === true, createIntent?.observabilityRequestedExplicitly === true); console.error( ` Sandbox '${sandboxName}' has managed MCP servers. Refusing the generic onboard recreation path.`, ); console.error( - ` Run \`${cliName()} ${sandboxName} rebuild --yes --tool-disclosure ${effectiveToolDisclosure}\` so MCP providers and adapter state are preserved transactionally.`, + ` Run \`${cliName()} ${sandboxName} rebuild --yes --tool-disclosure ${effectiveToolDisclosure}${explicitObservability ? ` ${explicitObservability}` : ""}\` so MCP providers and adapter state are preserved transactionally.`, ); process.exit(1); } @@ -2735,6 +2739,7 @@ async function createSandboxWithBaseImageResolution( const { activeMessagingChannels, initialSandboxPolicy, + policyTier: resolvedCreatePolicyTier, createArgs, messagingProviders, useDockerGpuPatch, @@ -2771,6 +2776,7 @@ async function createSandboxWithBaseImageResolution( getHermesToolGatewayProviderName: (targetSandbox) => getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), agentName: agent?.name, + policyTier: createIntent?.policyTier, }); if (initialSandboxPolicy.cleanup) { process.on("exit", initialSandboxPolicy.cleanup); @@ -2810,6 +2816,7 @@ async function createSandboxWithBaseImageResolution( const { createCommand, effectiveDashboardPort, prebuild, sandboxEnv, sandboxStartupCommand } = await sandboxCreateLaunch.prepareSandboxCreateLaunchWithPrebuild({ agent, + observabilityEnabled: createIntent?.observabilityEnabled === true, chatUiUrl, createArgs, sandboxName, @@ -3008,6 +3015,8 @@ async function createSandboxWithBaseImageResolution( imageTag: resolvedImageTag, appliedPolicies: initialSandboxPolicy.appliedPresets, toolDisclosure: effectiveToolDisclosure, + observabilityEnabled: createIntent?.observabilityEnabled === true, + policyTier: resolvedCreatePolicyTier, // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. ...sandboxRegistration.creationFidelity(webSearchConfig, fromDockerfile, normalizeHermesAuthMethod(hermesAuthMethod)), plannedMessagingState, @@ -4062,9 +4071,7 @@ async function preflightAuthoritativeRebuildTarget( const onboard = onboardEntryOptions.withNonInteractiveEnvironment(runOnboard); async function runOnboard(opts: OnboardOptions = {}): Promise { setupInferenceFactory.assertNoOpenShellGatewayEndpointOverride(); - const requestedToolDisclosure = toolDisclosureFlow.applyOnboardToolDisclosureRequest( - opts.toolDisclosure, - ); + const runtimeControlRequests = runtimeControlFlow.applyOnboardRuntimeControlRequests(opts); const authoritativeGateway = authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); const previousGatewayBinding = { name: GATEWAY_NAME, port: GATEWAY_PORT }; @@ -4209,7 +4216,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { authoritativeResumeConfig: opts.authoritativeResumeConfig === true, agentFlag: opts.agent || null, envAgent: process.env.NEMOCLAW_AGENT || null, - requestedToolDisclosure, + ...runtimeControlRequests, }, { loadSession: onboardSession.loadSession, @@ -4259,29 +4266,16 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { resume, canPrompt: !cannotPrompt, }); - const selectedAgentName = normalizeSandboxAgentName(agent?.name); - const recordedAgentName = normalizeSandboxAgentName(session?.agent); - let resumeAgentChanged = false; - let forceProviderSelectionForAgentChange = false; - if (resume && session && recordedAgentName !== selectedAgentName) { - resumeAgentChanged = true; - forceProviderSelectionForAgentChange = true; - note( - ` Agent changed from ${formatSandboxAgentName(recordedAgentName)} to ${formatSandboxAgentName(selectedAgentName)}; refreshing provider selection.`, - ); - await stopTrackedModelRouterForAgentChange( - session, - loadBlueprintProfile("routed")?.router.port || 4000, - ); - onboardSession.updateSession((current: Session) => - clearAgentScopedResumeState(current, selectedAgentName), - ); - } - setOnboardBrandingAgent(agent?.name || "openclaw"); - session = onboardSession.updateSession((s: Session) => { - s.agent = agent?.name ?? null; - return s; + const selectedAgentTransition = await runtimeControlFlow.applySelectedAgentTransition({ + resume, + session, + selectedAgentName: agent?.name, + routerPort: loadBlueprintProfile("routed")?.router.port || 4000, + note, }); + session = selectedAgentTransition.session; + const resumeAgentChanged = selectedAgentTransition.resumeAgentChanged; + const forceProviderSelectionForAgentChange = resumeAgentChanged; const recordedSandboxName = session?.steps?.sandbox?.status === "complete" ? session?.sandboxName || null : null; @@ -4517,6 +4511,9 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, sandbox: { resumeAgentChanged, + requestedObservabilityEnabled: runtimeControlRequests.requestedObservabilityEnabled, + authoritativePolicyTier: + opts.authoritativeResumeConfig === true ? (opts.policyTier ?? null) : null, controlUiPort: opts.controlUiPort || null, rootDir: ROOT, }, @@ -4621,6 +4618,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { import("./verify-deployment").VerifyDeploymentResult >({ branchState: agent ? "agent_setup" : "openclaw", + authoritativePolicyTier: + opts.authoritativeResumeConfig === true ? (opts.policyTier ?? null) : null, agentSetupDeps: { handleAgentSetup: agentOnboard.handleAgentSetup, agentSetupContext: () => ({ @@ -4814,7 +4813,7 @@ module.exports = { hasStaleGateway, getRequestedSandboxNameHint, getResumeSandboxConflict, - clearAgentScopedResumeState, + clearAgentScopedResumeState: runtimeControlFlow.clearAgentScopedResumeState, getSandboxReuseState, getSandboxStateFromOutputs, getPortConflictServiceHints, diff --git a/src/lib/onboard/agent-policy-presets.ts b/src/lib/onboard/agent-policy-presets.ts index 4e579a6f1d..a5be5335fb 100644 --- a/src/lib/onboard/agent-policy-presets.ts +++ b/src/lib/onboard/agent-policy-presets.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { HERMES_TOOL_GATEWAY_PRESET_NAMES } from "./hermes-managed-tools"; +import { DCODE_ONLY_POLICY_PRESETS, isDcodeAgent } from "./observability-policy-presets"; import { isOpenclawAgent, OPENCLAW_ONLY_POLICY_PRESETS } from "./openclaw-otel-policy-presets"; export { OPENCLAW_ONLY_POLICY_PRESETS }; @@ -16,6 +17,7 @@ export function setupPolicyPresetAppliesToAgent( ): boolean { const name = presetName.trim().toLowerCase(); if (HERMES_TOOL_GATEWAY_PRESET_NAMES.has(name)) return isHermesAgent(agent); + if (DCODE_ONLY_POLICY_PRESETS.has(name)) return isDcodeAgent(agent); if (OPENCLAW_ONLY_POLICY_PRESETS.has(name)) return isOpenclawAgent(agent); return true; } diff --git a/src/lib/onboard/command-support.test.ts b/src/lib/onboard/command-support.test.ts index 16589f9f1b..cb5251c4e2 100644 --- a/src/lib/onboard/command-support.test.ts +++ b/src/lib/onboard/command-support.test.ts @@ -30,3 +30,14 @@ describe("buildOnboardFlags --agent help (#5779)", () => { expect(flags.agent.description).toBe("Agent runtime to onboard"); }); }); + +describe("buildOnboardFlags --observability help", () => { + it("discloses the bounded content exported by the opt-in", () => { + const flags = buildOnboardFlags(); + + expect(flags.observability.description).toBe( + "Export bounded prompt, response, tool argument, and tool result content to a local OTLP collector (Deep Agents Code only)", + ); + expect(flags.observability.allowNo).toBe(true); + }); +}); diff --git a/src/lib/onboard/command-support.ts b/src/lib/onboard/command-support.ts index f87b3845fe..f83e19726f 100644 --- a/src/lib/onboard/command-support.ts +++ b/src/lib/onboard/command-support.ts @@ -46,7 +46,7 @@ function agentFlagDescription(): string { } export const onboardUsage = [ - `onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--control-ui-port ] [--yes | -y] [--no-ollama-autostart] [${NOTICE_ACCEPT_FLAG}]`, + `onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--observability | --no-observability] [--control-ui-port ] [--yes | -y] [--no-ollama-autostart] [${NOTICE_ACCEPT_FLAG}]`, ]; export const onboardExamples = [ @@ -75,6 +75,7 @@ export type OnboardFlags = { agent?: string; agents?: string; "tool-disclosure"?: ToolDisclosure; + observability?: boolean; "control-ui-port"?: number; yes?: boolean; "no-ollama-autostart"?: boolean; @@ -127,6 +128,11 @@ export function buildOnboardFlags(): Record { "Choose progressive tool discovery or direct exposure of all session-authorized tools", options: [...TOOL_DISCLOSURE_VALUES], }), + observability: Flags.boolean({ + allowNo: true, + description: + "Export bounded prompt, response, tool argument, and tool result content to a local OTLP collector (Deep Agents Code only)", + }), "control-ui-port": Flags.integer({ description: "Host port for the local control UI", max: 65535, diff --git a/src/lib/onboard/command.test.ts b/src/lib/onboard/command.test.ts index 158b14fa45..8a70799b88 100644 --- a/src/lib/onboard/command.test.ts +++ b/src/lib/onboard/command.test.ts @@ -44,6 +44,7 @@ describe("onboard command options", () => { "sandbox-gpu-device": "nvidia.com/gpu=0", agent: "dcode", "tool-disclosure": "direct", + observability: true, "control-ui-port": 18790, gpu: true, yes: true, @@ -65,6 +66,7 @@ describe("onboard command options", () => { agent: "langchain-deepagents-code", agentsManifest: null, toolDisclosure: "direct", + observabilityEnabled: true, controlUiPort: 18790, gpu: true, noGpu: false, @@ -87,6 +89,7 @@ describe("onboard command options", () => { agent: null, agentsManifest: null, toolDisclosure: null, + observabilityEnabled: null, controlUiPort: null, gpu: false, noGpu: false, @@ -95,6 +98,15 @@ describe("onboard command options", () => { }); }); + it("maps --no-observability to an explicit disabled request", () => { + expect( + resolve( + { agent: "dcode", observability: false }, + { listAgents: () => ["openclaw", "hermes", "langchain-deepagents-code"] }, + ).observabilityEnabled, + ).toBe(false); + }); + it("accepts the environment-based third-party notice acknowledgement", () => { expect( resolve({}, { env: { NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" } }).acceptThirdPartySoftware, @@ -146,6 +158,31 @@ describe("onboard command options", () => { expect(resolve({ agent: "nemohermes" }, { listAgents }).agent).toBe("hermes"); }); + it("rejects observability for an explicitly unsupported agent", () => { + const errors: string[] = []; + expect(() => + resolve( + { agent: "hermes", observability: true }, + { + listAgents: () => ["openclaw", "hermes", "langchain-deepagents-code"], + error: (message = "") => errors.push(message), + }, + ), + ).toThrow("exit:1"); + expect(errors.join("\n")).toContain( + "--observability is supported only with --agent langchain-deepagents-code", + ); + }); + + it("allows an explicit observability opt-out while selecting another agent", () => { + expect( + resolve( + { agent: "hermes", observability: false }, + { listAgents: () => ["openclaw", "hermes", "langchain-deepagents-code"] }, + ).observabilityEnabled, + ).toBe(false); + }); + it("rejects unknown agents with the available aliases", () => { const errors: string[] = []; expect(() => diff --git a/src/lib/onboard/command.ts b/src/lib/onboard/command.ts index 230ad7fee6..96dcce6734 100644 --- a/src/lib/onboard/command.ts +++ b/src/lib/onboard/command.ts @@ -12,6 +12,8 @@ import { } from "../tool-disclosure"; import { applyAgentsManifestEnv } from "./agents-manifest"; import type { OnboardFlags } from "./command-support"; +import { managedSandboxFeatureIssue } from "./managed-sandbox-feature"; +import { DCODE_OBSERVABILITY_FEATURE } from "./observability-policy-presets"; import { isOpenclawAgent } from "./openclaw-otel-policy-presets"; import { NOTICE_ACCEPT_ENV, NOTICE_ACCEPT_FLAG_NAME } from "./usage-notice"; @@ -28,6 +30,7 @@ export interface OnboardCommandOptions { agent: string | null; agentsManifest: string | null; toolDisclosure: ToolDisclosure | null; + observabilityEnabled: boolean | null; controlUiPort: number | null; gpu: boolean; noGpu: boolean; @@ -128,11 +131,26 @@ function resolveSandboxGpu(flags: OnboardFlags): "enable" | "disable" | null { return null; } +function validateObservabilityAgent( + requested: boolean | undefined, + agent: string | null, + deps: ResolveOnboardOptionsDeps, +): void { + if ( + agent && + managedSandboxFeatureIssue(DCODE_OBSERVABILITY_FEATURE, { agent, requested }) === + "unsupported-request" + ) { + fail(deps, " --observability is supported only with --agent langchain-deepagents-code."); + } +} + export function resolveOnboardOptions( flags: OnboardFlags, deps: ResolveOnboardOptionsDeps, ): OnboardCommandOptions { const agent = resolveAgent(flags.agent, deps); + validateObservabilityAgent(flags.observability, agent, deps); let toolDisclosure: ToolDisclosure | null; try { toolDisclosure = resolveToolDisclosureRequest(flags["tool-disclosure"], deps.env); @@ -153,6 +171,7 @@ export function resolveOnboardOptions( agent, agentsManifest: resolveAgentsManifest(flags.agents, agent, deps), toolDisclosure, + observabilityEnabled: typeof flags.observability === "boolean" ? flags.observability : null, controlUiPort: flags["control-ui-port"] ?? null, gpu: flags.gpu === true, noGpu: flags["no-gpu"] === true, diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 7b4b9b582c..a97edef2ca 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -34,6 +34,8 @@ export interface CoreOnboardFlowPhaseOptions< providerDeps: ProviderInferenceStateOptions["deps"]; sandbox: { resumeAgentChanged: boolean; + requestedObservabilityEnabled?: boolean | null; + authoritativePolicyTier?: string | null; controlUiPort: number | null; rootDir: string; }; @@ -110,7 +112,9 @@ export function createCoreOnboardFlowPhases< fresh: context.fresh, gatewayName: options.gatewayName, authoritativeResumeConfig: options.authoritativeResumeConfig, + authoritativePolicyTier: options.sandbox.authoritativePolicyTier, resumeAgentChanged: options.sandbox.resumeAgentChanged, + requestedObservabilityEnabled: options.sandbox.requestedObservabilityEnabled, session: context.session, sandboxName: context.sandboxName, model: context.model, diff --git a/src/lib/onboard/machine/final-flow-phases.ts b/src/lib/onboard/machine/final-flow-phases.ts index fdd9749031..85e91a96b6 100644 --- a/src/lib/onboard/machine/final-flow-phases.ts +++ b/src/lib/onboard/machine/final-flow-phases.ts @@ -24,6 +24,7 @@ export interface FinalOnboardFlowPhaseOptions< VerificationResult = unknown, > { branchState: "agent_setup" | "openclaw"; + authoritativePolicyTier?: string | null; agentSetupDeps: AgentSetupStateOptions["deps"]; policiesDeps: PoliciesStateOptions["deps"]; finalization: { @@ -70,6 +71,7 @@ export function createFinalOnboardFlowPhases< assertSandboxCreatedContext(context, "policies"); const policiesResult = await handlePoliciesState({ resume: context.resume, + authoritativePolicyTier: options.authoritativePolicyTier, sandboxName: context.sandboxName, provider: context.provider, model: context.model, diff --git a/src/lib/onboard/machine/handlers/policies-observability.test.ts b/src/lib/onboard/machine/handlers/policies-observability.test.ts new file mode 100644 index 0000000000..3837cb2e45 --- /dev/null +++ b/src/lib/onboard/machine/handlers/policies-observability.test.ts @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createSession, type SessionUpdates } from "../../../state/onboard-session"; +import { handlePoliciesState, type PoliciesStateOptions } from "./policies"; + +type Agent = { name: string }; + +describe("handlePoliciesState observability", () => { + it("threads durable observability intent into policy reconciliation", async () => { + const session = createSession({ observabilityEnabled: true }); + const prepareResume = vi.fn(() => ({ + policyPresets: [], + recordedPolicyPresetsNeedReconcile: false, + disabledMessagingPolicyPresetApplied: false, + suppressedAgentRequiredPresetsLive: false, + })); + const setupPolicies = vi.fn(async () => []); + const deps = { + loadSession: () => session, + getActiveSandbox: () => null, + mergePolicyMessagingChannels: () => [], + verifyCompatibleEndpointSandboxSmoke: vi.fn(), + preparePolicyPresetResumeSelection: prepareResume, + arePolicyPresetsApplied: () => false, + skippedStepMessage: vi.fn(), + recordStateSkipped: vi.fn(async () => session), + startRecordedStep: vi.fn(async () => undefined), + setupPoliciesWithSelection: setupPolicies, + updateSession: () => session, + recordStepComplete: vi.fn(async () => session), + toSessionUpdates: (updates: Record) => updates as SessionUpdates, + persistAppliedPolicyPresets: vi.fn(), + } satisfies PoliciesStateOptions["deps"]; + + await handlePoliciesState({ + resume: false, + sandboxName: "my-assistant", + provider: "provider", + model: "model", + endpointUrl: "https://example.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + selectedMessagingChannels: [], + webSearchConfig: null, + webSearchSupported: true, + hermesToolGateways: [], + agent: { name: "langchain-deepagents-code" }, + deps, + }); + + expect(prepareResume).toHaveBeenCalledWith( + "my-assistant", + expect.objectContaining({ observabilityEnabled: true }), + ); + expect(setupPolicies).toHaveBeenCalledWith( + "my-assistant", + expect.objectContaining({ observabilityEnabled: true }), + ); + }); + + it("keeps an authoritative rebuild tier through resume preparation and policy setup", async () => { + const session = createSession({ + observabilityEnabled: true, + policyPresets: ["observability-otlp-local"], + }); + const prepareResume = vi.fn(() => ({ + policyPresets: [], + recordedPolicyPresetsNeedReconcile: true, + disabledMessagingPolicyPresetApplied: false, + suppressedAgentRequiredPresetsLive: false, + })); + const setupPolicies = vi.fn(async () => []); + const deps = { + loadSession: () => session, + getActiveSandbox: () => ({ policyTier: null }), + mergePolicyMessagingChannels: () => [], + verifyCompatibleEndpointSandboxSmoke: vi.fn(), + preparePolicyPresetResumeSelection: prepareResume, + arePolicyPresetsApplied: () => false, + skippedStepMessage: vi.fn(), + recordStateSkipped: vi.fn(async () => session), + startRecordedStep: vi.fn(async () => undefined), + setupPoliciesWithSelection: setupPolicies, + updateSession: () => session, + recordStepComplete: vi.fn(async () => session), + toSessionUpdates: (updates: Record) => updates as SessionUpdates, + persistAppliedPolicyPresets: vi.fn(), + } satisfies PoliciesStateOptions["deps"]; + + await handlePoliciesState({ + resume: true, + authoritativePolicyTier: "restricted", + sandboxName: "my-assistant", + provider: "provider", + model: "model", + endpointUrl: "https://example.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + selectedMessagingChannels: [], + webSearchConfig: null, + webSearchSupported: true, + hermesToolGateways: [], + agent: { name: "langchain-deepagents-code" }, + deps, + }); + + expect(prepareResume).toHaveBeenCalledWith( + "my-assistant", + expect.objectContaining({ tierName: "restricted" }), + ); + expect(setupPolicies).toHaveBeenCalledWith( + "my-assistant", + expect.objectContaining({ tierName: "restricted", selectedPresets: [] }), + ); + }); +}); diff --git a/src/lib/onboard/machine/handlers/policies.ts b/src/lib/onboard/machine/handlers/policies.ts index 4ff825b2f6..84f67c2ca8 100644 --- a/src/lib/onboard/machine/handlers/policies.ts +++ b/src/lib/onboard/machine/handlers/policies.ts @@ -36,6 +36,8 @@ export interface PolicyResumeSelection { export interface PoliciesStateOptions { resume: boolean; + /** Internal rebuild tier that takes precedence over a not-yet-complete registry row. */ + authoritativePolicyTier?: string | null; sandboxName: string; provider: string; model: string; @@ -73,6 +75,7 @@ export interface PoliciesStateOptions { enabledChannels: string[]; hermesToolGateways: string[]; agent?: string | null; + observabilityEnabled?: boolean | null; webSearchConfig: WebSearchConfig | null; webSearchConfigChanged: boolean; webSearchSupported: boolean; @@ -98,6 +101,8 @@ export interface PoliciesStateOptions { webSearchConfig: WebSearchConfig | null; provider: string; agent?: string | null; + observabilityEnabled?: boolean | null; + tierName?: string | null; webSearchSupported: boolean; hermesToolGateways: string[]; onSelection: (policyPresets: string[]) => void; @@ -126,6 +131,7 @@ export interface PoliciesStateResult { export async function handlePoliciesState({ resume, + authoritativePolicyTier, sandboxName, provider, model, @@ -140,11 +146,13 @@ export async function handlePoliciesState({ deps, }: PoliciesStateOptions): Promise { const latestSession = deps.loadSession(); + const observabilityEnabled = latestSession?.observabilityEnabled === true; const recordedPolicyPresets = Array.isArray(latestSession?.policyPresets) ? latestSession.policyPresets : null; const recordedMessagingChannels = getActiveChannelsFromPlan(latestSession?.messagingPlan); const activeSandbox = deps.getActiveSandbox(sandboxName); + const effectivePolicyTier = authoritativePolicyTier ?? activeSandbox?.policyTier ?? null; const activePlan = activeSandbox?.messaging?.plan; const activeMessagingChannels = getActiveChannelsFromPlan(activePlan); const disabledChannels = getDisabledChannelsFromPlan(activePlan); @@ -170,10 +178,11 @@ export async function handlePoliciesState({ enabledChannels: policyMessagingChannels, hermesToolGateways, agent: normalizeAgentName((agent as { name?: string } | null)?.name), + observabilityEnabled, webSearchConfig, webSearchConfigChanged, webSearchSupported, - tierName: activeSandbox?.policyTier ?? null, + tierName: effectivePolicyTier, }); const recordedPolicyPresetsForSupport = policyResumeSelection.policyPresets; const resumePolicies = @@ -232,6 +241,8 @@ export async function handlePoliciesState({ // to "openclaw" so the auto-suggest gate still fires; explicit // Hermes runs keep their own name. agent: normalizeAgentName((agent as { name?: string } | null)?.name), + observabilityEnabled, + tierName: effectivePolicyTier, webSearchSupported, hermesToolGateways, onSelection: (policyPresets) => { diff --git a/src/lib/onboard/machine/handlers/sandbox-dcode-selection.test.ts b/src/lib/onboard/machine/handlers/sandbox-dcode-selection.test.ts index 285afcd467..9008ea4c47 100644 --- a/src/lib/onboard/machine/handlers/sandbox-dcode-selection.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-dcode-selection.test.ts @@ -29,6 +29,7 @@ function dcodeRegistryEntry( name, agent: "langchain-deepagents-code", nemoclawVersion: "0.1.0", + observabilityEnabled: false, toolDisclosure: "progressive", webSearchEnabled: false, webSearchProvider: null, @@ -70,6 +71,7 @@ describe("handleSandboxState live DCode selection", () => { expect(calls.createSandbox.mock.calls[0]?.at(-1)).toEqual({ recreate: true, toolDisclosure: "progressive", + observabilityEnabled: false, }); expect(calls.removeSandbox).not.toHaveBeenCalled(); }); @@ -88,6 +90,7 @@ describe("handleSandboxState live DCode selection", () => { expect(calls.createSandbox.mock.calls[0]?.at(-1)).toEqual({ recreate: true, toolDisclosure: "progressive", + observabilityEnabled: false, }); }); diff --git a/src/lib/onboard/machine/handlers/sandbox-resume.test.ts b/src/lib/onboard/machine/handlers/sandbox-resume.test.ts index ea230601e5..37c857934c 100644 --- a/src/lib/onboard/machine/handlers/sandbox-resume.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-resume.test.ts @@ -38,6 +38,7 @@ describe("decideSandboxResume", () => { ["sandbox GPU", { sandboxGpuConfigChanged: true }, true], ["messaging", { messagingChannelConfigChanged: true }, true], ["Hermes tool gateway", { hermesToolGatewayConfigChanged: true }, true], + ["observability", { observabilityChanged: true }, false], ["tool disclosure migration", { toolDisclosureMigrationNeeded: true }, false], ["tool disclosure", { toolDisclosureChanged: true }, false], ["live DCode inference selection", { inferenceSelectionChanged: true }, false], diff --git a/src/lib/onboard/machine/handlers/sandbox-resume.ts b/src/lib/onboard/machine/handlers/sandbox-resume.ts index 375050890a..7ea64ce9e6 100644 --- a/src/lib/onboard/machine/handlers/sandbox-resume.ts +++ b/src/lib/onboard/machine/handlers/sandbox-resume.ts @@ -15,6 +15,7 @@ export interface SandboxResumeSignals { readonly sandboxGpuConfigChanged: boolean; readonly messagingChannelConfigChanged: boolean; readonly hermesToolGatewayConfigChanged: boolean; + readonly observabilityChanged?: boolean; readonly toolDisclosureMigrationNeeded: boolean; readonly toolDisclosureChanged: boolean; readonly inferenceSelectionChanged: boolean; @@ -105,6 +106,7 @@ function canReuseSandbox(signals: SandboxResumeSignals): boolean { !signals.sandboxGpuConfigChanged && !signals.messagingChannelConfigChanged && !signals.hermesToolGatewayConfigChanged && + !signals.observabilityChanged && !signals.toolDisclosureMigrationNeeded && !signals.toolDisclosureChanged && signals.sandboxReuseState === "ready" @@ -159,11 +161,9 @@ function compatibilityResumeDecision(signals: SandboxResumeSignals): SandboxResu return null; } -export function decideSandboxResume(signals: SandboxResumeSignals): SandboxResumeDecision { - if (!signals.resume || !signals.sandboxStepComplete) return { kind: "create" }; - const compatibilityDecision = compatibilityResumeDecision(signals); - if (compatibilityDecision) return compatibilityDecision; - if (canReuseSandbox(signals)) return { kind: "reuse" }; +function runtimeConfigurationResumeDecision( + signals: SandboxResumeSignals, +): SandboxResumeDecision | null { if (signals.webSearchConfigChanged) { return { kind: "recreate", @@ -192,6 +192,24 @@ export function decideSandboxResume(signals: SandboxResumeSignals): SandboxResum removeRegistryEntry: true, }; } + if (signals.observabilityChanged) { + return { + kind: "recreate", + note: " [resume] Observability configuration changed; recreating sandbox.", + // Preserve the row until createSandbox captures registry-only fidelity. + removeRegistryEntry: false, + }; + } + return null; +} + +export function decideSandboxResume(signals: SandboxResumeSignals): SandboxResumeDecision { + if (!signals.resume || !signals.sandboxStepComplete) return { kind: "create" }; + const compatibilityDecision = compatibilityResumeDecision(signals); + if (compatibilityDecision) return compatibilityDecision; + if (canReuseSandbox(signals)) return { kind: "reuse" }; + const configurationDecision = runtimeConfigurationResumeDecision(signals); + if (configurationDecision) return configurationDecision; const toolDisclosureDecision = toolDisclosureResumeDecision(signals); if (toolDisclosureDecision) return toolDisclosureDecision; if (signals.sandboxReuseState === "not_ready") return { kind: "repair-and-recreate" }; diff --git a/src/lib/onboard/machine/handlers/sandbox-tool-disclosure.test.ts b/src/lib/onboard/machine/handlers/sandbox-tool-disclosure.test.ts index 9c702635c1..86146839a9 100644 --- a/src/lib/onboard/machine/handlers/sandbox-tool-disclosure.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-tool-disclosure.test.ts @@ -120,7 +120,7 @@ describe("handleSandboxState tool disclosure", () => { null, [], null, - { recreate: true, toolDisclosure: requestedMode }, + { recreate: true, toolDisclosure: requestedMode, observabilityEnabled: false }, ); }); diff --git a/src/lib/onboard/machine/handlers/sandbox.test.ts b/src/lib/onboard/machine/handlers/sandbox.test.ts index 560978b1d6..7d6f227a11 100644 --- a/src/lib/onboard/machine/handlers/sandbox.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox.test.ts @@ -21,6 +21,21 @@ vi.mock("../../messaging-channel-setup", () => ({ const detectMessagingChannelsFromEnvMock = vi.mocked(detectMessagingChannelsFromEnv); +function dcodeRegistryEntry(name: string, observabilityEnabled?: boolean) { + return { + name, + agent: "langchain-deepagents-code", + provider: "provider", + model: "model", + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: "openai-completions", + gatewayName: "nemoclaw", + toolDisclosure: "progressive" as const, + ...(typeof observabilityEnabled === "boolean" ? { observabilityEnabled } : {}), + }; +} + describe("handleSandboxState", () => { beforeEach(() => { detectMessagingChannelsFromEnvMock.mockReturnValue([]); @@ -55,7 +70,7 @@ describe("handleSandboxState", () => { null, [], null, - { recreate: false, toolDisclosure: "progressive" }, + { recreate: false, toolDisclosure: "progressive", observabilityEnabled: false }, ); expect(calls.updateSandbox).toHaveBeenCalledWith( "my-assistant", @@ -97,6 +112,318 @@ describe("handleSandboxState", () => { expect(result.webSearchConfig).toBeNull(); }); + it("carries durable observability intent in the sandbox create intent", async () => { + const session = createSession({ + observabilityEnabled: true, + observabilityRequestedExplicitly: true, + }); + const { deps, calls } = createDeps({ + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toEqual({ + recreate: false, + toolDisclosure: "progressive", + observabilityEnabled: true, + observabilityRequestedExplicitly: true, + }); + }); + + it("carries an authoritative rebuild tier in the sandbox create intent", async () => { + const { deps, calls } = createDeps(); + + await handleSandboxState({ + ...baseOptions(deps), + agent: { name: "langchain-deepagents-code" }, + authoritativeResumeConfig: true, + authoritativePolicyTier: "restricted", + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + policyTier: "restricted", + }); + }); + + it("rejects observability for a selected non-DCode agent", async () => { + const { deps, calls } = createDeps(); + + await expect( + handleSandboxState({ + ...baseOptions(deps), + agent: { name: "hermes" }, + requestedObservabilityEnabled: true, + }), + ).rejects.toThrow("exit 1"); + + expect(calls.error).toHaveBeenCalledWith( + " --observability is supported only with --agent langchain-deepagents-code.", + ); + expect(calls.createSandbox).not.toHaveBeenCalled(); + }); + + it("preserves recorded observability when a new onboard run omits the flag", async () => { + const session = createSession({ observabilityEnabled: false }); + const { deps, calls } = createDeps({ + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, true), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + sandboxName: "saved", + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + observabilityEnabled: true, + }); + expect(session.observabilityEnabled).toBe(true); + expect(session.observabilityRequestedExplicitly).toBe(false); + }); + + it.each([ + "openclaw", + "hermes", + ])("requires an explicit observability disable when switching DCode to %s", async (agentName) => { + const session = createSession({ + agent: "langchain-deepagents-code", + observabilityEnabled: true, + }); + const { deps, calls } = createDeps({ + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, true), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await expect( + handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: agentName }, + sandboxName: "saved", + }), + ).rejects.toThrow("exit 1"); + + expect(calls.error).toHaveBeenCalledWith(expect.stringContaining("--no-observability")); + expect(calls.createSandbox).not.toHaveBeenCalled(); + expect(session.observabilityEnabled).toBe(true); + }); + + it("requires an explicit disable when resumed session state has observability enabled", async () => { + const session = createSession({ + agent: "langchain-deepagents-code", + observabilityEnabled: true, + observabilityRequestedExplicitly: true, + }); + const { deps, calls } = createDeps({ + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, false), + }); + + await expect( + handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "hermes" }, + resume: true, + sandboxName: "saved", + }), + ).rejects.toThrow("exit 1"); + + expect(calls.error).toHaveBeenCalledWith(expect.stringContaining("--no-observability")); + expect(calls.createSandbox).not.toHaveBeenCalled(); + }); + + it("clears DCode observability during an explicitly acknowledged agent switch", async () => { + const session = createSession({ + agent: "langchain-deepagents-code", + observabilityEnabled: true, + }); + const { deps, calls } = createDeps({ + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, true), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "hermes" }, + sandboxName: "saved", + requestedObservabilityEnabled: false, + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + observabilityEnabled: false, + observabilityRequestedExplicitly: true, + }); + expect(session.observabilityEnabled).toBe(false); + expect(session.observabilityRequestedExplicitly).toBe(true); + }); + + it("records an explicit request even when its enabled value already matches", async () => { + const session = createSession({ observabilityEnabled: true }); + const updateSession = vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }); + const { deps } = createDeps({ + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, true), + updateSession, + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + sandboxName: "saved", + requestedObservabilityEnabled: true, + }); + + expect(session.observabilityEnabled).toBe(true); + expect(session.observabilityRequestedExplicitly).toBe(true); + expect(updateSession).toHaveBeenCalled(); + }); + + it.each([ + { recorded: true, requested: false }, + { recorded: false, requested: true }, + ])("gives current explicit observability=$requested precedence on resume", async ({ + recorded, + requested, + }) => { + const session = createSession({ + sandboxName: "saved", + observabilityEnabled: recorded, + observabilityRequestedExplicitly: true, + }); + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps({ + getSandboxReuseState: () => "ready", + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, recorded), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + resume: true, + sandboxName: "saved", + requestedObservabilityEnabled: requested, + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + recreate: true, + observabilityEnabled: requested, + }); + expect(calls.note).toHaveBeenCalledWith( + " [resume] Observability configuration changed; recreating sandbox.", + ); + expect(session.observabilityEnabled).toBe(requested); + expect(session.observabilityRequestedExplicitly).toBe(true); + }); + + it.each([ + { recorded: false, requested: true }, + { recorded: true, requested: false }, + ])("preserves interrupted explicit observability=$requested over registry=$recorded", async ({ + recorded, + requested, + }) => { + const session = createSession({ + sandboxName: "saved", + observabilityEnabled: requested, + observabilityRequestedExplicitly: true, + }); + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps({ + getSandboxReuseState: () => "ready", + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, recorded), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + resume: true, + sandboxName: "saved", + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + recreate: true, + observabilityEnabled: requested, + }); + expect(session.observabilityEnabled).toBe(requested); + expect(session.observabilityRequestedExplicitly).toBe(true); + }); + + it("does not treat an interrupted omitted request as an explicit disable", async () => { + const session = createSession({ + sandboxName: "saved", + observabilityEnabled: false, + observabilityRequestedExplicitly: false, + }); + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps({ + getSandboxReuseState: () => "ready", + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name, true), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + resume: true, + sandboxName: "saved", + }); + + expect(calls.createSandbox).not.toHaveBeenCalled(); + expect(session.observabilityEnabled).toBe(true); + expect(session.observabilityRequestedExplicitly).toBe(false); + }); + + it("recreates a ready DCode sandbox before opting out from unknown legacy state", async () => { + const session = createSession({ + sandboxName: "saved", + observabilityEnabled: false, + }); + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps({ + getSandboxReuseState: () => "ready", + getSandboxRegistryEntry: (name: string) => dcodeRegistryEntry(name), + updateSession: vi.fn((mutator: (value: Session) => Session | void) => { + return mutator(session) ?? session; + }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + agent: { name: "langchain-deepagents-code" }, + resume: true, + sandboxName: "saved", + requestedObservabilityEnabled: false, + }); + + expect(calls.createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + recreate: true, + observabilityEnabled: false, + }); + expect(calls.note).toHaveBeenCalledWith( + " [resume] Observability configuration changed; recreating sandbox.", + ); + }); + it("removes the conflicting Hermes nous-web gateway when Tavily is selected", async () => { const { deps, calls } = createDeps(); @@ -122,7 +449,7 @@ describe("handleSandboxState", () => { null, ["nous-audio"], null, - { recreate: false, toolDisclosure: "progressive" }, + { recreate: false, toolDisclosure: "progressive", observabilityEnabled: false }, ); expect(result.hermesToolGateways).toEqual(["nous-audio"]); expect(calls.note).toHaveBeenCalledWith( @@ -227,7 +554,7 @@ describe("handleSandboxState", () => { null, [], null, - { recreate: true, toolDisclosure: "progressive" }, + { recreate: true, toolDisclosure: "progressive", observabilityEnabled: false }, ); }); @@ -430,7 +757,7 @@ describe("handleSandboxState", () => { null, [], null, - { recreate: true, toolDisclosure: "progressive" }, + { recreate: true, toolDisclosure: "progressive", observabilityEnabled: false }, ); expect(result.webSearchConfigChanged).toBe(true); }); @@ -544,7 +871,7 @@ describe("handleSandboxState", () => { null, [], null, - { recreate: true, toolDisclosure: "progressive" }, + { recreate: true, toolDisclosure: "progressive", observabilityEnabled: false }, ); expect(result.webSearchConfig).toBeNull(); }); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 908addb145..019a603b47 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -20,6 +20,16 @@ import type { SandboxEntry } from "../../../state/registry"; import { getSandboxEntryInference } from "../../../state/registry-entry-view"; import { toolDisclosureOrDefault } from "../../../tool-disclosure"; import { resolveSandboxGatewayName } from "../../gateway-binding"; +import { + type ManagedSandboxFeatureIssue, + managedSandboxFeatureNeedsSessionUpdate, + resolveManagedSandboxFeature, +} from "../../managed-sandbox-feature"; +import { + DCODE_OBSERVABILITY_FEATURE, + hasDcodeObservabilityDrift, + isDcodeAgent, +} from "../../observability-policy-presets"; import { withSandboxPhaseTrace } from "../../tracing"; import type { SandboxCreateIntent } from "../../types"; import { branchTo, type OnboardStateTransitionResult } from "../result"; @@ -45,7 +55,10 @@ export interface SandboxStateOptions< fresh: boolean; /** Internal rebuild mode: null web-search state is an authoritative disable, not a prompt. */ authoritativeResumeConfig?: boolean; + /** Internal rebuild tier that must govern create-time and resumed policy selection. */ + authoritativePolicyTier?: string | null; resumeAgentChanged: boolean; + requestedObservabilityEnabled?: boolean | null; gatewayName: string; session: Session | null; sandboxName: string | null; @@ -273,6 +286,18 @@ function mcpRegistryRemovalBlockReason( return ` Sandbox '${sandboxName}' has managed MCP state. Use the transactional rebuild command before changing settings that recreate the sandbox.`; } +function observabilityRequestValidationError( + issue: ManagedSandboxFeatureIssue | null, +): string | null { + if (issue === "unsupported-request") { + return " --observability is supported only with --agent langchain-deepagents-code."; + } + if (issue === "recorded-state-on-unsupported-agent") { + return " Recorded observability belongs to the existing Deep Agents Code sandbox. Pass --no-observability explicitly when switching agents."; + } + return null; +} + class SandboxStateFlow< Gpu, Agent, @@ -422,12 +447,59 @@ class SandboxStateFlow< recordedToolGateways, effectiveToolGateways, ), + observabilityChanged: hasDcodeObservabilityDrift({ + liveExists: sandboxReuseState === "ready", + managedDcodeAgent: isDcodeAgent((this.options.agent as { name?: string } | null)?.name), + hasRegistryEntry: registryEntry !== null, + recordedObservabilityEnabled: registryEntry?.observabilityEnabled, + requestedObservabilityEnabled: state.session?.observabilityEnabled, + }), ...toolDisclosureSignals, ...dcodeResumeSignals, }); return dcodeResume.preserveManagedDcodeRegistryEntry(this.options, decision); } + private applyObservabilityRequest( + state: SandboxStepState, + ): SandboxStepState { + const registryEntry = state.sandboxName + ? this.deps.getSandboxRegistryEntry(state.sandboxName) + : null; + const selectedAgent = (this.options.agent as { name?: string } | null)?.name; + const requested = this.options.requestedObservabilityEnabled; + const resolution = resolveManagedSandboxFeature(DCODE_OBSERVABILITY_FEATURE, { + agent: selectedAgent, + requested, + resume: this.options.resume, + sessionValue: state.session?.observabilityEnabled, + sessionRequestedExplicitly: state.session?.observabilityRequestedExplicitly, + registryValue: registryEntry?.observabilityEnabled, + }); + const validationError = observabilityRequestValidationError(resolution.issue); + if (validationError) { + this.deps.error(validationError); + return this.deps.exitProcess(1); + } + if ( + !managedSandboxFeatureNeedsSessionUpdate( + DCODE_OBSERVABILITY_FEATURE, + state.session?.observabilityEnabled, + state.session?.observabilityRequestedExplicitly, + resolution, + ) + ) { + return state; + } + const session = this.deps.updateSession((current) => { + current.observabilityEnabled = resolution.value; + current.observabilityRequestedExplicitly = + current.observabilityRequestedExplicitly || resolution.requestedExplicitly; + return current; + }); + return { ...state, session }; + } + private assertGatewayRouteCompatible(sandboxName: string | null): void { const targetEntry = sandboxName ? this.deps.getSandboxRegistryEntry(sandboxName) : null; if (!sandboxName || !targetEntry) { @@ -607,6 +679,13 @@ class SandboxStateFlow< { recreate: decision.kind !== "create", toolDisclosure: toolDisclosureOrDefault(state.session?.toolDisclosure), + observabilityEnabled: state.session?.observabilityEnabled === true, + ...(state.session?.observabilityRequestedExplicitly === true + ? { observabilityRequestedExplicitly: true as const } + : {}), + ...(this.options.authoritativePolicyTier + ? { policyTier: this.options.authoritativePolicyTier } + : {}), }, ), ); @@ -728,7 +807,7 @@ class SandboxStateFlow< } async run(): Promise> { - const initialState = this.prepareWebSearchSupport(); + const initialState = this.applyObservabilityRequest(this.prepareWebSearchSupport()); const decision = this.resolveResumeDecision(initialState); const completedState = decision.kind === "reuse" diff --git a/src/lib/onboard/managed-sandbox-feature.test.ts b/src/lib/onboard/managed-sandbox-feature.test.ts new file mode 100644 index 0000000000..1c6f28511a --- /dev/null +++ b/src/lib/onboard/managed-sandbox-feature.test.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + type ManagedSandboxFeature, + managedSandboxFeatureHasDrift, + managedSandboxFeatureIssue, + managedSandboxFeatureNeedsSessionUpdate, + resolveManagedSandboxFeature, +} from "./managed-sandbox-feature"; + +const feature: ManagedSandboxFeature = { + id: "test-feature", + defaultValue: false, + isValue: (value): value is boolean => typeof value === "boolean", + isEnabled: (value) => value, + supportsAgent: (agent) => agent === "supported", +}; + +describe("managed sandbox feature", () => { + it("resolves explicit, resumable, registry, session, and default intent in order", () => { + expect( + resolveManagedSandboxFeature(feature, { + agent: "supported", + requested: false, + resume: true, + sessionValue: true, + sessionRequestedExplicitly: true, + registryValue: true, + }), + ).toMatchObject({ value: false, source: "explicit", requestedExplicitly: true }); + expect( + resolveManagedSandboxFeature(feature, { + agent: "supported", + resume: true, + sessionValue: true, + sessionRequestedExplicitly: true, + registryValue: false, + }), + ).toMatchObject({ value: true, source: "session-explicit" }); + expect( + resolveManagedSandboxFeature(feature, { + agent: "supported", + sessionValue: false, + registryValue: true, + }), + ).toMatchObject({ value: true, source: "registry" }); + expect( + resolveManagedSandboxFeature(feature, { agent: "supported", sessionValue: true }), + ).toMatchObject({ value: true, source: "session" }); + expect(resolveManagedSandboxFeature(feature, { agent: "supported" })).toMatchObject({ + value: false, + source: "default", + }); + }); + + it("classifies unsupported enablement and permits an explicit disable", () => { + expect(managedSandboxFeatureIssue(feature, { agent: "unsupported", requested: true })).toBe( + "unsupported-request", + ); + expect( + managedSandboxFeatureIssue(feature, { + agent: "unsupported", + sessionValue: true, + }), + ).toBe("recorded-state-on-unsupported-agent"); + expect( + managedSandboxFeatureIssue(feature, { + agent: "unsupported", + requested: false, + sessionValue: true, + }), + ).toBeNull(); + }); + + it("updates session provenance only for an explicit request", () => { + const inherited = resolveManagedSandboxFeature(feature, { + agent: "supported", + registryValue: true, + }); + const explicit = resolveManagedSandboxFeature(feature, { + agent: "supported", + requested: true, + }); + expect(managedSandboxFeatureNeedsSessionUpdate(feature, true, false, inherited)).toBe(false); + expect(managedSandboxFeatureNeedsSessionUpdate(feature, true, false, explicit)).toBe(true); + }); + + it("treats missing authoritative registry state as drift only for a live supported sandbox", () => { + const base = { + liveExists: true, + hasRegistryEntry: true, + agent: "supported", + desiredValue: true, + }; + expect(managedSandboxFeatureHasDrift(feature, { ...base, recordedValue: undefined })).toBe( + true, + ); + expect(managedSandboxFeatureHasDrift(feature, { ...base, recordedValue: false })).toBe(true); + expect(managedSandboxFeatureHasDrift(feature, { ...base, recordedValue: true })).toBe(false); + expect( + managedSandboxFeatureHasDrift(feature, { ...base, liveExists: false, recordedValue: false }), + ).toBe(false); + }); +}); diff --git a/src/lib/onboard/managed-sandbox-feature.ts b/src/lib/onboard/managed-sandbox-feature.ts new file mode 100644 index 0000000000..0f622c2330 --- /dev/null +++ b/src/lib/onboard/managed-sandbox-feature.ts @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** A durable, agent-scoped sandbox setting that can participate in resume and recreate. */ +export interface ManagedSandboxFeature { + readonly id: string; + readonly defaultValue: T; + readonly isValue: (value: unknown) => value is T; + readonly isEnabled: (value: T) => boolean; + readonly supportsAgent: (agent: string | null | undefined) => boolean; + readonly equals?: (left: T, right: T) => boolean; +} + +export type ManagedSandboxFeatureIssue = + | "unsupported-request" + | "recorded-state-on-unsupported-agent"; + +export type ManagedSandboxFeatureIntentSource = + | "explicit" + | "session-explicit" + | "registry" + | "session" + | "default"; + +export interface ManagedSandboxFeatureResolution { + value: T; + source: ManagedSandboxFeatureIntentSource; + requestedExplicitly: boolean; + issue: ManagedSandboxFeatureIssue | null; +} + +export interface ManagedSandboxFeatureIntentInput { + agent: string | null | undefined; + requested?: T | null; + resume?: boolean; + sessionValue?: T | null; + sessionRequestedExplicitly?: boolean; + registryValue?: T | null; +} + +function featureValuesEqual(feature: ManagedSandboxFeature, left: T, right: T): boolean { + return feature.equals ? feature.equals(left, right) : Object.is(left, right); +} + +export function managedSandboxFeatureIssue( + feature: ManagedSandboxFeature, + input: Pick< + ManagedSandboxFeatureIntentInput, + "agent" | "requested" | "sessionValue" | "registryValue" + >, +): ManagedSandboxFeatureIssue | null { + if (feature.supportsAgent(input.agent)) return null; + if (feature.isValue(input.requested) && feature.isEnabled(input.requested)) { + return "unsupported-request"; + } + if (feature.isValue(input.requested)) return null; + const recordedEnabled = [input.sessionValue, input.registryValue].some( + (value) => feature.isValue(value) && feature.isEnabled(value), + ); + return recordedEnabled ? "recorded-state-on-unsupported-agent" : null; +} + +/** Resolve explicit intent before durable registry/session state, preserving its provenance. */ +export function resolveManagedSandboxFeature( + feature: ManagedSandboxFeature, + input: ManagedSandboxFeatureIntentInput, +): ManagedSandboxFeatureResolution { + const issue = managedSandboxFeatureIssue(feature, input); + const requestedExplicitly = feature.isValue(input.requested); + if (!feature.supportsAgent(input.agent)) { + return { + value: feature.defaultValue, + source: "default", + requestedExplicitly, + issue, + }; + } + if (feature.isValue(input.requested)) { + return { value: input.requested, source: "explicit", requestedExplicitly: true, issue }; + } + if ( + input.resume === true && + input.sessionRequestedExplicitly === true && + feature.isValue(input.sessionValue) + ) { + return { + value: input.sessionValue, + source: "session-explicit", + requestedExplicitly: false, + issue, + }; + } + if (feature.isValue(input.registryValue)) { + return { + value: input.registryValue, + source: "registry", + requestedExplicitly: false, + issue, + }; + } + if (feature.isValue(input.sessionValue)) { + return { + value: input.sessionValue, + source: "session", + requestedExplicitly: false, + issue, + }; + } + return { + value: feature.defaultValue, + source: "default", + requestedExplicitly: false, + issue, + }; +} + +export function managedSandboxFeatureNeedsSessionUpdate( + feature: ManagedSandboxFeature, + sessionValue: T | null | undefined, + sessionRequestedExplicitly: boolean | null | undefined, + resolution: ManagedSandboxFeatureResolution, +): boolean { + return ( + !feature.isValue(sessionValue) || + !featureValuesEqual(feature, sessionValue, resolution.value) || + (resolution.requestedExplicitly && sessionRequestedExplicitly !== true) + ); +} + +export function managedSandboxFeatureHasDrift( + feature: ManagedSandboxFeature, + input: { + liveExists: boolean; + hasRegistryEntry: boolean; + agent: string | null | undefined; + recordedValue: T | null | undefined; + desiredValue: T; + }, +): boolean { + if (!input.liveExists || !input.hasRegistryEntry || !feature.supportsAgent(input.agent)) { + return false; + } + // A legacy row without authoritative create-time state must be recreated so + // either enabling or disabling can establish the requested durable value. + if (!feature.isValue(input.recordedValue)) return true; + return !featureValuesEqual(feature, input.recordedValue, input.desiredValue); +} diff --git a/src/lib/onboard/observability-command-flag.ts b/src/lib/onboard/observability-command-flag.ts new file mode 100644 index 0000000000..7f84bb7e9d --- /dev/null +++ b/src/lib/onboard/observability-command-flag.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Render an observability flag only when the operator explicitly requested that state. */ +export function explicitObservabilityFlag( + enabled: boolean, + requestedExplicitly: boolean, +): "--observability" | "--no-observability" | null { + if (!requestedExplicitly) return null; + return enabled ? "--observability" : "--no-observability"; +} diff --git a/src/lib/onboard/observability-policy-presets.test.ts b/src/lib/onboard/observability-policy-presets.test.ts new file mode 100644 index 0000000000..7f5e846bdb --- /dev/null +++ b/src/lib/onboard/observability-policy-presets.test.ts @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + hasDcodeObservabilityDrift, + hasRegisteredDcodeObservabilityDrift, + isInactiveObservabilityPolicyPreset, + mergeRequiredObservabilityPolicyPresets, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + requiredObservabilityPolicyPresets, +} from "./observability-policy-presets"; +import { suppressedAgentRequiredPresets } from "./policy-tier-suppression"; + +describe("observability policy presets", () => { + it("detects enabled and disabled drift for a live managed DCode sandbox", () => { + const base = { + liveExists: true, + managedDcodeAgent: true, + hasRegistryEntry: true, + }; + + expect( + hasDcodeObservabilityDrift({ + ...base, + recordedObservabilityEnabled: true, + requestedObservabilityEnabled: false, + }), + ).toBe(true); + expect( + hasDcodeObservabilityDrift({ + ...base, + recordedObservabilityEnabled: false, + requestedObservabilityEnabled: true, + }), + ).toBe(true); + expect( + hasDcodeObservabilityDrift({ + ...base, + recordedObservabilityEnabled: undefined, + requestedObservabilityEnabled: false, + }), + ).toBe(true); + expect( + hasDcodeObservabilityDrift({ + ...base, + recordedObservabilityEnabled: undefined, + requestedObservabilityEnabled: true, + }), + ).toBe(true); + expect( + hasDcodeObservabilityDrift({ + ...base, + recordedObservabilityEnabled: true, + requestedObservabilityEnabled: true, + }), + ).toBe(false); + expect( + hasDcodeObservabilityDrift({ + ...base, + liveExists: false, + recordedObservabilityEnabled: true, + requestedObservabilityEnabled: false, + }), + ).toBe(false); + expect( + hasRegisteredDcodeObservabilityDrift(true, true, { observabilityEnabled: false }, true), + ).toBe(true); + expect(hasRegisteredDcodeObservabilityDrift(true, true, null, true)).toBe(false); + }); + + it("requires the fixed local OTLP preset only for enabled Deep Agents Code", () => { + expect(requiredObservabilityPolicyPresets("langchain-deepagents-code", true)).toEqual([ + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + ]); + expect(requiredObservabilityPolicyPresets("langchain-deepagents-code", false)).toEqual([]); + expect(requiredObservabilityPolicyPresets("openclaw", true)).toEqual([]); + expect(requiredObservabilityPolicyPresets("hermes", true)).toEqual([]); + }); + + it("adds only a known preset and prunes an inactive built-in selection", () => { + expect( + mergeRequiredObservabilityPolicyPresets(["npm"], { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + knownPresetNames: ["npm", OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET], + }), + ).toEqual(["npm", OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET]); + expect( + mergeRequiredObservabilityPolicyPresets(["npm"], { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + knownPresetNames: ["npm"], + }), + ).toEqual(["npm"]); + expect( + isInactiveObservabilityPolicyPreset(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, { + agent: "langchain-deepagents-code", + observabilityEnabled: false, + }), + ).toBe(true); + }); + + it("suppresses the built-in when exact custom content owns its policy key", () => { + expect( + mergeRequiredObservabilityPolicyPresets(["npm"], { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + knownPresetNames: ["npm", OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET], + customOwnsObservability: true, + }), + ).toEqual(["npm"]); + expect( + isInactiveObservabilityPolicyPreset(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + customOwnsObservability: true, + }), + ).toBe(true); + expect( + isInactiveObservabilityPolicyPreset(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + customPresetNames: new Set([OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET]), + customOwnsObservability: false, + }), + ).toBe(false); + }); + + it("suppresses local trace egress on the restricted tier", () => { + expect(suppressedAgentRequiredPresets("restricted", "langchain-deepagents-code")).toEqual([ + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + ]); + expect(suppressedAgentRequiredPresets("balanced", "langchain-deepagents-code")).toEqual([]); + }); +}); diff --git a/src/lib/onboard/observability-policy-presets.ts b/src/lib/onboard/observability-policy-presets.ts new file mode 100644 index 0000000000..a1e9b692a5 --- /dev/null +++ b/src/lib/onboard/observability-policy-presets.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ManagedPolicyBinding } from "../policy/managed-policy-binding"; +import { + type ManagedSandboxFeature, + managedSandboxFeatureHasDrift, +} from "./managed-sandbox-feature"; + +export const DCODE_AGENT_NAME = "langchain-deepagents-code"; +export const OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET = "observability-otlp-local"; +export const OBSERVABILITY_POLICY_BINDING = new ManagedPolicyBinding({ + presetName: OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, +}); + +export const DCODE_ONLY_POLICY_PRESETS = new Set([OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET]); + +export function isDcodeAgent(agent: string | null | undefined): boolean { + return typeof agent === "string" && agent.trim().toLowerCase() === DCODE_AGENT_NAME; +} + +export const DCODE_OBSERVABILITY_FEATURE: ManagedSandboxFeature = { + id: "observability", + defaultValue: false, + isValue: (value): value is boolean => typeof value === "boolean", + isEnabled: (value) => value, + supportsAgent: isDcodeAgent, +}; + +export function hasDcodeObservabilityDrift(options: { + liveExists: boolean; + managedDcodeAgent: boolean; + hasRegistryEntry: boolean; + recordedObservabilityEnabled: boolean | null | undefined; + requestedObservabilityEnabled: boolean | null | undefined; +}): boolean { + return managedSandboxFeatureHasDrift(DCODE_OBSERVABILITY_FEATURE, { + liveExists: options.liveExists, + hasRegistryEntry: options.hasRegistryEntry, + agent: options.managedDcodeAgent ? DCODE_AGENT_NAME : null, + recordedValue: options.recordedObservabilityEnabled, + desiredValue: options.requestedObservabilityEnabled === true, + }); +} + +export function hasRegisteredDcodeObservabilityDrift( + liveExists: boolean, + managedDcodeAgent: boolean, + registryEntry: { observabilityEnabled?: boolean | null } | null, + requestedObservabilityEnabled: boolean | null | undefined, +): boolean { + return hasDcodeObservabilityDrift({ + liveExists, + managedDcodeAgent, + hasRegistryEntry: registryEntry !== null, + recordedObservabilityEnabled: registryEntry?.observabilityEnabled, + requestedObservabilityEnabled, + }); +} + +export function requiredObservabilityPolicyPresets( + agent: string | null | undefined, + observabilityEnabled: boolean | null | undefined, +): string[] { + return observabilityEnabled === true && isDcodeAgent(agent) + ? [OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET] + : []; +} + +export function isInactiveObservabilityPolicyPreset( + presetName: string, + options: { + agent?: string | null; + observabilityEnabled?: boolean | null; + customPresetNames?: ReadonlySet | null; + customOwnsObservability?: boolean; + } = {}, +): boolean { + const name = presetName.trim().toLowerCase(); + if (options.customPresetNames?.has(name)) return false; + if (name === OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET && options.customOwnsObservability) { + return true; + } + return ( + name === OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET && + isDcodeAgent(options.agent) && + options.observabilityEnabled !== true + ); +} + +export function mergeRequiredObservabilityPolicyPresets( + selectedPresets: string[], + options: { + agent?: string | null; + observabilityEnabled?: boolean | null; + knownPresetNames?: Iterable | null; + customOwnsObservability?: boolean; + } = {}, +): string[] { + const merged = [...selectedPresets]; + const selected = new Set(merged); + const known = options.knownPresetNames ? new Set(options.knownPresetNames) : null; + + for (const preset of requiredObservabilityPolicyPresets( + options.agent, + options.observabilityEnabled, + )) { + if (options.customOwnsObservability) continue; + if (known && !known.has(preset)) continue; + if (selected.has(preset)) continue; + merged.push(preset); + selected.add(preset); + } + + return merged; +} diff --git a/src/lib/onboard/policy-preset-reconciliation.ts b/src/lib/onboard/policy-preset-reconciliation.ts new file mode 100644 index 0000000000..efb3953daf --- /dev/null +++ b/src/lib/onboard/policy-preset-reconciliation.ts @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type WebSearchConfig, webSearchProviderForConfig } from "../inference/web-search"; +import { filterSetupPolicyPresetNamesForAgent } from "./agent-policy-presets"; +import { mergeRequiredHermesToolGatewayPolicyPresets } from "./hermes-managed-tools"; +import { + mergeEnabledMessagingChannelPolicyPresets, + pruneDisabledMessagingPolicyPresets, +} from "./messaging-policy-presets"; +import { + isInactiveObservabilityPolicyPreset, + mergeRequiredObservabilityPolicyPresets, +} from "./observability-policy-presets"; +import { mergeRequiredOpenclawOtelPolicyPresets } from "./openclaw-otel-policy-presets"; +import { filterSuppressedAgentRequiredPresets } from "./policy-tier-suppression"; + +export type RequiredSetupPolicyPresetOptions = { + enabledChannels?: string[] | null; + hermesToolGateways?: string[] | null; + agent?: string | null; + observabilityEnabled?: boolean | null; + knownPresetNames?: string[] | Set | null; + env?: NodeJS.ProcessEnv; + tierName?: string | null; + webSearchConfig?: WebSearchConfig | null; + customPresetNames?: ReadonlySet | null; + customOwnsObservability?: boolean; +}; + +export function mergeRequiredSetupPolicyPresets( + policyPresets: string[], + options: RequiredSetupPolicyPresetOptions = {}, +): string[] { + const agentFilteredPresets = filterSetupPolicyPresetNamesForAgent( + policyPresets, + options.agent, + ).filter( + (name) => + !isInactiveObservabilityPolicyPreset(name, { + agent: options.agent, + observabilityEnabled: options.observabilityEnabled, + customPresetNames: options.customPresetNames, + customOwnsObservability: options.customOwnsObservability, + }), + ); + const effectiveHermesToolGateways = (options.hermesToolGateways ?? []).filter( + (name) => + !isStaleBuiltinWebSearchPolicyPreset(name, { + webSearchConfig: options.webSearchConfig, + customPresetNames: options.customPresetNames, + }), + ); + const mergedPresets = mergeRequiredObservabilityPolicyPresets( + mergeRequiredOpenclawOtelPolicyPresets( + mergeEnabledMessagingChannelPolicyPresets( + mergeRequiredHermesToolGatewayPolicyPresets( + agentFilteredPresets, + effectiveHermesToolGateways, + options.knownPresetNames, + ), + options.enabledChannels, + options.knownPresetNames, + ), + { + agent: options.agent, + knownPresetNames: options.knownPresetNames, + env: options.env, + }, + ), + { + agent: options.agent, + observabilityEnabled: options.observabilityEnabled, + knownPresetNames: options.knownPresetNames, + customOwnsObservability: options.customOwnsObservability, + }, + ); + const agentScoped = filterSetupPolicyPresetNamesForAgent(mergedPresets, options.agent); + return filterSuppressedAgentRequiredPresets(agentScoped, options.tierName, options.agent); +} + +export function isStaleBuiltinBravePolicyPreset( + name: string, + options: { + webSearchConfig?: WebSearchConfig | null; + customPresetNames?: ReadonlySet | null; + } = {}, +): boolean { + return isStaleBuiltinWebSearchPolicyPreset(name, options); +} + +export function isStaleBuiltinWebSearchPolicyPreset( + name: string, + options: { + webSearchConfig?: WebSearchConfig | null; + customPresetNames?: ReadonlySet | null; + } = {}, +): boolean { + if (options.customPresetNames?.has(name)) return false; + if (name === "nous-web") { + return Boolean( + options.webSearchConfig && webSearchProviderForConfig(options.webSearchConfig) === "tavily", + ); + } + if (name !== "brave" && name !== "tavily") return false; + if (!options.webSearchConfig) return true; + return name !== webSearchProviderForConfig(options.webSearchConfig); +} + +export function createUnavailablePolicyPresetPruner(options: { + disabledChannels?: string[] | null; + agent?: string | null; + observabilityEnabled?: boolean | null; + webSearchConfig?: WebSearchConfig | null; + customPresetNames?: ReadonlySet | null; + customOwnsObservability?: boolean; +}): (presetNames: string[], pruning?: { preserveExplicitWebSearch?: boolean }) => string[] { + // Custom and interactive selections may explicitly opt into a built-in web-search + // preset without storing provider config. Inactive observability remains ineligible. + return (presetNames, pruning = {}) => + pruneDisabledMessagingPolicyPresets(presetNames, options.disabledChannels).filter( + (name) => + (pruning.preserveExplicitWebSearch || + !isStaleBuiltinWebSearchPolicyPreset(name, options)) && + !isInactiveObservabilityPolicyPreset(name, options), + ); +} diff --git a/src/lib/onboard/policy-presets.ts b/src/lib/onboard/policy-presets.ts index dfa2b1789b..fa0f24388a 100644 --- a/src/lib/onboard/policy-presets.ts +++ b/src/lib/onboard/policy-presets.ts @@ -7,6 +7,7 @@ import { listMessagingCredentialMetadata, listMessagingPolicyPresetMetadata, } from "../messaging/channels"; +import { requiredObservabilityPolicyPresets } from "./observability-policy-presets"; const { LOCAL_INFERENCE_PROVIDERS } = require("./providers") as { LOCAL_INFERENCE_PROVIDERS: string[]; @@ -19,6 +20,7 @@ export interface SuggestedPolicyPresetOptions { webSearchConfig?: WebSearchConfig | null; provider?: string | null; agent?: string | null; + observabilityEnabled?: boolean | null; isNonInteractive?: () => boolean; env?: NodeJS.ProcessEnv; } @@ -28,6 +30,7 @@ export function getSuggestedPolicyPresets({ webSearchConfig = null, provider = null, agent = null, + observabilityEnabled = false, isNonInteractive, env = process.env, }: SuggestedPolicyPresetOptions = {}): string[] { @@ -40,6 +43,7 @@ export function getSuggestedPolicyPresets({ suggestions.push("openclaw-pricing"); suggestions.push(...requiredOpenclawOtelPolicyPresets(agent, env)); } + suggestions.push(...requiredObservabilityPolicyPresets(agent, observabilityEnabled)); const usesExplicitMessagingSelection = Array.isArray(enabledChannels); const nonInteractive = isNonInteractive?.() ?? process.env.NEMOCLAW_NON_INTERACTIVE === "1"; diff --git a/src/lib/onboard/policy-resume-selection.test.ts b/src/lib/onboard/policy-resume-selection.test.ts index ebe16d1387..5237cef858 100644 --- a/src/lib/onboard/policy-resume-selection.test.ts +++ b/src/lib/onboard/policy-resume-selection.test.ts @@ -7,13 +7,19 @@ import { preparePolicyPresetResumeSelection } from "./policy-resume-selection"; type Preset = { name: string; access?: string }; -function policies(options: { applied?: string[]; custom?: string[] } = {}) { - const setupPresets = ["npm", "brave", "tavily"].map((name) => ({ name })); +function policies( + options: { applied?: string[]; custom?: string[]; customOwnsObservability?: boolean } = {}, +) { + const setupPresets = ["npm", "brave", "tavily", "observability-otlp-local"].map((name) => ({ + name, + })); const customPresets = (options.custom ?? []).map((name) => ({ name })); return { setupPolicyPresetSupported: () => true, listSetupPolicyPresets: () => setupPresets, listCustomPresets: () => customPresets, + customPresetOwnsNetworkPolicyKey: () => options.customOwnsObservability === true, + removeBuiltinPresetAttribution: () => undefined, getAppliedPresets: () => options.applied ?? [], clampSetupPolicyPresetNames( names: string[], @@ -80,3 +86,79 @@ describe("preparePolicyPresetResumeSelection web search reconciliation", () => { expect(result.recordedPolicyPresetsNeedReconcile).toBe(true); }); }); + +describe("preparePolicyPresetResumeSelection observability reconciliation", () => { + it("adds the local OTLP preset only while Deep Agents Code observability is enabled", () => { + const enabled = preparePolicyPresetResumeSelection({ policies: policies() }, "alpha", { + recordedPolicyPresets: ["npm"], + agent: "langchain-deepagents-code", + observabilityEnabled: true, + webSearchConfig: null, + webSearchSupported: true, + }); + const disabled = preparePolicyPresetResumeSelection({ policies: policies() }, "alpha", { + recordedPolicyPresets: ["npm", "observability-otlp-local"], + agent: "langchain-deepagents-code", + observabilityEnabled: false, + webSearchConfig: null, + webSearchSupported: true, + }); + + expect(enabled.policyPresets).toEqual(["npm", "observability-otlp-local"]); + expect(enabled.recordedPolicyPresetsNeedReconcile).toBe(true); + expect(disabled.policyPresets).toEqual(["npm"]); + expect(disabled.recordedPolicyPresetsNeedReconcile).toBe(true); + }); + + it("suppresses the enabled local OTLP preset on the restricted tier", () => { + const result = preparePolicyPresetResumeSelection({ policies: policies() }, "alpha", { + recordedPolicyPresets: ["npm"], + agent: "langchain-deepagents-code", + observabilityEnabled: true, + webSearchConfig: null, + webSearchSupported: true, + tierName: "restricted", + }); + + expect(result.policyPresets).toEqual(["npm"]); + }); + + it("keeps exact custom OTLP ownership without carrying built-in attribution on resume", () => { + const result = preparePolicyPresetResumeSelection( + { + policies: policies({ + applied: ["observability-otlp-local", "corp-otel"], + custom: ["corp-otel"], + customOwnsObservability: true, + }), + }, + "alpha", + { + recordedPolicyPresets: ["observability-otlp-local", "corp-otel"], + agent: "langchain-deepagents-code", + observabilityEnabled: true, + webSearchConfig: null, + webSearchSupported: true, + }, + ); + + expect(result.policyPresets).toEqual(["corp-otel"]); + expect(result.recordedPolicyPresetsNeedReconcile).toBe(true); + }); + + it("preserves same-name different-key custom collision semantics on resume", () => { + const result = preparePolicyPresetResumeSelection( + { policies: policies({ custom: ["observability-otlp-local"] }) }, + "alpha", + { + recordedPolicyPresets: ["observability-otlp-local"], + agent: "langchain-deepagents-code", + observabilityEnabled: true, + webSearchConfig: null, + webSearchSupported: true, + }, + ); + + expect(result.policyPresets).toEqual(["observability-otlp-local"]); + }); +}); diff --git a/src/lib/onboard/policy-resume-selection.ts b/src/lib/onboard/policy-resume-selection.ts index 058333571e..497348ef58 100644 --- a/src/lib/onboard/policy-resume-selection.ts +++ b/src/lib/onboard/policy-resume-selection.ts @@ -11,6 +11,10 @@ import { mergeAppliedPolicyPresetsForDisabledMessagingCleanup, pruneDisabledMessagingPolicyPresets, } from "./messaging-policy-presets"; +import { + isInactiveObservabilityPolicyPreset, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, +} from "./observability-policy-presets"; import { isStaleBuiltinWebSearchPolicyPreset, mergeRequiredSetupPolicyPresets, @@ -31,6 +35,8 @@ type PoliciesApi = { ): Preset[]; listCustomPresets(sandboxName: string): Preset[]; getAppliedPresets(sandboxName: string): string[]; + customPresetOwnsNetworkPolicyKey?(sandboxName: string, policyKey: string): boolean; + removeBuiltinPresetAttribution?(sandboxName: string, presetName: string): void; clampSetupPolicyPresetNames( names: string[], selectablePresets: Preset[], @@ -48,6 +54,7 @@ export function preparePolicyPresetResumeSelection( enabledChannels?: string[] | null; hermesToolGateways?: string[] | null; agent?: string | null; + observabilityEnabled?: boolean | null; webSearchConfig?: WebSearchConfig | null; webSearchConfigChanged?: boolean; webSearchSupported?: boolean | null; @@ -56,7 +63,28 @@ export function preparePolicyPresetResumeSelection( }, ): PreparedPolicyResumeSelection { const supportOptions = { webSearchSupported: options.webSearchSupported, agent: options.agent }; - const appliedPolicyPresets = deps.policies.getAppliedPresets(sandboxName); + const customPolicyPresetNames = new Set( + deps.policies.listCustomPresets(sandboxName).map((preset) => preset.name), + ); + const customOwnsObservability = + deps.policies.customPresetOwnsNetworkPolicyKey?.( + sandboxName, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + ) === true; + if (customOwnsObservability) { + deps.policies.removeBuiltinPresetAttribution?.( + sandboxName, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + ); + } + const rawAppliedPolicyPresets = deps.policies.getAppliedPresets(sandboxName); + const appliedPolicyPresets = customOwnsObservability + ? [...new Set(rawAppliedPolicyPresets)].filter( + (name) => + name !== OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET || + customPolicyPresetNames.has(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET), + ) + : rawAppliedPolicyPresets; const selectablePolicyPresets = [ ...filterSetupPolicyPresetsForAgent( deps.policies.listSetupPolicyPresets(sandboxName, supportOptions), @@ -66,9 +94,6 @@ export function preparePolicyPresetResumeSelection( name, })), ]; - const customPolicyPresetNames = new Set( - deps.policies.listCustomPresets(sandboxName).map((preset) => preset.name), - ); const clampedRecordedPolicyPresets = deps.policies.clampSetupPolicyPresetNames( options.recordedPolicyPresets || [], selectablePolicyPresets, @@ -80,11 +105,20 @@ export function preparePolicyPresetResumeSelection( webSearchConfig: options.webSearchConfig, customPresetNames: customPolicyPresetNames, }); + const isInactiveObservability = (name: string) => + isInactiveObservabilityPolicyPreset(name, { + agent: options.agent, + observabilityEnabled: options.observabilityEnabled, + customPresetNames: customPolicyPresetNames, + customOwnsObservability, + }); const recordedBuiltinWebSearchProviderChanged = clampedRecordedPolicyPresets.some( (name) => (name === "brave" || name === "tavily") && isStaleBuiltinWebSearch(name), ); let policyPresets = pruneDisabledMessagingPolicyPresets( - clampedRecordedPolicyPresets.filter((name) => !isStaleBuiltinWebSearch(name)), + clampedRecordedPolicyPresets.filter( + (name) => !isStaleBuiltinWebSearch(name) && !isInactiveObservability(name), + ), options.disabledChannels, ); const appliedPolicyPresetsForSupport = deps.policies @@ -94,7 +128,7 @@ export function preparePolicyPresetResumeSelection( supportOptions, customPolicyPresetNames, ) - .filter((name) => !isStaleBuiltinWebSearch(name)); + .filter((name) => !isStaleBuiltinWebSearch(name) && !isInactiveObservability(name)); const disabledMessagingPolicyPresetApplied = hasDisabledMessagingPolicyPreset( appliedPolicyPresetsForSupport, options.disabledChannels, @@ -109,11 +143,13 @@ export function preparePolicyPresetResumeSelection( enabledChannels: options.enabledChannels, hermesToolGateways: options.hermesToolGateways, agent: options.agent, + observabilityEnabled: options.observabilityEnabled, knownPresetNames: selectablePolicyPresets.map((preset) => preset.name), env: options.env, tierName: options.tierName, webSearchConfig: options.webSearchConfig, customPresetNames: customPolicyPresetNames, + customOwnsObservability, }); // Provider switches are build-time changes, but their matching egress diff --git a/src/lib/onboard/policy-selection-recorded-tier.test.ts b/src/lib/onboard/policy-selection-recorded-tier.test.ts new file mode 100644 index 0000000000..e3f212c317 --- /dev/null +++ b/src/lib/onboard/policy-selection-recorded-tier.test.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { type SetupPolicySelectionDeps, setupPoliciesWithSelection } from "./policy-selection"; + +describe("policy selection after interrupted onboarding", () => { + it("reuses the recorded restricted tier before policy presets exist", async () => { + const selectPolicyTier = vi.fn(async () => "balanced"); + const setPolicyTier = vi.fn(); + const syncPresetSelection = vi.fn(); + const onSelection = vi.fn(); + const deps = { + policies: { + setupPolicyPresetSupported: vi.fn(() => true), + listSetupPolicyPresets: vi.fn(() => [{ name: "observability-otlp-local" }]), + listCustomPresets: vi.fn(() => []), + getAppliedPresets: vi.fn(() => []), + customPresetOwnsNetworkPolicyKey: vi.fn(() => false), + removeBuiltinPresetAttribution: vi.fn(), + clampSetupPolicyPresetNames: vi.fn((names: string[]) => [...names]), + }, + tiers: { + resolveTierPresets: vi.fn((tierName: string) => + tierName === "balanced" ? [{ name: "observability-otlp-local" }] : [], + ), + getTier: vi.fn(() => ({})), + }, + localInferenceProviders: [], + step: vi.fn(), + note: vi.fn(), + isNonInteractive: vi.fn(() => true), + waitForSandboxReady: vi.fn(() => true), + syncPresetSelection, + selectPolicyTier, + setPolicyTier, + getRecordedPolicyTier: vi.fn(() => null), + selectTierPresetsAndAccess: vi.fn(async () => []), + parsePolicyPresetEnv: vi.fn(() => []), + env: { NEMOCLAW_POLICY_MODE: "suggested" }, + } satisfies SetupPolicySelectionDeps; + + await expect( + setupPoliciesWithSelection(deps, "alpha", { + selectedPresets: null, + tierName: "restricted", + agent: "langchain-deepagents-code", + observabilityEnabled: true, + onSelection, + }), + ).resolves.toEqual([]); + + expect(selectPolicyTier).not.toHaveBeenCalled(); + expect(setPolicyTier).toHaveBeenCalledWith("alpha", "restricted"); + expect(onSelection).toHaveBeenCalledWith([]); + expect(syncPresetSelection).toHaveBeenCalledWith("alpha", [], []); + }); +}); diff --git a/src/lib/onboard/policy-selection.ts b/src/lib/onboard/policy-selection.ts index a7b7cb780f..3bf8168fcd 100644 --- a/src/lib/onboard/policy-selection.ts +++ b/src/lib/onboard/policy-selection.ts @@ -10,23 +10,31 @@ import { import { allHermesToolGatewayPolicyPresets, HERMES_TOOL_GATEWAY_PRESET_NAMES, - mergeRequiredHermesToolGatewayPolicyPresets, } from "./hermes-managed-tools"; +import { allMessagingChannelPolicyPresets } from "./messaging-policy-presets"; import { - allMessagingChannelPolicyPresets, - mergeEnabledMessagingChannelPolicyPresets, - pruneDisabledMessagingPolicyPresets, -} from "./messaging-policy-presets"; -import { mergeRequiredOpenclawOtelPolicyPresets } from "./openclaw-otel-policy-presets"; + isInactiveObservabilityPolicyPreset, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + requiredObservabilityPolicyPresets, +} from "./observability-policy-presets"; import { seedInitialPolicyContext } from "./policy-context-seed"; +import { + createUnavailablePolicyPresetPruner, + isStaleBuiltinWebSearchPolicyPreset, + mergeRequiredSetupPolicyPresets, +} from "./policy-preset-reconciliation"; import { agentRequiredPresetAdditions, emitSuppressedAgentRequiredPresetsNote, - filterSuppressedAgentRequiredPresets, RESTRICTED_TIER_NAME, } from "./policy-tier-suppression"; import { withPolicyApplicationTrace } from "./tracing"; +export { + isStaleBuiltinBravePolicyPreset, + isStaleBuiltinWebSearchPolicyPreset, + mergeRequiredSetupPolicyPresets, +} from "./policy-preset-reconciliation"; export { suppressedAgentRequiredPresets } from "./policy-tier-suppression"; type Preset = { name: string; access?: string }; @@ -36,6 +44,8 @@ type PoliciesApi = { listSetupPolicyPresets(sandboxName: string, options?: SupportOptions): Preset[]; listCustomPresets(sandboxName: string): Preset[]; getAppliedPresets(sandboxName: string): string[]; + customPresetOwnsNetworkPolicyKey?(sandboxName: string, policyKey: string): boolean; + removeBuiltinPresetAttribution?(sandboxName: string, presetName: string): void; clampSetupPolicyPresetNames( names: string[], selectablePresets: Preset[], @@ -53,10 +63,12 @@ export type SetupPresetSuggestionOptions = { webSearchConfig?: WebSearchConfig | null; provider?: string | null; agent?: string | null; + observabilityEnabled?: boolean | null; knownPresetNames?: string[] | null; webSearchSupported?: boolean | null; hermesToolGateways?: string[] | null; customPresetNames?: ReadonlySet | null; + customOwnsObservability?: boolean; env?: NodeJS.ProcessEnv; }; @@ -67,6 +79,9 @@ export type SetupPolicySelectionOptions = { enabledChannels?: string[] | null; provider?: string | null; agent?: string | null; + observabilityEnabled?: boolean | null; + /** Authoritative tier for transactional resume before registry registration is complete. */ + tierName?: string | null; knownPresetNames?: string[]; webSearchSupported?: boolean | null; hermesToolGateways?: string[] | null; @@ -106,75 +121,6 @@ export type PreparedPolicyResumeSelection = { suppressedAgentRequiredPresetsLive: boolean; }; -export function mergeRequiredSetupPolicyPresets( - policyPresets: string[], - options: { - enabledChannels?: string[] | null; - hermesToolGateways?: string[] | null; - agent?: string | null; - knownPresetNames?: string[] | Set | null; - env?: NodeJS.ProcessEnv; - tierName?: string | null; - webSearchConfig?: WebSearchConfig | null; - customPresetNames?: ReadonlySet | null; - } = {}, -): string[] { - const agentFilteredPresets = filterSetupPolicyPresetNamesForAgent(policyPresets, options.agent); - const effectiveHermesToolGateways = (options.hermesToolGateways ?? []).filter( - (name) => - !isStaleBuiltinWebSearchPolicyPreset(name, { - webSearchConfig: options.webSearchConfig, - customPresetNames: options.customPresetNames, - }), - ); - const mergedPresets = mergeRequiredOpenclawOtelPolicyPresets( - mergeEnabledMessagingChannelPolicyPresets( - mergeRequiredHermesToolGatewayPolicyPresets( - agentFilteredPresets, - effectiveHermesToolGateways, - options.knownPresetNames, - ), - options.enabledChannels, - options.knownPresetNames, - ), - { - agent: options.agent, - knownPresetNames: options.knownPresetNames, - env: options.env, - }, - ); - const agentScoped = filterSetupPolicyPresetNamesForAgent(mergedPresets, options.agent); - return filterSuppressedAgentRequiredPresets(agentScoped, options.tierName, options.agent); -} - -export function isStaleBuiltinBravePolicyPreset( - name: string, - options: { - webSearchConfig?: WebSearchConfig | null; - customPresetNames?: ReadonlySet | null; - } = {}, -): boolean { - return isStaleBuiltinWebSearchPolicyPreset(name, options); -} - -export function isStaleBuiltinWebSearchPolicyPreset( - name: string, - options: { - webSearchConfig?: WebSearchConfig | null; - customPresetNames?: ReadonlySet | null; - } = {}, -): boolean { - if (options.customPresetNames?.has(name)) return false; - if (name === "nous-web") { - return Boolean( - options.webSearchConfig && webSearchProviderForConfig(options.webSearchConfig) === "tavily", - ); - } - if (name !== "brave" && name !== "tavily") return false; - if (!options.webSearchConfig) return true; - return name !== webSearchProviderForConfig(options.webSearchConfig); -} - export function computeSetupPresetSuggestions( deps: { policies: PoliciesApi; @@ -190,6 +136,7 @@ export function computeSetupPresetSuggestions( webSearchConfig = null, provider = null, agent = null, + observabilityEnabled = false, env = process.env, } = options; const known = Array.isArray(options.knownPresetNames) ? new Set(options.knownPresetNames) : null; @@ -205,10 +152,29 @@ export function computeSetupPresetSuggestions( customPresetNames: options.customPresetNames, }), ) + .filter( + (name) => + !isInactiveObservabilityPolicyPreset(name, { + agent, + observabilityEnabled, + customPresetNames: options.customPresetNames, + customOwnsObservability: options.customOwnsObservability, + }), + ) .filter((name) => deps.policies.setupPolicyPresetSupported(name, supportOptions)) .filter((name) => !known || known.has(name)); const add = (name: string) => { if (!setupPolicyPresetAppliesToAgent(name, agent)) return; + if ( + isInactiveObservabilityPolicyPreset(name, { + agent, + observabilityEnabled, + customPresetNames: options.customPresetNames, + customOwnsObservability: options.customOwnsObservability, + }) + ) { + return; + } if ( isStaleBuiltinWebSearchPolicyPreset(name, { webSearchConfig, @@ -226,6 +192,9 @@ export function computeSetupPresetSuggestions( if (provider && deps.localInferenceProviders.includes(provider)) add("local-inference"); if (tierName !== RESTRICTED_TIER_NAME) { for (const preset of agentRequiredPresetAdditions(agent, env)) add(preset); + for (const preset of requiredObservabilityPolicyPresets(agent, observabilityEnabled)) { + add(preset); + } } if (tierName === "open" && typeof agent === "string" && agent.trim().toLowerCase() === "hermes") { for (const preset of allHermesToolGatewayPolicyPresets()) add(preset); @@ -272,6 +241,7 @@ async function setupPoliciesWithSelectionInner( const enabledChannels = Array.isArray(options.enabledChannels) ? options.enabledChannels : null; const provider = options.provider || null; const agent = options.agent || null; + const observabilityEnabled = options.observabilityEnabled === true; const hermesToolGateways = Array.isArray(options.hermesToolGateways) ? options.hermesToolGateways : null; @@ -290,7 +260,25 @@ async function setupPoliciesWithSelectionInner( const customPresetNames = new Set( deps.policies.listCustomPresets(sandboxName).map((preset) => preset.name), ); - const currentAppliedPresets = deps.policies.getAppliedPresets(sandboxName); + const customOwnsObservability = + deps.policies.customPresetOwnsNetworkPolicyKey?.( + sandboxName, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + ) === true; + if (customOwnsObservability) { + deps.policies.removeBuiltinPresetAttribution?.( + sandboxName, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, + ); + } + const rawCurrentAppliedPresets = deps.policies.getAppliedPresets(sandboxName); + const currentAppliedPresets = customOwnsObservability + ? [...new Set(rawCurrentAppliedPresets)].filter( + (name) => + name !== OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET || + customPresetNames.has(OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET), + ) + : rawCurrentAppliedPresets; const selectablePresets = [ ...allPresets, ...filterSetupPolicyPresetNamesForAgent(currentAppliedPresets, agent).map((name) => ({ @@ -303,14 +291,15 @@ async function setupPoliciesWithSelectionInner( supportOptions, customPresetNames, ); - const isStaleBuiltinWebSearch = (name: string) => - isStaleBuiltinWebSearchPolicyPreset(name, { webSearchConfig, customPresetNames }); - const appliedForPreservation = pruneDisabledMessagingPolicyPresets( - applied, + const pruneUnavailablePresets = createUnavailablePolicyPresetPruner({ disabledChannels, - ).filter((name) => !isStaleBuiltinWebSearch(name)); - const pruneDisabledPresets = (presetNames: string[]) => - pruneDisabledMessagingPolicyPresets(presetNames, disabledChannels); + agent, + observabilityEnabled, + webSearchConfig, + customPresetNames, + customOwnsObservability, + }); + const appliedForPreservation = pruneUnavailablePresets(applied); const filterSupportedPresetNames = (presetNames: string[]) => filterSetupPolicyPresetNamesForAgent(presetNames, agent).filter( (name) => @@ -326,24 +315,26 @@ async function setupPoliciesWithSelectionInner( customPresetNames, ) : null; - // Resume (selectedPresets !== null) keeps the recorded tier so stale - // suppressed presets from that tier still get filtered; fresh onboarding - // below uses the newly-selected `tierName` from `selectPolicyTier()`. - const recordedTierName = deps.getRecordedPolicyTier?.(sandboxName) ?? null; + // Resume keeps the recorded tier so stale suppressed presets from that tier + // still get filtered. An interrupted create can reach this fresh-selection + // branch before presets are recorded, so its persisted tier must also win + // over a new prompt or non-interactive default. + const recordedTierName = options.tierName ?? deps.getRecordedPolicyTier?.(sandboxName) ?? null; if (chosen !== null) { - chosen = chosen.filter((name) => !isStaleBuiltinWebSearch(name)); const knownSelectablePresets = new Set(selectablePresets.map((preset) => preset.name)); chosen = mergeRequiredSetupPolicyPresets(chosen, { enabledChannels, hermesToolGateways, agent, + observabilityEnabled, knownPresetNames: knownSelectablePresets, env: deps.env, tierName: recordedTierName, webSearchConfig, customPresetNames, + customOwnsObservability, }); - chosen = pruneDisabledPresets(chosen); + chosen = pruneUnavailablePresets(chosen); } if (selectedPresets !== null) { @@ -358,15 +349,17 @@ async function setupPoliciesWithSelectionInner( return resumeSelection; } - const tierName = await deps.selectPolicyTier(); + const tierName = recordedTierName ?? (await deps.selectPolicyTier()); deps.setPolicyTier?.(sandboxName, tierName); - const suggestions = pruneDisabledPresets( + const suggestions = pruneUnavailablePresets( computeSetupPresetSuggestions(deps, tierName, { enabledChannels, webSearchConfig, customPresetNames, + customOwnsObservability, provider, agent, + observabilityEnabled, knownPresetNames: allPresets.map((preset) => preset.name), webSearchSupported: options.webSearchSupported, hermesToolGateways, @@ -396,9 +389,7 @@ async function setupPoliciesWithSelectionInner( } else if (policyMode === "suggested" || policyMode === "default" || policyMode === "auto") { const envPresets = deps.parsePolicyPresetEnv(deps.env?.NEMOCLAW_POLICY_PRESETS || ""); if (envPresets.length > 0) { - chosen = filterSupportedPresetNames(envPresets).filter( - (name) => !isStaleBuiltinWebSearch(name), - ); + chosen = filterSupportedPresetNames(envPresets); } } else { console.warn(` Unsupported NEMOCLAW_POLICY_MODE: ${policyMode}`); @@ -417,13 +408,17 @@ async function setupPoliciesWithSelectionInner( enabledChannels, hermesToolGateways, agent, + observabilityEnabled, knownPresetNames: knownPresets, env: deps.env, tierName, webSearchConfig, customPresetNames, + customOwnsObservability, + }); + chosen = pruneUnavailablePresets(chosen, { + preserveExplicitWebSearch: isAuthoritative, }); - chosen = pruneDisabledPresets(chosen); const invalidPresets = chosen.filter((name) => !knownPresets.has(name)); if (invalidPresets.length > 0) { @@ -441,7 +436,6 @@ async function setupPoliciesWithSelectionInner( const kept: string[] = []; for (const name of appliedForPreservation) { if (chosenSet.has(name)) continue; - if (isStaleBuiltinWebSearch(name)) continue; if (suppressedNames.has(name)) continue; chosen.push(name); chosenSet.add(name); @@ -472,20 +466,23 @@ async function setupPoliciesWithSelectionInner( allPresets, extraSelected, ); - const interactiveChoice = pruneDisabledPresets( + const interactiveChoice = pruneUnavailablePresets( mergeRequiredSetupPolicyPresets( resolvedPresets.map((preset) => preset.name), { enabledChannels, hermesToolGateways, agent, + observabilityEnabled, knownPresetNames: knownNames, env: deps.env, tierName, webSearchConfig, customPresetNames, + customOwnsObservability, }, ), + { preserveExplicitWebSearch: true }, ); if (onSelection) onSelection(interactiveChoice); diff --git a/src/lib/onboard/policy-tier-suppression.ts b/src/lib/onboard/policy-tier-suppression.ts index 08d01f4be8..277dbda6a5 100644 --- a/src/lib/onboard/policy-tier-suppression.ts +++ b/src/lib/onboard/policy-tier-suppression.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { + isDcodeAgent, + OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, +} from "./observability-policy-presets"; import { isOpenclawAgent, OPENCLAW_OTEL_LOCAL_POLICY_PRESET, @@ -9,6 +13,11 @@ import { export const RESTRICTED_TIER_NAME = "restricted"; +export function normalizePolicyTierName(tierName: string | null | undefined): string | null { + if (typeof tierName !== "string") return null; + return tierName.trim().toLowerCase() || null; +} + export function agentRequiredPresetAdditions( agent: string | null | undefined, env: NodeJS.ProcessEnv, @@ -18,8 +27,11 @@ export function agentRequiredPresetAdditions( } function restrictedIncompatibleAgentRequiredPresets(agent: string | null | undefined): string[] { - if (!isOpenclawAgent(agent)) return []; - return ["openclaw-pricing", OPENCLAW_OTEL_LOCAL_POLICY_PRESET]; + if (isOpenclawAgent(agent)) { + return ["openclaw-pricing", OPENCLAW_OTEL_LOCAL_POLICY_PRESET]; + } + if (isDcodeAgent(agent)) return [OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET]; + return []; } /** @@ -82,7 +94,7 @@ export function suppressedAgentRequiredPresets( tierName: string, agent: string | null | undefined, ): string[] { - if (tierName !== RESTRICTED_TIER_NAME) return []; + if (normalizePolicyTierName(tierName) !== RESTRICTED_TIER_NAME) return []; return restrictedIncompatibleAgentRequiredPresets(agent); } diff --git a/src/lib/onboard/resume-config.test.ts b/src/lib/onboard/resume-config.test.ts index 67da110aea..0ecb8b8834 100644 --- a/src/lib/onboard/resume-config.test.ts +++ b/src/lib/onboard/resume-config.test.ts @@ -62,4 +62,16 @@ describe("authoritative rebuild resume config", () => { recorded: "invalid", }); }); + + it("allows explicit observability changes to reach sandbox drift reconciliation", () => { + const session = { + sandboxName: "demo", + provider: "nvidia-prod", + model: "test-model", + observabilityEnabled: true, + }; + + expect(getResumeConfigConflicts(session, {})).toEqual([]); + expect(getResumeConfigConflicts(session, { observabilityEnabled: false })).toEqual([]); + }); }); diff --git a/src/lib/onboard/resume-config.ts b/src/lib/onboard/resume-config.ts index cef8f5a038..48f428c5cb 100644 --- a/src/lib/onboard/resume-config.ts +++ b/src/lib/onboard/resume-config.ts @@ -14,6 +14,7 @@ export interface ResumeSessionLike { model?: string | null; agent?: string | null; toolDisclosure?: ToolDisclosure; + observabilityEnabled?: boolean; metadata?: { fromDockerfile?: string | null } | null; steps?: { sandbox?: { status?: string | null } | null } | null; } @@ -108,6 +109,7 @@ export function getResumeConfigConflicts( sandboxName?: string | null; agent?: string | null; toolDisclosure?: ToolDisclosure | null; + observabilityEnabled?: boolean | null; /** * Internal rebuild-resume mode: the caller already rewrote the session from * validated registry state, so credential aliases must not synthesize a new diff --git a/src/lib/onboard/runtime-control-flow.test.ts b/src/lib/onboard/runtime-control-flow.test.ts new file mode 100644 index 0000000000..c3e8b7b2ec --- /dev/null +++ b/src/lib/onboard/runtime-control-flow.test.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createSession } from "../state/onboard-session"; +import { + applyOnboardRuntimeControlRequests, + applySelectedAgentTransition, + updateSessionAgent, +} from "./runtime-control-flow"; + +afterEach(() => { + delete process.env.NEMOCLAW_TOOL_DISCLOSURE; +}); + +describe("onboard runtime control flow", () => { + it("normalizes explicit runtime control requests for session bootstrap", () => { + expect( + applyOnboardRuntimeControlRequests({ + toolDisclosure: "direct", + observabilityEnabled: true, + }), + ).toEqual({ + requestedToolDisclosure: "direct", + requestedObservabilityEnabled: true, + }); + delete process.env.NEMOCLAW_TOOL_DISCLOSURE; + expect(applyOnboardRuntimeControlRequests({})).toEqual({ + requestedToolDisclosure: null, + requestedObservabilityEnabled: null, + }); + }); + + it("keeps an authoritative inherited observability value out of explicit request handling", () => { + expect( + applyOnboardRuntimeControlRequests({ + observabilityEnabled: false, + observabilityRequestedExplicitly: false, + }), + ).toEqual({ + requestedToolDisclosure: null, + requestedObservabilityEnabled: null, + }); + }); + + it("records the selected DCode agent when observability is enabled", () => { + const session = createSession({ observabilityEnabled: true }); + + expect(updateSessionAgent(session, "langchain-deepagents-code")).toBe(session); + expect(session.agent).toBe("langchain-deepagents-code"); + }); + + it("rejects enabled observability for a non-DCode agent", () => { + const session = createSession({ + agent: "langchain-deepagents-code", + observabilityEnabled: true, + provider: "nvidia", + routerPid: 1234, + }); + const before = structuredClone(session); + const error = vi.fn(); + const exitProcess = vi.fn(() => { + throw new Error("exit 1"); + }); + + expect(() => updateSessionAgent(session, "openclaw", { error, exitProcess })).toThrow("exit 1"); + expect(error).toHaveBeenCalledWith( + " Recorded observability belongs to Deep Agents Code. Pass --no-observability explicitly when switching agents.", + ); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(session).toEqual(before); + }); + + it("rejects an invalid resumed agent transition before router or session mutation", async () => { + const session = createSession({ + agent: "langchain-deepagents-code", + observabilityEnabled: true, + provider: "nvidia", + routerPid: 1234, + }); + const before = structuredClone(session); + const stopTrackedModelRouterForAgentChange = vi.fn(async () => undefined); + const clearAgentScopedResumeState = vi.fn((current) => current); + const setOnboardBrandingAgent = vi.fn(); + const updateSession = vi.fn((mutator) => mutator(session) ?? session); + const note = vi.fn(); + const error = vi.fn(); + const exitProcess = vi.fn(() => { + throw new Error("exit 1"); + }); + + await expect( + applySelectedAgentTransition( + { + resume: true, + session, + selectedAgentName: "openclaw", + routerPort: 4000, + note, + }, + { + stopTrackedModelRouterForAgentChange, + clearAgentScopedResumeState, + setOnboardBrandingAgent, + updateSession, + error, + exitProcess, + }, + ), + ).rejects.toThrow("exit 1"); + + expect(stopTrackedModelRouterForAgentChange).not.toHaveBeenCalled(); + expect(clearAgentScopedResumeState).not.toHaveBeenCalled(); + expect(setOnboardBrandingAgent).not.toHaveBeenCalled(); + expect(updateSession).not.toHaveBeenCalled(); + expect(note).not.toHaveBeenCalled(); + expect(session).toEqual(before); + }); +}); diff --git a/src/lib/onboard/runtime-control-flow.ts b/src/lib/onboard/runtime-control-flow.ts new file mode 100644 index 0000000000..48dd0ca1be --- /dev/null +++ b/src/lib/onboard/runtime-control-flow.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type Session, updateSession } from "../state/onboard-session"; +import { clearAgentScopedResumeState } from "./agent-resume-state"; +import { setOnboardBrandingAgent } from "./branding"; +import { managedSandboxFeatureIssue } from "./managed-sandbox-feature"; +import { stopTrackedModelRouterForAgentChange } from "./model-router-process"; +import { DCODE_OBSERVABILITY_FEATURE } from "./observability-policy-presets"; +import { formatSandboxAgentName, normalizeSandboxAgentName } from "./sandbox-agent"; +import { applyOnboardToolDisclosureRequest } from "./tool-disclosure-flow"; +import type { OnboardOptions } from "./types"; + +export { clearAgentScopedResumeState }; + +export interface RuntimeControlAgentDeps { + error(message: string): void; + exitProcess(code: number): never; +} + +export interface SelectedAgentTransitionDeps extends RuntimeControlAgentDeps { + note(message: string): void; + stopTrackedModelRouterForAgentChange(session: Session, routerPort: number): Promise; + clearAgentScopedResumeState(session: Session, selectedAgentName: string): Session; + setOnboardBrandingAgent(agentName: string): void; + updateSession(mutator: (session: Session) => Session | void): Session; +} + +type SelectedAgentTransitionOverrides = Partial>; + +export function applyOnboardRuntimeControlRequests( + opts: Pick< + OnboardOptions, + "toolDisclosure" | "observabilityEnabled" | "observabilityRequestedExplicitly" + >, +) { + const observabilityIsExplicit = opts.observabilityRequestedExplicitly !== false; + return { + requestedToolDisclosure: applyOnboardToolDisclosureRequest(opts.toolDisclosure), + requestedObservabilityEnabled: + observabilityIsExplicit && typeof opts.observabilityEnabled === "boolean" + ? opts.observabilityEnabled + : null, + }; +} + +export function updateSessionAgent( + session: Session, + agentName: string | null | undefined, + deps: RuntimeControlAgentDeps = { + error: console.error, + exitProcess: (code) => process.exit(code), + }, +): Session { + validateSessionAgentObservability(session, agentName, deps); + session.agent = agentName ?? null; + return session; +} + +export function validateSessionAgentObservability( + session: Pick | null, + agentName: string | null | undefined, + deps: RuntimeControlAgentDeps = { + error: console.error, + exitProcess: (code) => process.exit(code), + }, +): void { + if ( + managedSandboxFeatureIssue(DCODE_OBSERVABILITY_FEATURE, { + agent: agentName, + sessionValue: session?.observabilityEnabled, + }) === "recorded-state-on-unsupported-agent" + ) { + deps.error( + " Recorded observability belongs to Deep Agents Code. Pass --no-observability explicitly when switching agents.", + ); + deps.exitProcess(1); + } +} + +export async function applySelectedAgentTransition( + input: { + resume: boolean; + session: Session | null; + selectedAgentName: string | null | undefined; + routerPort: number; + note(message: string): void; + }, + overrides: SelectedAgentTransitionOverrides = {}, +): Promise<{ session: Session; resumeAgentChanged: boolean }> { + const deps: SelectedAgentTransitionDeps = { + note: input.note, + stopTrackedModelRouterForAgentChange, + clearAgentScopedResumeState, + setOnboardBrandingAgent, + updateSession, + error: console.error, + exitProcess: (code) => process.exit(code), + ...overrides, + }; + validateSessionAgentObservability(input.session, input.selectedAgentName, deps); + + const selectedAgentName = normalizeSandboxAgentName(input.selectedAgentName); + const recordedAgentName = normalizeSandboxAgentName(input.session?.agent); + const resumeAgentChanged = Boolean( + input.resume && input.session && recordedAgentName !== selectedAgentName, + ); + if (resumeAgentChanged && input.session) { + deps.note( + ` Agent changed from ${formatSandboxAgentName(recordedAgentName)} to ${formatSandboxAgentName(selectedAgentName)}; refreshing provider selection.`, + ); + await deps.stopTrackedModelRouterForAgentChange(input.session, input.routerPort); + deps.updateSession((current) => deps.clearAgentScopedResumeState(current, selectedAgentName)); + } + deps.setOnboardBrandingAgent(input.selectedAgentName || "openclaw"); + const session = deps.updateSession((current) => + updateSessionAgent(current, input.selectedAgentName, deps), + ); + return { session, resumeAgentChanged }; +} diff --git a/src/lib/onboard/sandbox-create-launch-observability.test.ts b/src/lib/onboard/sandbox-create-launch-observability.test.ts new file mode 100644 index 0000000000..1419c169e0 --- /dev/null +++ b/src/lib/onboard/sandbox-create-launch-observability.test.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; + +const disabledHermesDashboardState = { config: null, enabled: false }; + +describe("prepareSandboxCreateLaunch observability", () => { + it("forwards only the backend-neutral observability enable bit to Deep Agents Code", () => { + const result = prepareSandboxCreateLaunch({ + agent: { name: "langchain-deepagents-code" } as any, + observabilityEnabled: true, + chatUiUrl: "", + createArgs: ["--name", "dcode-demo"], + sandboxName: "dcode-demo", + env: { + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example/v1/traces", + OTEL_EXPORTER_OTLP_HEADERS: "authorization=secret", + LANGSMITH_API_KEY: "must-not-enter", + }, + extraPlaceholderKeys: [], + getDashboardForwardPort: vi.fn(() => "0"), + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + buildEnv: () => ({}), + }); + + expect(result.envArgs).toContain("NEMOCLAW_OBSERVABILITY=1"); + const serialized = result.envArgs.join("\n"); + expect(serialized).not.toContain("collector.example"); + expect(serialized).not.toContain("OTEL_EXPORTER_OTLP_HEADERS"); + expect(serialized).not.toContain("LANGSMITH"); + expect(serialized).not.toContain("secret"); + }); + + it("does not forward observability for another agent or when disabled", () => { + const render = (name: string, observabilityEnabled: boolean) => + prepareSandboxCreateLaunch({ + agent: { name } as any, + observabilityEnabled, + chatUiUrl: "", + createArgs: [], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: vi.fn(() => "0"), + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + buildEnv: () => ({}), + }); + + expect(render("langchain-deepagents-code", false).envArgs).not.toContain( + "NEMOCLAW_OBSERVABILITY=1", + ); + expect(render("hermes", true).envArgs).not.toContain("NEMOCLAW_OBSERVABILITY=1"); + }); +}); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 8337eeae5e..f809c75734 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -42,6 +42,7 @@ function appendOpenClawAutoPairRuntimeEnvArgs( export interface SandboxCreateLaunchInput { agent: AgentDefinition | null | undefined; + observabilityEnabled?: boolean; chatUiUrl: string; createArgs: readonly string[]; sandboxName?: string; @@ -117,6 +118,9 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San if (sandboxName) { envArgs.push(formatEnvAssignment("NEMOCLAW_SANDBOX_NAME", sandboxName)); } + if (input.observabilityEnabled === true) { + envArgs.push(formatEnvAssignment("NEMOCLAW_OBSERVABILITY", "1")); + } } appendExtraPlaceholderKeysEnvArg(envArgs, input.extraPlaceholderKeys, formatEnvAssignment); diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 838149207c..3a135c25e6 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { MessagingTokenDef } from "./messaging-prep"; import { materializeSandboxCreatePlan, @@ -16,6 +16,10 @@ const sandboxGpuConfig: SandboxGpuCreateConfig = { sandboxGpuDevice: "nvidia.com/gpu=0", }; +afterEach(() => { + vi.unstubAllEnvs(); +}); + const channels = [ { name: "telegram", @@ -327,6 +331,8 @@ describe("resolveSandboxCreateIntent", () => { describe("prepareSandboxCreatePlan", () => { it("builds create args, policy, providers, and active channels in onboard order", () => { + vi.stubEnv("NEMOCLAW_NON_INTERACTIVE", "1"); + vi.stubEnv("NEMOCLAW_POLICY_TIER", "restricted"); const events: string[] = []; const appendResourceFlags = vi.fn((args: string[]) => { events.push("resources"); @@ -402,9 +408,10 @@ describe("prepareSandboxCreatePlan", () => { dockerGpuPatch: false, additionalPresets: ["github"], agentName: "langchain-deepagents-code", - policyTier: null, + policyTier: "restricted", }, ); + expect(result.policyTier).toBe("restricted"); expect(result.createArgs).toEqual([ "--from", "/tmp/nemoclaw-build-1/Dockerfile", diff --git a/src/lib/onboard/sandbox-create-plan.ts b/src/lib/onboard/sandbox-create-plan.ts index 71b611c279..84ec37405f 100644 --- a/src/lib/onboard/sandbox-create-plan.ts +++ b/src/lib/onboard/sandbox-create-plan.ts @@ -89,6 +89,8 @@ export type PrepareSandboxCreatePlanInput = { export type SandboxCreatePlan = { activeMessagingChannels: string[]; initialSandboxPolicy: InitialSandboxPolicy; + /** Tier resolved before create, persisted with the registry entry for safe resume. */ + policyTier: string | null; createArgs: string[]; messagingProviders: string[]; useDockerGpuPatch: boolean; @@ -390,6 +392,7 @@ export function materializeSandboxCreatePlan({ return { activeMessagingChannels: [...intent.activeMessagingChannels], initialSandboxPolicy, + policyTier: intent.policy.options.policyTier, createArgs, messagingProviders, useDockerGpuPatch: intent.useDockerGpuPatch, diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index f9c381260c..579aff39af 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -43,6 +43,8 @@ describe("buildCreatedSandboxRegistryEntry", () => { agentVersionKnown: true, imageTag: "nemoclaw-demo:123", appliedPolicies: ["discord", "slack"], + observabilityEnabled: true, + policyTier: "restricted", webSearchEnabled: true, fromDockerfile: "/tmp/Dockerfile.custom", hermesAuthMethod: "api_key", @@ -67,6 +69,8 @@ describe("buildCreatedSandboxRegistryEntry", () => { imageTag: "nemoclaw-demo:123", policies: ["discord", "slack"], toolDisclosure: "progressive", + observabilityEnabled: true, + policyTier: "restricted", webSearchEnabled: true, fromDockerfile: "/tmp/Dockerfile.custom", hermesAuthMethod: "api_key", @@ -142,6 +146,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { expect(entry.fromDockerfile).toBeNull(); expect(entry.hermesAuthMethod).toBeNull(); expect(entry.toolDisclosure).toBe("progressive"); + expect(entry.observabilityEnabled).toBe(false); }); it("carries a durable MCP rebuild manifest into the replacement registry entry", () => { diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index b807f08976..52cb7f48e9 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -36,6 +36,8 @@ export interface CreatedSandboxRegistryEntryInput { imageTag: string | null; appliedPolicies: string[]; toolDisclosure?: ToolDisclosure; + observabilityEnabled?: boolean; + policyTier?: SandboxEntry["policyTier"]; webSearchEnabled?: boolean; webSearchProvider?: SandboxEntry["webSearchProvider"]; fromDockerfile?: string | null; @@ -113,6 +115,8 @@ export function buildCreatedSandboxRegistryEntry( imageTag: input.imageTag, policies: input.appliedPolicies, toolDisclosure: input.toolDisclosure ?? DEFAULT_TOOL_DISCLOSURE, + observabilityEnabled: input.observabilityEnabled === true, + ...(input.policyTier !== undefined ? { policyTier: input.policyTier } : {}), webSearchEnabled: input.webSearchEnabled === true, webSearchProvider: input.webSearchEnabled === true ? (input.webSearchProvider ?? "brave") : null, diff --git a/src/lib/onboard/session-bootstrap.test.ts b/src/lib/onboard/session-bootstrap.test.ts index 00c9b1877e..13b3a3c459 100644 --- a/src/lib/onboard/session-bootstrap.test.ts +++ b/src/lib/onboard/session-bootstrap.test.ts @@ -72,6 +72,7 @@ describe("prepareOnboardSession", () => { cannotPrompt: false, nonInteractive: true, requestedToolDisclosure: "direct", + requestedObservabilityEnabled: true, }, deps, ); @@ -81,6 +82,8 @@ describe("prepareOnboardSession", () => { expect(result.session?.mode).toBe("non-interactive"); expect(result.session?.metadata.fromDockerfile).toBe("/abs/Dockerfile.custom"); expect(result.session?.toolDisclosure).toBe("direct"); + expect(result.session?.observabilityEnabled).toBe(true); + expect(result.session?.observabilityRequestedExplicitly).toBe(true); expect(getSession()?.sessionId).not.toBe("old-session"); }); @@ -98,6 +101,8 @@ describe("prepareOnboardSession", () => { deps, ); expect(result.session?.toolDisclosure).toBe("progressive"); + expect(result.session?.observabilityEnabled).toBe(false); + expect(result.session?.observabilityRequestedExplicitly).toBe(false); }); it("resumes an existing session and falls back to the recorded Dockerfile", async () => { @@ -111,6 +116,8 @@ describe("prepareOnboardSession", () => { metadata: { gatewayName: "nemoclaw", fromDockerfile: "Dockerfile.recorded" }, sandboxName: "demo", status: "failed", + observabilityEnabled: true, + observabilityRequestedExplicitly: true, steps: { ...createSession().steps, sandbox: completeSandboxStep(), @@ -135,10 +142,44 @@ describe("prepareOnboardSession", () => { expect(result.session?.mode).toBe("non-interactive"); expect(result.session?.failure).toBeNull(); expect(result.session?.status).toBe("in_progress"); + expect(result.session?.observabilityEnabled).toBe(true); + expect(result.session?.observabilityRequestedExplicitly).toBe(true); expect(deps.repairResumeMachineSnapshot).toHaveBeenCalledWith(initial); expect(deps.setOnboardBrandingAgent).toHaveBeenCalledWith("hermes"); }); + it.each([ + { recorded: true, requested: false }, + { recorded: false, requested: true }, + ])("records an explicit observability request while resuming", async ({ + recorded, + requested, + }) => { + const { deps } = createDeps( + createSession({ + sandboxName: "demo", + observabilityEnabled: recorded, + status: "failed", + }), + ); + + const result = await prepareOnboardSession( + { + resume: true, + fresh: false, + requestedFromDockerfile: null, + requestedSandboxName: null, + cannotPrompt: false, + nonInteractive: false, + requestedObservabilityEnabled: requested, + }, + deps, + ); + + expect(result.session?.observabilityEnabled).toBe(requested); + expect(result.session?.observabilityRequestedExplicitly).toBe(true); + }); + it("records and reports resume conflicts before exiting", async () => { const conflict: ResumeConfigConflict = { field: "fromDockerfile", diff --git a/src/lib/onboard/session-bootstrap.ts b/src/lib/onboard/session-bootstrap.ts index 14c1b0b7cf..ebe062358a 100644 --- a/src/lib/onboard/session-bootstrap.ts +++ b/src/lib/onboard/session-bootstrap.ts @@ -16,6 +16,7 @@ export interface OnboardSessionBootstrapInput { agentFlag?: string | null; envAgent?: string | null; requestedToolDisclosure?: ToolDisclosure | null; + requestedObservabilityEnabled?: boolean | null; } export interface OnboardSessionBootstrapDeps { @@ -34,6 +35,7 @@ export interface OnboardSessionBootstrapDeps { sandboxName?: string | null; agent?: string | null; toolDisclosure?: ToolDisclosure | null; + observabilityEnabled?: boolean | null; authoritativeResumeConfig?: boolean; }, ): ResumeConfigConflict[]; @@ -158,6 +160,7 @@ async function prepareResumeSession( sandboxName: input.requestedSandboxName, agent: input.agentFlag || null, toolDisclosure: input.requestedToolDisclosure ?? null, + observabilityEnabled: input.requestedObservabilityEnabled ?? null, authoritativeResumeConfig: input.authoritativeResumeConfig, }); if (resumeConflicts.length > 0) { @@ -166,6 +169,10 @@ async function prepareResumeSession( deps.updateSession((current: Session) => { deps.repairResumeMachineSnapshot(current); + if (typeof input.requestedObservabilityEnabled === "boolean") { + current.observabilityEnabled = input.requestedObservabilityEnabled; + current.observabilityRequestedExplicitly = true; + } current.mode = mode(input.nonInteractive); current.failure = null; current.status = "in_progress"; @@ -190,6 +197,8 @@ function prepareFreshSession( deps.createSession({ mode: mode(input.nonInteractive), toolDisclosure: input.requestedToolDisclosure ?? DEFAULT_TOOL_DISCLOSURE, + observabilityEnabled: input.requestedObservabilityEnabled === true, + observabilityRequestedExplicitly: typeof input.requestedObservabilityEnabled === "boolean", metadata: { gatewayName: "nemoclaw", fromDockerfile: fromDockerfile || null }, }), ); diff --git a/src/lib/onboard/session-updates.ts b/src/lib/onboard/session-updates.ts index c122224025..3838fa3cc8 100644 --- a/src/lib/onboard/session-updates.ts +++ b/src/lib/onboard/session-updates.ts @@ -18,6 +18,7 @@ export interface OnboardSessionUpdateInput { nimContainer?: string | null; webSearchConfig?: WebSearchConfig | null; toolDisclosure?: ToolDisclosure | string; + observabilityEnabled?: boolean; policyPresets?: string[] | null; messagingPlan?: SandboxMessagingPlan | null; hermesToolGateways?: string[] | null; @@ -60,6 +61,9 @@ export function toSessionUpdates(updates: OnboardSessionUpdateInput = {}): Sessi const toolDisclosure = normalizeToolDisclosure(updates.toolDisclosure); if (toolDisclosure) normalized.toolDisclosure = toolDisclosure; } + if (typeof updates.observabilityEnabled === "boolean") { + normalized.observabilityEnabled = updates.observabilityEnabled; + } if (updates.policyPresets !== undefined) normalized.policyPresets = updates.policyPresets; if (updates.messagingPlan !== undefined) normalized.messagingPlan = updates.messagingPlan; if (updates.hermesToolGateways !== undefined) diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index 14cc10132b..5c066707bc 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -56,6 +56,11 @@ export type ModelValidationResult = ModelValidationSuccess | ModelValidationFail export interface SandboxCreateIntent { readonly recreate: boolean; readonly toolDisclosure: import("../tool-disclosure").ToolDisclosure; + readonly observabilityEnabled: boolean; + /** Present only when the operator explicitly selected observability on or off. */ + readonly observabilityRequestedExplicitly?: true; + /** Internal authoritative rebuild tier used before replacement registration completes. */ + readonly policyTier?: string | null; } export type OnboardOptions = { @@ -85,6 +90,11 @@ export type OnboardOptions = { acceptThirdPartySoftware?: boolean; agent?: string | null; toolDisclosure?: import("../tool-disclosure").ToolDisclosure | null; + observabilityEnabled?: boolean | null; + /** Internal provenance for an authoritative observability value. */ + observabilityRequestedExplicitly?: boolean; + /** Internal authoritative rebuild tier; never exposed as an onboard CLI option. */ + policyTier?: string | null; controlUiPort?: number | null; gpu?: boolean; noGpu?: boolean; diff --git a/src/lib/policy/custom-preset-ownership.test.ts b/src/lib/policy/custom-preset-ownership.test.ts new file mode 100644 index 0000000000..f4c73879a4 --- /dev/null +++ b/src/lib/policy/custom-preset-ownership.test.ts @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { getCustomPolicies, runCapture } = vi.hoisted(() => ({ + getCustomPolicies: vi.fn(), + runCapture: vi.fn(), +})); + +vi.mock("../runner", async (importOriginal) => ({ + ...(await importOriginal()), + runCapture, +})); + +vi.mock("../state/registry", () => ({ getCustomPolicies })); + +import { customPresetOwnsNetworkPolicyKey } from "./index"; + +const MATCHING_POLICY = `version: 1 +network_policies: + shared-otel: + endpoints: + - host: collector.internal + port: 4318 +`; + +const DRIFTED_PRESET = `preset: + name: drifted +network_policies: + shared-otel: + endpoints: + - host: stale.internal + port: 4318 +`; + +const MATCHING_PRESET = `preset: + name: matching +network_policies: + shared-otel: + endpoints: + - host: collector.internal + port: 4318 +`; + +const MATCHING_KEY_WITH_DRIFTED_SIBLING = `preset: + name: matching-with-sibling +network_policies: + shared-otel: + endpoints: + - host: collector.internal + port: 4318 + unrelated-policy: + endpoints: + - host: stale.internal + port: 443 +`; + +describe("customPresetOwnsNetworkPolicyKey", () => { + beforeEach(() => { + getCustomPolicies.mockReset(); + runCapture.mockReset(); + }); + + it("compares two matching-key candidates against one live policy read (#3915)", () => { + getCustomPolicies.mockReturnValue([ + { name: "drifted", content: DRIFTED_PRESET }, + { name: "matching", content: MATCHING_PRESET }, + ]); + runCapture.mockReturnValue(MATCHING_POLICY); + + expect(customPresetOwnsNetworkPolicyKey("my-sandbox", "shared-otel")).toBe(true); + expect(runCapture).toHaveBeenCalledOnce(); + expect(runCapture.mock.calls[0]?.[0]?.slice(1)).toEqual([ + "policy", + "get", + "--base", + "my-sandbox", + ]); + }); + + it("compares only the requested key when another key in the custom preset drifts", () => { + getCustomPolicies.mockReturnValue([ + { name: "matching-with-sibling", content: MATCHING_KEY_WITH_DRIFTED_SIBLING }, + ]); + runCapture.mockReturnValue( + `${MATCHING_POLICY} unrelated-policy:\n endpoints:\n - host: live.internal\n port: 443\n`, + ); + + expect(customPresetOwnsNetworkPolicyKey("my-sandbox", "shared-otel")).toBe(true); + expect(runCapture).toHaveBeenCalledOnce(); + }); + + it("does not read live policy when no custom candidate owns the key (#3915)", () => { + getCustomPolicies.mockReturnValue([ + { + name: "unrelated", + content: "network_policies:\n unrelated:\n endpoints: []\n", + }, + ]); + + expect(customPresetOwnsNetworkPolicyKey("my-sandbox", "shared-otel")).toBe(false); + expect(runCapture).not.toHaveBeenCalled(); + }); + + it("aborts before mutation when registered custom ownership content is malformed", () => { + getCustomPolicies.mockReturnValue([ + { + name: "corrupt-otel", + content: "network_policies:\n shared-otel: [unterminated", + }, + ]); + + expect(() => customPresetOwnsNetworkPolicyKey("my-sandbox", "shared-otel")).toThrow( + /Could not inspect registered custom policy ownership.*refusing to reconcile/, + ); + expect(runCapture).not.toHaveBeenCalled(); + }); + + it("aborts reconciliation when the single live policy read fails (#3915)", () => { + getCustomPolicies.mockReturnValue([{ name: "matching", content: MATCHING_PRESET }]); + runCapture.mockImplementation(() => { + throw new Error("gateway unavailable"); + }); + + expect(() => customPresetOwnsNetworkPolicyKey("my-sandbox", "shared-otel")).toThrow( + /Could not read live policy ownership.*refusing to reconcile/, + ); + expect(runCapture).toHaveBeenCalledOnce(); + }); + + it("aborts reconciliation when the live policy response is indeterminate", () => { + getCustomPolicies.mockReturnValue([{ name: "matching", content: MATCHING_PRESET }]); + runCapture.mockReturnValue("version: [invalid"); + + expect(() => customPresetOwnsNetworkPolicyKey("my-sandbox", "shared-otel")).toThrow( + /Could not determine live policy ownership.*refusing to reconcile/, + ); + expect(runCapture).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/policy/gateway-state.ts b/src/lib/policy/gateway-state.ts index e56b2df9f7..202d54ffbe 100644 --- a/src/lib/policy/gateway-state.ts +++ b/src/lib/policy/gateway-state.ts @@ -62,7 +62,7 @@ export function inspectGatewayPresetNames( } export function inspectPresetContentGatewayState( - options: GatewayInspectionOptions & { presetContent: string }, + options: GatewayInspectionOptions & { presetContent: string; policyKey?: string }, ): PresetContentGatewayState { const parsed = readParsedPolicy(options); if (!parsed) return null; @@ -83,8 +83,12 @@ export function inspectPresetContentGatewayState( } const currentPolicies = current as Record; const expectedPolicies = expected as Record; - const expectedKeys = Object.keys(expectedPolicies); + const expectedKeys = + options.policyKey === undefined ? Object.keys(expectedPolicies) : [options.policyKey]; if (expectedKeys.length === 0) return "drift"; + if (options.policyKey !== undefined && !Object.hasOwn(expectedPolicies, options.policyKey)) { + return "drift"; + } const presentKeys = expectedKeys.filter((key) => Object.hasOwn(currentPolicies, key)); if (presentKeys.length === 0) return "absent"; if (presentKeys.length !== expectedKeys.length) return "drift"; diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index 90b8938f31..4ab9855a76 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -178,6 +178,12 @@ function parsePresetPolicyKeys(presetContent: string | null | undefined): string return Object.keys(parseNetworkPolicies(`network_policies:\n${presetEntries}`) || {}); } +/** Preserve invalid registered content as indeterminate for ownership decisions. */ +function parsePresetPolicyKeysForOwnership(presetContent: string): string[] | null { + const networkPolicies = parseNetworkPolicies(presetContent); + return networkPolicies === null ? null : Object.keys(networkPolicies); +} + const AGENT_PRESET_KEY_ALIASES: Readonly> = getMessagingPolicyKeyAliases(); @@ -1253,6 +1259,60 @@ function listCustomPresets(sandboxName: string): PresetInfo[] { })); } +/** Return whether registered custom content owns an exact live network-policy key. */ +function customPresetOwnsNetworkPolicyKey(sandboxName: string, policyKey: string): boolean { + let candidates: ReturnType; + try { + candidates = []; + for (const entry of registry.getCustomPolicies(sandboxName)) { + const keys = parsePresetPolicyKeysForOwnership(entry.content); + if (keys === null) { + throw new Error("invalid registered custom policy content"); + } + if (keys.includes(policyKey)) candidates.push(entry); + } + } catch { + throw new Error( + `Could not inspect registered custom policy ownership for '${policyKey}' in sandbox '${sandboxName}'; refusing to reconcile overlapping built-in policy content.`, + ); + } + if (candidates.length === 0) return false; + + let rawPolicy: string; + try { + rawPolicy = runCapture(buildPolicyGetCommand(sandboxName)); + } catch { + throw new Error( + `Could not read live policy ownership for '${policyKey}' in sandbox '${sandboxName}'; refusing to reconcile overlapping built-in policy content.`, + ); + } + const states = candidates.map((entry) => + inspectPresetContentGatewayState({ + readPolicy: () => rawPolicy, + parseCurrentPolicy: parseCurrentPolicyOrEmpty, + extractPresetEntries, + presetContent: entry.content, + policyKey, + }), + ); + if (states.includes("match")) return true; + if (states.includes(null)) { + throw new Error( + `Could not determine live policy ownership for '${policyKey}' in sandbox '${sandboxName}'; refusing to reconcile overlapping built-in policy content.`, + ); + } + return false; +} + +/** Drop built-in registry attribution without mutating overlapping live policy content. */ +function removeBuiltinPresetAttribution(sandboxName: string, presetName: string): void { + const sandbox = registry.getSandbox(sandboxName); + if (!sandbox) return; + const policies = (sandbox.policies ?? []).filter((name) => name !== presetName); + if (policies.length === (sandbox.policies ?? []).length) return; + registry.updateSandbox(sandboxName, { policies }); +} + /** * Query the gateway for the currently loaded policy and determine which * presets are actually enforced by matching network_policies entries @@ -1296,12 +1356,14 @@ function getGatewayPresets(sandboxName: string): string[] | null { function getPresetContentGatewayState( sandboxName: string, presetContent: string, + policyKey?: string, ): "match" | "absent" | "drift" | null { return inspectPresetContentGatewayState({ readPolicy: () => runCapture(buildPolicyGetCommand(sandboxName)), parseCurrentPolicy: parseCurrentPolicyOrEmpty, extractPresetEntries, presetContent, + policyKey, }); } @@ -1428,6 +1490,7 @@ export { buildPolicyGetFullCommand, buildPolicySetCommand, clampSetupPolicyPresetNames, + customPresetOwnsNetworkPolicyKey, extractPresetEntries, filterSetupPolicyPresets, getAppliedPresets, @@ -1450,6 +1513,7 @@ export { parseCurrentPolicyOrEmpty as parseCurrentPolicy, parsePresetPolicyKeys, presetContentMatchesGateway, + removeBuiltinPresetAttribution, removePreset, removePresetFromPolicy, resolvePermissivePolicyPath, diff --git a/src/lib/policy/managed-policy-binding.test.ts b/src/lib/policy/managed-policy-binding.test.ts new file mode 100644 index 0000000000..2165b5a4f2 --- /dev/null +++ b/src/lib/policy/managed-policy-binding.test.ts @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + ManagedPolicyBinding, + type ManagedPolicyBindingRuntime, + type ManagedPolicyContentState, +} from "./managed-policy-binding"; + +const CONTENT = "network_policies:\n managed-key:\n name: managed-key\n"; + +function runtime(states: ManagedPolicyContentState[] = ["match", "absent"]) { + const stateQueue = [...states]; + return { + getPresetContentGatewayState: vi.fn(() => stateQueue.shift() ?? null), + loadPresetForSandbox: vi.fn(() => CONTENT), + removePreset: vi.fn(() => true), + } as unknown as ManagedPolicyBindingRuntime; +} + +describe("managed policy binding", () => { + const binding = new ManagedPolicyBinding({ + presetName: "managed-preset", + policyKey: "managed-key", + }); + + it("normalizes preset identity and registry attribution", () => { + expect(binding.matchesPreset(" Managed-Preset ")).toBe(true); + expect(binding.setAttribution(["npm", "MANAGED-PRESET"], true)).toEqual([ + "npm", + "managed-preset", + ]); + expect(binding.setAttribution(["npm", "managed-preset"], false)).toEqual(["npm"]); + }); + + it("requires matching policy keys and exact live content for custom ownership", () => { + const deps = runtime(["drift", "match"]); + expect(binding.hasLiveCustomOwner("alpha", [CONTENT, CONTENT], deps)).toBe(true); + expect(deps.getPresetContentGatewayState).toHaveBeenNthCalledWith( + 1, + "alpha", + CONTENT, + "managed-key", + ); + expect(binding.hasLiveCustomOwner("alpha", ["network_policies:\n other: {}\n"], deps)).toBe( + false, + ); + }); + + it("aborts managed reconciliation when custom ownership is indeterminate", () => { + const deps = runtime([null]); + expect(() => binding.hasLiveCustomOwner("alpha", [CONTENT], deps)).toThrow( + /Could not determine live policy ownership.*refusing to reconcile/, + ); + }); + + it("aborts before inspection when registered custom content is malformed", () => { + const deps = runtime(); + expect(() => + binding.hasLiveCustomOwner("alpha", ["network_policies:\n managed-key: [invalid"], deps), + ).toThrow(/Could not determine live policy ownership.*refusing to reconcile/); + expect(deps.getPresetContentGatewayState).not.toHaveBeenCalled(); + }); + + it("loads and inspects managed content without exposing policy read failures", () => { + const deps = runtime(["match"]); + expect(binding.load("alpha", deps)).toEqual({ content: CONTENT, state: "match" }); + vi.mocked(deps.loadPresetForSandbox).mockImplementation(() => { + throw new Error("gateway unavailable"); + }); + expect(binding.load("alpha", deps)).toEqual({ content: null, state: null }); + }); + + it("removes only exact managed content and verifies absence afterward", () => { + const deps = runtime(["match", "absent"]); + expect(binding.removeExact("alpha", CONTENT, deps)).toMatchObject({ + before: "match", + after: "absent", + attempted: true, + reportedSuccess: true, + failureDetail: null, + verifiedAbsent: true, + }); + expect(deps.removePreset).toHaveBeenCalledWith("alpha", "managed-preset"); + }); + + it("retains an actionable failure when removal cannot prove absence", () => { + const deps = runtime(["match", "drift"]); + vi.mocked(deps.removePreset).mockReturnValue(false); + expect(binding.removeExact("alpha", CONTENT, deps)).toMatchObject({ + after: "drift", + reportedSuccess: false, + failureDetail: "remove failed; post-remove content drifted", + verifiedAbsent: false, + }); + }); +}); diff --git a/src/lib/policy/managed-policy-binding.ts b/src/lib/policy/managed-policy-binding.ts new file mode 100644 index 0000000000..9507f9f329 --- /dev/null +++ b/src/lib/policy/managed-policy-binding.ts @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { parseNetworkPolicies } from "./preset-parsing"; + +export type ManagedPolicyContentState = "match" | "absent" | "drift" | null; + +export type ManagedPolicyBindingRuntime = Pick< + typeof import("./index"), + "getPresetContentGatewayState" | "loadPresetForSandbox" | "removePreset" +>; + +export interface ManagedPolicyBindingRemovalInput { + knownBefore?: ManagedPolicyContentState; + removeOptions?: Parameters[2]; +} + +export interface ManagedPolicyBindingRemovalResult { + before: ManagedPolicyContentState; + after: ManagedPolicyContentState; + attempted: boolean; + reportedSuccess: boolean | null; + errorMessage: string | null; + failureDetail: string | null; + verifiedAbsent: boolean; +} + +/** Exact-content ownership and removal contract for one NemoClaw-managed policy preset. */ +export class ManagedPolicyBinding { + readonly presetName: string; + readonly policyKey: string; + + constructor(input: { presetName: string; policyKey?: string }) { + this.presetName = input.presetName.trim().toLowerCase(); + this.policyKey = (input.policyKey ?? input.presetName).trim().toLowerCase(); + } + + matchesPreset(name: string): boolean { + return name.trim().toLowerCase() === this.presetName; + } + + private contentOwnershipState(content: string): boolean | null { + try { + const policies = parseNetworkPolicies(content); + return policies === null + ? null + : Object.prototype.hasOwnProperty.call(policies, this.policyKey); + } catch { + return null; + } + } + + ownsContent(content: string): boolean { + return this.contentOwnershipState(content) === true; + } + + inspectContent( + sandboxName: string, + content: string, + runtime: ManagedPolicyBindingRuntime, + policyKey?: string, + ): ManagedPolicyContentState { + try { + return policyKey === undefined + ? runtime.getPresetContentGatewayState(sandboxName, content) + : runtime.getPresetContentGatewayState(sandboxName, content, policyKey); + } catch { + return null; + } + } + + load( + sandboxName: string, + runtime: ManagedPolicyBindingRuntime, + ): { content: string | null; state: ManagedPolicyContentState } { + let content: string | null = null; + try { + content = runtime.loadPresetForSandbox(sandboxName, this.presetName); + } catch { + content = null; + } + return { + content, + state: content ? this.inspectContent(sandboxName, content, runtime) : null, + }; + } + + hasLiveCustomOwner( + sandboxName: string, + contents: readonly string[], + runtime: ManagedPolicyBindingRuntime, + ): boolean { + let indeterminate = false; + for (const content of contents) { + const ownsContent = this.contentOwnershipState(content); + if (ownsContent === null) { + indeterminate = true; + continue; + } + if (!ownsContent) continue; + const state = this.inspectContent(sandboxName, content, runtime, this.policyKey); + if (state === "match") return true; + if (state === null) indeterminate = true; + } + if (indeterminate) { + throw new Error( + `Could not determine live policy ownership for '${this.policyKey}' in sandbox '${sandboxName}'; refusing to reconcile overlapping managed policy content.`, + ); + } + return false; + } + + setAttribution(names: readonly string[], enabled: boolean): string[] { + const withoutBinding = names.filter((name) => !this.matchesPreset(name)); + return enabled ? [...withoutBinding, this.presetName] : withoutBinding; + } + + removeExact( + sandboxName: string, + content: string, + runtime: ManagedPolicyBindingRuntime, + input: ManagedPolicyBindingRemovalInput = {}, + ): ManagedPolicyBindingRemovalResult { + const before = + input.knownBefore === undefined + ? this.inspectContent(sandboxName, content, runtime) + : input.knownBefore; + if (before !== "match") { + return { + before, + after: before, + attempted: false, + reportedSuccess: null, + errorMessage: null, + failureDetail: null, + verifiedAbsent: before === "absent", + }; + } + + let reportedSuccess = false; + let errorMessage: string | null = null; + try { + reportedSuccess = + input.removeOptions === undefined + ? runtime.removePreset(sandboxName, this.presetName) + : runtime.removePreset(sandboxName, this.presetName, input.removeOptions); + } catch (error) { + errorMessage = error instanceof Error ? error.message : String(error); + } + const after = this.inspectContent(sandboxName, content, runtime); + const mutationFailure = errorMessage + ? `remove: ${errorMessage}` + : reportedSuccess + ? null + : "remove failed"; + const stateFailure = + after === "absent" + ? null + : after === "match" + ? "exact content still live after remove" + : after === "drift" + ? "post-remove content drifted" + : "post-remove state unavailable"; + return { + before, + after, + attempted: true, + reportedSuccess, + errorMessage, + failureDetail: + mutationFailure && stateFailure + ? `${mutationFailure}; ${stateFailure}` + : (mutationFailure ?? stateFailure), + verifiedAbsent: after === "absent", + }; + } +} diff --git a/src/lib/registry-recovery-action.test.ts b/src/lib/registry-recovery-action.test.ts index d827725b3f..6b0f344f9a 100644 --- a/src/lib/registry-recovery-action.test.ts +++ b/src/lib/registry-recovery-action.test.ts @@ -125,6 +125,8 @@ describe("recoverRegistryEntries seed-time guard (#2753)", () => { model: "nemotron", policyPresets: ["npm"], nimContainer: null, + observabilityEnabled: true, + agent: "langchain-deepagents-code", steps: { sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, }, @@ -136,6 +138,7 @@ describe("recoverRegistryEntries seed-time guard (#2753)", () => { const recovered = result.sandboxes.find((s) => s.name === "alpha"); expect(recovered).toBeDefined(); expect(recovered?.policies).toEqual(["npm"]); + expect(recovered?.observabilityEnabled).toBe(true); }); it("restores complete custom-route identity from a confirmed session", async () => { @@ -261,6 +264,34 @@ describe("recoverRegistryEntries seed-time guard (#2753)", () => { expect(mockRegistryState.sandboxes["my-hermes"]?.agent).toBe("hermes"); }); + it("preserves recorded observability when an older confirmed session omits the field", async () => { + mockRegistryState.sandboxes.alpha = { + name: "alpha", + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + gpuEnabled: false, + policies: [], + nimContainer: null, + agent: "langchain-deepagents-code", + observabilityEnabled: true, + }; + vi.mocked(loadSession).mockReturnValue({ + sandboxName: "alpha", + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + policyPresets: [], + nimContainer: null, + agent: "langchain-deepagents-code", + steps: { + sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, + }, + } as never); + + await recoverRegistryEntries(); + + expect(mockRegistryState.sandboxes.alpha?.observabilityEnabled).toBe(true); + }); + it("does not evict a registered sandbox even when its session step is incomplete (avoids false positives)", async () => { // A user with a real registered sandbox alpha and a stale session that // happens to record alpha with an incomplete sandbox step (e.g. a diff --git a/src/lib/registry-recovery-action.ts b/src/lib/registry-recovery-action.ts index ce53389df3..0e36723af5 100644 --- a/src/lib/registry-recovery-action.ts +++ b/src/lib/registry-recovery-action.ts @@ -46,6 +46,7 @@ type RecoveredSandboxMetadata = Partial< | "policies" | "nimContainer" | "agent" + | "observabilityEnabled" | "endpointUrl" | "credentialEnv" | "preferredInferenceApi" @@ -86,6 +87,9 @@ function buildRecoveredSandboxEntry( if (metadata.agent !== undefined && metadata.agent !== null) { entry.agent = metadata.agent; } + if (typeof metadata.observabilityEnabled === "boolean") { + entry.observabilityEnabled = metadata.observabilityEnabled; + } return entry; } @@ -228,6 +232,9 @@ function seedRecoveryMetadata( endpointUrl: session.endpointUrl ?? null, credentialEnv: session.credentialEnv ?? null, preferredInferenceApi: session.preferredInferenceApi ?? null, + ...(typeof session.observabilityEnabled === "boolean" + ? { observabilityEnabled: session.observabilityEnabled } + : {}), }), ); const sessionSandboxMissing = !current.sandboxes.some( diff --git a/src/lib/state/onboard-session.test.ts b/src/lib/state/onboard-session.test.ts index 1664ac1b0d..caa6325c5f 100644 --- a/src/lib/state/onboard-session.test.ts +++ b/src/lib/state/onboard-session.test.ts @@ -154,6 +154,8 @@ describe("onboard session", () => { expect(saved.mode).toBe("non-interactive"); expect(saved.toolDisclosure).toBe("progressive"); + expect(saved.observabilityEnabled).toBe(false); + expect(saved.observabilityRequestedExplicitly).toBe(false); expect(saved.machine).toMatchObject({ version: 1, state: "init", @@ -165,6 +167,35 @@ describe("onboard session", () => { expect(dirStat.mode & 0o777).toBe(0o700); }); + it.each([ + true, + false, + ])("persists explicit observability intent when enabled=$enabled", (observabilityEnabled) => { + session.saveSession( + session.createSession({ + observabilityEnabled, + observabilityRequestedExplicitly: true, + }), + ); + const loaded = requireLoadedSession(session.loadSession()); + const summary = requireDebugSummary(session.summarizeForDebug()); + + expect(loaded.observabilityEnabled).toBe(observabilityEnabled); + expect(loaded.observabilityRequestedExplicitly).toBe(true); + expect(summary.observabilityEnabled).toBe(observabilityEnabled); + expect(summary.observabilityRequestedExplicitly).toBe(true); + }); + + it("defaults legacy observability intent and provenance off", () => { + const legacy = session.createSession() as unknown as Record; + delete legacy.observabilityEnabled; + delete legacy.observabilityRequestedExplicitly; + const normalized = requireLoadedSession(session.normalizeSession(legacy as never)); + + expect(normalized.observabilityEnabled).toBe(false); + expect(normalized.observabilityRequestedExplicitly).toBe(false); + }); + it("redacts credential-bearing endpoint URLs before persisting them", () => { session.saveSession(session.createSession()); markStepCompleteLegacy(session, stepMutation, "provider_selection", { diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index b0269a1696..a1dac16985 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -115,6 +115,10 @@ export interface Session { webSearchConfig: WebSearchConfig | null; /** Selected preference, retained even when a model-specific safeguard downgrades it. */ toolDisclosure: ToolDisclosure; + /** Enables credential-free OTLP trace export to NemoClaw's fixed local collector boundary. */ + observabilityEnabled: boolean; + /** True when observability was explicitly enabled or disabled for this resumable run. */ + observabilityRequestedExplicitly: boolean; hermesToolGateways: string[] | null; policyPresets: string[] | null; messagingPlan: SandboxMessagingPlan | null; @@ -186,6 +190,7 @@ export interface SessionUpdates { routerCredentialHash?: string; webSearchConfig?: WebSearchConfig | null; toolDisclosure?: ToolDisclosure; + observabilityEnabled?: boolean; hermesToolGateways?: string[] | null; policyPresets?: string[] | null; messagingPlan?: SandboxMessagingPlan | null; @@ -214,6 +219,8 @@ export interface DebugSessionSummary { compatibleEndpointReasoning: string | null; nimContainer: string | null; toolDisclosure: ToolDisclosure; + observabilityEnabled: boolean; + observabilityRequestedExplicitly: boolean; hermesToolGateways: string[] | null; policyPresets: string[] | null; gpuPassthrough: boolean; @@ -469,6 +476,8 @@ export function createSession(overrides: Partial = {}): Session { routerCredentialHash: overrides.routerCredentialHash ?? null, webSearchConfig: normalizeWebSearchConfig(overrides.webSearchConfig), toolDisclosure: normalizeSessionToolDisclosure(overrides.toolDisclosure), + observabilityEnabled: overrides.observabilityEnabled === true, + observabilityRequestedExplicitly: overrides.observabilityRequestedExplicitly === true, hermesToolGateways: readStringArray(overrides.hermesToolGateways), policyPresets: readStringArray(overrides.policyPresets), messagingPlan: parseSandboxMessagingPlan(overrides.messagingPlan), @@ -513,6 +522,8 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): routerCredentialHash: readString(data.routerCredentialHash), webSearchConfig: parseWebSearchConfig(data.webSearchConfig), toolDisclosure: normalizeSessionToolDisclosure(data.toolDisclosure), + observabilityEnabled: data.observabilityEnabled === true, + observabilityRequestedExplicitly: data.observabilityRequestedExplicitly === true, hermesToolGateways: readStringArray(data.hermesToolGateways), policyPresets: readStringArray(data.policyPresets), messagingPlan: parseSandboxMessagingPlan(data.messagingPlan), @@ -1010,6 +1021,9 @@ export function filterSafeUpdates(updates: SessionUpdates): Partial { safe.webSearchConfig = null; } assignSafeToolDisclosureUpdate(safe, updates.toolDisclosure); + if (typeof updates.observabilityEnabled === "boolean") { + safe.observabilityEnabled = updates.observabilityEnabled; + } if (updates.hermesToolGateways === null) { safe.hermesToolGateways = null; } else if (Array.isArray(updates.hermesToolGateways)) { @@ -1304,6 +1318,8 @@ export function summarizeForDebug( compatibleEndpointReasoning: session.compatibleEndpointReasoning, nimContainer: session.nimContainer, toolDisclosure: session.toolDisclosure, + observabilityEnabled: session.observabilityEnabled, + observabilityRequestedExplicitly: session.observabilityRequestedExplicitly, hermesToolGateways: session.hermesToolGateways, policyPresets: session.policyPresets, gpuPassthrough: session.gpuPassthrough, diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index b39ba51b9d..b7a1cf0405 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -105,6 +105,8 @@ export interface SandboxEntry extends Partial { webSearchEnabled?: boolean; /** Selected disclosure preference; model compatibility safeguards may downgrade runtime behavior. */ toolDisclosure?: ToolDisclosure; + /** Enables backend-neutral trace export to the fixed local OTLP collector boundary. */ + observabilityEnabled?: boolean; /** Durable provider identity for enabled managed web search. */ webSearchProvider?: WebSearchProvider | null; agent?: string | null; @@ -489,6 +491,8 @@ export function registerSandbox(entry: SandboxEntry): void { // Preserve absence on reconstructed legacy rows. Only a freshly built // sandbox registration may claim the new progressive default. toolDisclosure: normalizeToolDisclosure(entry.toolDisclosure) ?? undefined, + observabilityEnabled: + typeof entry.observabilityEnabled === "boolean" ? entry.observabilityEnabled : undefined, webSearchProvider: entry.webSearchEnabled === true && (entry.webSearchProvider === "brave" || entry.webSearchProvider === "tavily") diff --git a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh index d05d2b0dde..3ddecd348f 100755 --- a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh +++ b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh @@ -192,7 +192,7 @@ if ! reonboard_output="$( NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ OPENSHELL_GATEWAY=nemoclaw \ "$CLI" onboard --agent langchain-deepagents-code --name "$SANDBOX_NAME" \ - --fresh --non-interactive --yes --yes-i-accept-third-party-software 2>&1 + --fresh --non-interactive --observability --yes --yes-i-accept-third-party-software 2>&1 )"; then fail "same-name --fresh re-onboard failed: $reonboard_output" fi diff --git a/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh b/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh new file mode 100755 index 0000000000..fa82b4ea24 --- /dev/null +++ b/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Composed Deep Agents Code observability boundary. +# +# This check runs against a real OpenShell sandbox created with +# --observability. It captures Relay's OTLP/HTTP requests on the host, proves +# the exact host/path/method/binary policy allowlist, exercises both login-shell +# and direct-exec dcode launch paths, and inspects the wire payload for useful +# OpenInference content without ever uploading the captured trace artifact. + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-}}" +REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)}" +CLI="${NEMOCLAW_CLI_BIN:-${REPO}/bin/nemoclaw.js}" +PREFIX="11-deepagents-code-observability" +COLLECTOR_HOST="host.openshell.internal" +COLLECTOR_PORT=4318 +DECOY_PORT=4319 +CAPTURE_DIR="$(mktemp -d /tmp/nemoclaw-otlp-live.XXXXXX)" +COLLECTOR_LOG="${CAPTURE_DIR}/collector.log" +COLLECTOR_PID="" +CAPTURE_SERVER="${REPO}/test/e2e/live/deepagents-otlp-capture-server.ts" +CONTRACT_HELPER="${REPO}/test/e2e/live/deepagents-observability-contract.ts" +TSX="${REPO}/node_modules/.bin/tsx" +SERVICE_NAME="nemoclaw-langchain-deepagents-code" +ALLOWED_PROBE="NEMOCLAW_OTLP_ALLOWED_PROBE" +DIRECT_PROMPT="NEMOCLAW_OTLP_DIRECT_PROMPT_SENTINEL" +DIRECT_RESPONSE="NEMOCLAW_OTLP_DIRECT_RESPONSE_SENTINEL" +LOGIN_PROMPT="NEMOCLAW_OTLP_LOGIN_PROMPT_SENTINEL" +LOGIN_RESPONSE="NEMOCLAW_OTLP_LOGIN_RESPONSE_SENTINEL" +TOOL_NAME="nemoclaw_otlp_e2e_tool" +TOOL_ARGUMENT="NEMOCLAW_OTLP_TOOL_ARGUMENT_SENTINEL" +TOOL_RESULT="NEMOCLAW_OTLP_TOOL_RESULT_SENTINEL" +AMBIENT_CANARY="NEMOCLAW_OTLP_AMBIENT_EXPORTER_CANARY" + +fail() { + printf '%s: FAIL: %s\n' "$PREFIX" "$1" >&2 + exit 1 +} + +pass() { + printf '%s: OK (%s)\n' "$PREFIX" "$1" +} + +sandbox_exec() { + openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c "$1" 2>&1 +} + +cleanup() { + if [ -n "$COLLECTOR_PID" ] && kill -0 "$COLLECTOR_PID" 2>/dev/null; then + kill "$COLLECTOR_PID" 2>/dev/null || true + wait "$COLLECTOR_PID" 2>/dev/null || true + fi + rm -rf "$CAPTURE_DIR" +} +trap cleanup EXIT + +[ -n "$SANDBOX_NAME" ] || fail "sandbox name is required" + +# The generic cloud-onboard target runs every shared check against its OpenClaw +# sandbox. Typed DCode targets reject this SKIP through their required-check +# wrapper, so this guard only prevents cross-agent execution in the shared run. +if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1" >/dev/null; then + printf '%s: SKIP: sandbox %q is not a Deep Agents Code sandbox\n' "$PREFIX" "$SANDBOX_NAME" + exit 0 +fi + +[ -x "$CLI" ] || fail "NemoClaw CLI is not executable at $CLI" +[ -x "$TSX" ] || fail "tsx is not executable at $TSX" +[ -f "$CAPTURE_SERVER" ] || fail "OTLP capture server helper is absent" +[ -f "$CONTRACT_HELPER" ] || fail "OTLP contract helper is absent" +command -v ip >/dev/null 2>&1 || fail "host ip command is required" +command -v curl >/dev/null 2>&1 || fail "host curl command is required" + +bind_output="$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ + sh -c "getent ahostsv4 ${COLLECTOR_HOST} | awk 'NR == 1 { print \"NEMOCLAW_OTLP_BIND_IP=\" \$1 }'" \ + 2>&1)" \ + || fail "could not resolve $COLLECTOR_HOST from the sandbox: $bind_output" +OTLP_BIND_IP="$( + printf '%s\n' "$bind_output" | tr -d '\r' | sed -n 's/^NEMOCLAW_OTLP_BIND_IP=//p' | tail -n 1 +)" +case "$OTLP_BIND_IP" in + 10.* | 192.168.* | 172.1[6-9].* | 172.2[0-9].* | 172.3[01].*) ;; + *) fail "sandbox resolved $COLLECTOR_HOST to non-private address '$OTLP_BIND_IP'" ;; +esac +if ! ip -o -4 address show \ + | awk '{ sub(/\/.*/, "", $4); print $4 }' \ + | grep -Fxq "$OTLP_BIND_IP"; then + fail "sandbox bridge address $OTLP_BIND_IP is not assigned to a host interface" +fi + +"$TSX" "$CAPTURE_SERVER" "$CAPTURE_DIR" "$OTLP_BIND_IP" "$COLLECTOR_PORT" "$DECOY_PORT" \ + >"$COLLECTOR_LOG" 2>&1 & +COLLECTOR_PID=$! + +collector_ready=0 +for _attempt in $(seq 1 30); do + if grep -Fq 'CAPTURE_READY:' "$COLLECTOR_LOG" \ + && curl --noproxy '*' -fsS --max-time 1 \ + "http://${OTLP_BIND_IP}:${COLLECTOR_PORT}/health" >/dev/null 2>&1 \ + && curl --noproxy '*' -fsS --max-time 1 "http://${OTLP_BIND_IP}:${DECOY_PORT}/health" \ + >/dev/null 2>&1; then + collector_ready=1 + break + fi + if ! kill -0 "$COLLECTOR_PID" 2>/dev/null; then + fail "host OTLP capture server exited: $(tr '\n' ' ' <"$COLLECTOR_LOG")" + fi + sleep 1 +done +[ "$collector_ready" -eq 1 ] || fail "host OTLP capture server did not become ready" + +request_count() { + find "$CAPTURE_DIR" -maxdepth 1 -type f -name '*.json' | wc -l | tr -d '[:space:]' +} + +python_probe_source() { + cat <<'PY' +import sys +import urllib.error +import urllib.request + +# NemoClaw's exec wrapper sources the managed, credential-free OpenShell proxy +# route. Keep it intact so this probe exercises the same enforcement path +# as managed DCode and Relay instead of attempting an unsupported direct socket. +method, url, body = sys.argv[1:] +data = body.encode("utf-8") if method != "GET" else None +request = urllib.request.Request( + url, + data=data, + method=method, + headers={"content-type": "application/x-protobuf"}, +) +try: + with urllib.request.urlopen(request, timeout=10) as response: + print(f"REACHED:{response.status}") +except urllib.error.HTTPError as error: + body = error.read(512).decode("utf-8", "replace") + print(f"FAILED:HTTPError:{error}:{body}") + raise SystemExit(7) +except Exception as error: + print(f"FAILED:{type(error).__name__}:{error}") + raise SystemExit(7) +PY +} + +sandbox_python_probe() { + local method="$1" + local url="$2" + local body="$3" + local encoded + encoded="$(python_probe_source | base64 | tr -d '\n')" + "$CLI" "$SANDBOX_NAME" exec -- \ + /opt/venv/bin/python3 -I -c \ + "import base64; exec(compile(base64.b64decode('${encoded}'), '', 'exec'))" \ + "$method" "$url" "$body" 2>&1 +} + +expect_blocked_without_capture() { + local label="$1" + local method="$2" + local url="$3" + local before output status after denial_state + before="$(request_count)" + set +e + output="$(sandbox_python_probe "$method" "$url" "NEMOCLAW_OTLP_DENIED_PROBE" 2>&1)" + status=$? + set -e + sleep 1 + after="$(request_count)" + [ "$status" -ne 0 ] || fail "$label unexpectedly returned success: $output" + [ "$after" = "$before" ] || fail "$label reached the host capture server" + denial_state="$(printf '%s\n' "$output" | "$TSX" "$CONTRACT_HELPER" denial-state)" \ + || fail "$label denial classifier failed: $output" + [ "$denial_state" = "policy-denied" ] \ + || fail "$label failed without confirmed OpenShell policy-denial evidence: $output" + pass "$label is denied before the host collector" +} + +policy_output="$("$CLI" "$SANDBOX_NAME" policy-list 2>&1)" || fail "could not inspect active policy" +policy_state="$(printf '%s\n' "$policy_output" | "$TSX" "$CONTRACT_HELPER" policy-state)" \ + || fail "could not parse observability policy state: $policy_output" +[ "$policy_state" = "active" ] \ + || fail "observability-otlp-local is not exactly active (state: $policy_state)" + +registry_output="$( + SANDBOX_NAME="$SANDBOX_NAME" node - <<'NODE' +const fs = require("node:fs"); +const path = require("node:path"); +const registry = JSON.parse( + fs.readFileSync(path.join(process.env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"), +); +const entry = registry.sandboxes?.[process.env.SANDBOX_NAME]; +process.stdout.write(entry?.observabilityEnabled === true ? "enabled" : "disabled"); +NODE +)" +[ "$registry_output" = "enabled" ] || fail "host registry does not record observability enabled" + +marker_output="$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ + sh -c 'test -f /tmp/nemoclaw-observability-enabled && cat /tmp/nemoclaw-observability-enabled' \ + 2>&1)" || fail "managed observability marker is absent" +[ "$marker_output" = "1" ] || fail "managed observability marker has an unexpected value" +pass "host registry, live policy, and sandbox marker agree on enabled observability" + +allowed_output="$(sandbox_python_probe POST \ + "http://${COLLECTOR_HOST}:${COLLECTOR_PORT}/v1/traces" \ + "$ALLOWED_PROBE")" || fail "allowed OTLP request failed: $allowed_output" +printf '%s\n' "$allowed_output" | grep -Fq 'REACHED:200' \ + || fail "allowed OTLP request lacked HTTP 200 evidence: $allowed_output" +pass "managed Python can POST only to the configured OTLP route" + +expect_blocked_without_capture \ + "alternate OTLP host" POST "http://example.com:${COLLECTOR_PORT}/v1/traces" +expect_blocked_without_capture \ + "alternate OTLP path" POST "http://${COLLECTOR_HOST}:${COLLECTOR_PORT}/not-traces" +expect_blocked_without_capture \ + "alternate OTLP method" GET "http://${COLLECTOR_HOST}:${COLLECTOR_PORT}/v1/traces" +expect_blocked_without_capture \ + "alternate OTLP port" POST "http://${COLLECTOR_HOST}:${DECOY_PORT}/v1/traces" + +openshell sandbox exec --name "$SANDBOX_NAME" -- test -x /usr/bin/curl >/dev/null 2>&1 \ + || fail "/usr/bin/curl is absent or not executable in the sandbox" +before_binary="$(request_count)" +set +e +binary_output="$("$CLI" "$SANDBOX_NAME" exec -- \ + /usr/bin/curl --fail-with-body -sS --max-time 10 -X POST \ + -H 'content-type: application/x-protobuf' \ + --data-binary 'NEMOCLAW_OTLP_DENIED_BINARY_PROBE' \ + "http://${COLLECTOR_HOST}:${COLLECTOR_PORT}/v1/traces" 2>&1)" +binary_status=$? +set -e +sleep 1 +after_binary="$(request_count)" +[ "$binary_status" -ne 0 ] || fail "unmanaged curl binary unexpectedly reached OTLP: $binary_output" +[ "$after_binary" = "$before_binary" ] || fail "unmanaged curl binary reached the host collector" +binary_denial_state="$(printf '%s\n' "$binary_output" | "$TSX" "$CONTRACT_HELPER" denial-state)" \ + || fail "unmanaged curl denial classifier failed: $binary_output" +[ "$binary_denial_state" = "policy-denied" ] \ + || fail "unmanaged curl failed without confirmed OpenShell policy-denial evidence: $binary_output" +pass "OTLP route is denied to an unmanaged binary" + +run_dcode_direct() { + openshell sandbox exec --name "$SANDBOX_NAME" -- \ + env OTEL_SERVICE_NAME="$AMBIENT_CANARY" \ + OTEL_RESOURCE_ATTRIBUTES="ambient.canary=${AMBIENT_CANARY}" \ + dcode -n \ + "Reply with exactly ${DIRECT_RESPONSE}. Do not repeat the input marker ${DIRECT_PROMPT}." 2>&1 +} + +run_dcode_login() { + local prompt + prompt="Reply with exactly ${LOGIN_RESPONSE}. Do not repeat the input marker ${LOGIN_PROMPT}." + openshell sandbox exec --name "$SANDBOX_NAME" -- bash -lc \ + "OTEL_SERVICE_NAME=${AMBIENT_CANARY@Q} OTEL_RESOURCE_ATTRIBUTES=$(printf '%q' "ambient.canary=${AMBIENT_CANARY}") dcode -n ${prompt@Q}" \ + 2>&1 +} + +tool_trace_source() { + cat <<'PY' +import os +import sys + +from langchain.agents.middleware.types import ToolCallRequest +from deepagents_code import nemoclaw_observability as observability + +tool_name, argument_marker, result_marker = sys.argv[1:] +os.environ["NEMOCLAW_OBSERVABILITY"] = "1" +if not observability.initialize_observability(): + raise RuntimeError("managed observability did not initialize") +try: + middleware = observability.new_relay_middleware() + request = ToolCallRequest( + tool_call={ + "name": tool_name, + "args": {"command": argument_marker}, + "id": "nemoclaw-otlp-live-tool", + }, + tool=None, + state={}, + runtime=None, + ) + + def handler(inner_request): + if inner_request.tool_call["args"] != {"command": argument_marker}: + raise AssertionError("managed tool arguments changed") + return {"stdout": result_marker} + + result = middleware.wrap_tool_call(request, handler) + if result != {"stdout": result_marker}: + raise AssertionError("managed tool result changed") + print("TOOL_TRACE_OK") +finally: + observability.shutdown_observability() +PY +} + +run_deterministic_tool_trace() { + local encoded + encoded="$(tool_trace_source | base64 | tr -d '\n')" + "$CLI" "$SANDBOX_NAME" exec -- \ + env NEMOCLAW_OBSERVABILITY=1 \ + OTEL_SERVICE_NAME="$AMBIENT_CANARY" \ + OTEL_RESOURCE_ATTRIBUTES="ambient.canary=${AMBIENT_CANARY}" \ + /opt/venv/bin/python3 -I -c \ + "import base64; exec(compile(base64.b64decode('${encoded}'), '', 'exec'))" \ + "$TOOL_NAME" "$TOOL_ARGUMENT" "$TOOL_RESULT" 2>&1 +} + +direct_output="$(run_dcode_direct)" || fail "direct-exec dcode observability turn failed: $direct_output" +printf '%s\n' "$direct_output" | grep -Fq "$DIRECT_RESPONSE" \ + || fail "direct-exec dcode response omitted its requested marker" +pass "direct-exec dcode completed with observability enabled" + +login_output="$(run_dcode_login)" || fail "login-shell dcode observability turn failed: $login_output" +printf '%s\n' "$login_output" | grep -Fq "$LOGIN_RESPONSE" \ + || fail "login-shell dcode response omitted its requested marker" +pass "login-shell dcode completed with observability enabled" + +tool_trace_output="$(run_deterministic_tool_trace)" \ + || fail "deterministic managed tool trace failed: $tool_trace_output" +printf '%s\n' "$tool_trace_output" | grep -Fq 'TOOL_TRACE_OK' \ + || fail "deterministic managed tool trace lacked completion evidence: $tool_trace_output" +pass "managed instrumentation emitted a deterministic tool trace" + +payload_ready=0 +validation_output="" +for _attempt in $(seq 1 45); do + set +e + validation_output="$( + COLLECTOR_PORT="$COLLECTOR_PORT" \ + ALLOWED_PROBE="$ALLOWED_PROBE" \ + SERVICE_NAME="$SERVICE_NAME" \ + DIRECT_PROMPT="$DIRECT_PROMPT" \ + DIRECT_RESPONSE="$DIRECT_RESPONSE" \ + LOGIN_PROMPT="$LOGIN_PROMPT" \ + LOGIN_RESPONSE="$LOGIN_RESPONSE" \ + TOOL_NAME="$TOOL_NAME" \ + TOOL_ARGUMENT="$TOOL_ARGUMENT" \ + TOOL_RESULT="$TOOL_RESULT" \ + AMBIENT_CANARY="$AMBIENT_CANARY" \ + "$TSX" "$CONTRACT_HELPER" validate-captures "$CAPTURE_DIR" 2>&1 + )" + validation_status=$? + set -e + if [ "$validation_status" -eq 0 ]; then + payload_ready=1 + break + fi + sleep 1 +done +[ "$payload_ready" -eq 1 ] \ + || fail "captured OTLP contract did not become valid: $validation_output" + +pass "decoded OTLP associates model/tool content and excludes ambient exporter configuration" +printf '%s: 11 passed, 0 failed\n' "$PREFIX" diff --git a/test/e2e/fixtures/phases/onboarding.ts b/test/e2e/fixtures/phases/onboarding.ts index ec34fa3b2e..f6a1f04d78 100644 --- a/test/e2e/fixtures/phases/onboarding.ts +++ b/test/e2e/fixtures/phases/onboarding.ts @@ -229,7 +229,7 @@ export class OnboardingPhaseFixture { const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); this.registerSandboxCleanup(sandboxName); - const result = await this.host.nemoclaw(ONBOARD_ARGS, { + const result = await this.host.nemoclaw([...ONBOARD_ARGS, "--observability"], { artifactName: "onboard-cloud-langchain-deepagents-code", env: commandEnv(sandboxName, { NEMOCLAW_AGENT: "langchain-deepagents-code", diff --git a/test/e2e/live/cloud-experimental-check-list.ts b/test/e2e/live/cloud-experimental-check-list.ts index 76cf4f9b82..fc444f2263 100644 --- a/test/e2e/live/cloud-experimental-check-list.ts +++ b/test/e2e/live/cloud-experimental-check-list.ts @@ -3,6 +3,8 @@ export const DEEPAGENTS_FRESH_REONBOARD_CHECK = "test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh"; +export const DEEPAGENTS_OBSERVABILITY_CHECK = + "test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh"; export const DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS = [ DEEPAGENTS_FRESH_REONBOARD_CHECK, @@ -12,6 +14,7 @@ export const DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS = [ "test/e2e/e2e-cloud-experimental/checks/08-deepagents-code-secret-boundary.sh", "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", + DEEPAGENTS_OBSERVABILITY_CHECK, ] as const; export function cloudExperimentalChecksForOnboarding( diff --git a/test/e2e/live/cloud-experimental-checks.ts b/test/e2e/live/cloud-experimental-checks.ts index 5b218b216b..9f590e9202 100644 --- a/test/e2e/live/cloud-experimental-checks.ts +++ b/test/e2e/live/cloud-experimental-checks.ts @@ -9,11 +9,15 @@ import { resultText } from "../fixtures/clients/command.ts"; import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { DEEPAGENTS_FRESH_REONBOARD_CHECK } from "./cloud-experimental-check-list.ts"; +import { + DEEPAGENTS_FRESH_REONBOARD_CHECK, + DEEPAGENTS_OBSERVABILITY_CHECK, +} from "./cloud-experimental-check-list.ts"; const REQUIRED_CHECK_SKIP_PATTERN = /(^|\n).*\bSKIP\b/i; const DEFAULT_CHECK_TIMEOUT_MS = 180_000; const FRESH_REONBOARD_TIMEOUT_MS = 15 * 60_000; +const OBSERVABILITY_TIMEOUT_MS = 8 * 60_000; export type CloudExperimentalChecksEvidence = { targetId: string; @@ -83,9 +87,9 @@ export function assertRequiredCloudExperimentalResult( } export function cloudExperimentalCheckTimeoutMs(scriptPath: string): number { - return scriptPath === DEEPAGENTS_FRESH_REONBOARD_CHECK - ? FRESH_REONBOARD_TIMEOUT_MS - : DEFAULT_CHECK_TIMEOUT_MS; + if (scriptPath === DEEPAGENTS_FRESH_REONBOARD_CHECK) return FRESH_REONBOARD_TIMEOUT_MS; + if (scriptPath === DEEPAGENTS_OBSERVABILITY_CHECK) return OBSERVABILITY_TIMEOUT_MS; + return DEFAULT_CHECK_TIMEOUT_MS; } async function assertDeepAgentsRuntimeObserved( diff --git a/test/e2e/live/deepagents-observability-contract.ts b/test/e2e/live/deepagents-observability-contract.ts new file mode 100644 index 0000000000..b0143a75a6 --- /dev/null +++ b/test/e2e/live/deepagents-observability-contract.ts @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import * as policyDenialNamespace from "../../../src/lib/actions/sandbox/exec-policy-hint-detection.ts"; +import type { DecodedOtlpSpan } from "./otlp-trace-decoder.ts"; +import * as decoderNamespace from "./otlp-trace-decoder.ts"; +import * as policyStateNamespace from "./policy-list-state.ts"; + +function moduleExports(namespace: T & { default?: T }): T { + return namespace.default ?? namespace; +} + +const { isPolicyDenialLine } = moduleExports(policyDenialNamespace); +const { decodeExportTraceServiceRequest, otlpValueContains } = moduleExports(decoderNamespace); +const { parsePolicyPresetState } = moduleExports(policyStateNamespace); + +const INPUT_ATTRIBUTE_KEYS = ["input.value", "llm.input_messages"] as const; +const OUTPUT_ATTRIBUTE_KEYS = ["output.value", "llm.output_messages"] as const; +const TOOL_INPUT_ATTRIBUTE_KEYS = ["tool.parameters", "input.value"] as const; +const CONFIRMED_EXEC_HINT = + /^[a-z][a-z0-9-]*: recent network policy denial detected(?: for [^\r\n]+)? inside sandbox '[a-zA-Z0-9][a-zA-Z0-9_-]*'\.$/mu; +export type LlmTraceExpectation = { + label: string; + promptMarker: string; + responseMarker: string; +}; + +export type ToolTraceExpectation = { + argumentMarker: string; + name: string; + resultMarker: string; +}; + +export type DeepAgentsTraceExpectations = { + ambientCanary: string; + llmExchanges: readonly LlmTraceExpectation[]; + serviceName: string; + tool: ToolTraceExpectation; +}; + +type CaptureMetadata = { + accepted?: unknown; + contentType?: unknown; + method?: unknown; + path?: unknown; + port?: unknown; + rejection?: unknown; +}; + +function spanKind(span: DecodedOtlpSpan): string { + const kind = span.attributes["openinference.span.kind"]; + return typeof kind === "string" ? kind.toUpperCase() : ""; +} + +function markerInAttributes( + span: DecodedOtlpSpan, + keys: readonly string[], + marker: string, +): boolean { + return keys.some((key) => otlpValueContains(span.attributes[key], marker)); +} + +function hasManagedService(span: DecodedOtlpSpan, serviceName: string): boolean { + return span.resourceAttributes["service.name"] === serviceName; +} + +function assertLlmExchange( + spans: readonly DecodedOtlpSpan[], + serviceName: string, + expectation: LlmTraceExpectation, +): void { + const match = spans.find( + (span) => + hasManagedService(span, serviceName) && + spanKind(span) === "LLM" && + markerInAttributes(span, INPUT_ATTRIBUTE_KEYS, expectation.promptMarker) && + markerInAttributes(span, OUTPUT_ATTRIBUTE_KEYS, expectation.responseMarker), + ); + if (!match) { + throw new Error( + `${expectation.label} prompt and response markers were not associated on one managed LLM span`, + ); + } +} + +function assertToolCall( + spans: readonly DecodedOtlpSpan[], + serviceName: string, + expectation: ToolTraceExpectation, +): void { + const match = spans.find( + (span) => + hasManagedService(span, serviceName) && + spanKind(span) === "TOOL" && + otlpValueContains(span.attributes["tool.name"], expectation.name) && + markerInAttributes(span, TOOL_INPUT_ATTRIBUTE_KEYS, expectation.argumentMarker) && + markerInAttributes(span, OUTPUT_ATTRIBUTE_KEYS, expectation.resultMarker), + ); + if (!match) { + throw new Error( + "tool name, argument, and result markers were not associated on one managed TOOL span", + ); + } +} + +export function assertDeepAgentsTraceContract( + bodies: readonly Uint8Array[], + expectations: DeepAgentsTraceExpectations, +): { requestCount: number; spanCount: number } { + if (bodies.length === 0) throw new Error("no managed OTLP trace requests were captured"); + const canary = Buffer.from(expectations.ambientCanary); + const spans = bodies.flatMap((body, index) => { + if (Buffer.from(body).includes(canary)) { + throw new Error("ambient exporter configuration reached OTLP"); + } + try { + return decodeExportTraceServiceRequest(body); + } catch (error) { + throw new Error( + `captured OTLP request ${index + 1} is not a valid ExportTraceServiceRequest: ${error instanceof Error ? error.message : String(error)}`, + ); + } + }); + + for (const expectation of expectations.llmExchanges) { + assertLlmExchange(spans, expectations.serviceName, expectation); + } + assertToolCall(spans, expectations.serviceName, expectations.tool); + return { requestCount: bodies.length, spanCount: spans.length }; +} + +export function hasConfirmedOpenShellPolicyDenial(output: string): boolean { + return output.split(/\r?\n/u).some(isPolicyDenialLine) || CONFIRMED_EXEC_HINT.test(output); +} + +export function observabilityPresetState(output: string): string { + return parsePolicyPresetState(output, "observability-otlp-local"); +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`required environment variable ${name} is missing`); + return value; +} + +function captureMetadata(value: unknown, filename: string): CaptureMetadata { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${filename} does not contain a capture metadata object`); + } + return value as CaptureMetadata; +} + +export function validateCaptureDirectory( + captureDir: string, + collectorPort: number, + allowedProbeBody: string, + expectations: DeepAgentsTraceExpectations, +): { requestCount: number; spanCount: number } { + const metadataFiles = fs + .readdirSync(captureDir) + .filter((filename) => filename.endsWith(".json")) + .sort(); + const traceBodies: Buffer[] = []; + let allowedProbeCount = 0; + + for (const metadataFile of metadataFiles) { + const metadata = captureMetadata( + JSON.parse(fs.readFileSync(path.join(captureDir, metadataFile), "utf8")), + metadataFile, + ); + if (metadata.accepted !== true) { + throw new Error(`${metadataFile} records a rejected request: ${String(metadata.rejection)}`); + } + if ( + metadata.port !== collectorPort || + metadata.method !== "POST" || + metadata.path !== "/v1/traces" + ) { + throw new Error( + `unexpected captured route ${String(metadata.method)} ${String(metadata.path)} on ${String(metadata.port)}`, + ); + } + if (metadata.contentType !== "application/x-protobuf") { + throw new Error(`${metadataFile} is not OTLP binary protobuf`); + } + const body = fs.readFileSync(path.join(captureDir, metadataFile.replace(/\.json$/u, ".body"))); + if (body.equals(Buffer.from(allowedProbeBody))) { + allowedProbeCount += 1; + continue; + } + traceBodies.push(body); + } + + if (allowedProbeCount !== 1) { + throw new Error(`expected one managed-Python allow probe, captured ${allowedProbeCount}`); + } + return assertDeepAgentsTraceContract(traceBodies, expectations); +} + +async function main(): Promise { + const [command, argument] = process.argv.slice(2); + const input = command === "validate-captures" ? "" : fs.readFileSync(0, "utf8"); + if (command === "policy-state") { + process.stdout.write(`${observabilityPresetState(input)}\n`); + return; + } + if (command === "denial-state") { + process.stdout.write( + `${hasConfirmedOpenShellPolicyDenial(input) ? "policy-denied" : "other-failure"}\n`, + ); + return; + } + if (command === "validate-captures" && argument) { + const result = validateCaptureDirectory( + argument, + Number(requiredEnvironment("COLLECTOR_PORT")), + requiredEnvironment("ALLOWED_PROBE"), + { + ambientCanary: requiredEnvironment("AMBIENT_CANARY"), + serviceName: requiredEnvironment("SERVICE_NAME"), + llmExchanges: [ + { + label: "direct-exec", + promptMarker: requiredEnvironment("DIRECT_PROMPT"), + responseMarker: requiredEnvironment("DIRECT_RESPONSE"), + }, + { + label: "login-shell", + promptMarker: requiredEnvironment("LOGIN_PROMPT"), + responseMarker: requiredEnvironment("LOGIN_RESPONSE"), + }, + ], + tool: { + argumentMarker: requiredEnvironment("TOOL_ARGUMENT"), + name: requiredEnvironment("TOOL_NAME"), + resultMarker: requiredEnvironment("TOOL_RESULT"), + }, + }, + ); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } + throw new Error( + "usage: deepagents-observability-contract.ts [capture-dir]", + ); +} + +const invokedAsScript = + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; +if (invokedAsScript) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exit(1); + }); +} diff --git a/test/e2e/live/deepagents-otlp-capture-server.ts b/test/e2e/live/deepagents-otlp-capture-server.ts new file mode 100644 index 0000000000..325c24c048 --- /dev/null +++ b/test/e2e/live/deepagents-otlp-capture-server.ts @@ -0,0 +1,363 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import http, { type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export const DEFAULT_MAX_OTLP_BODY_BYTES = 1_048_576; +export const DEFAULT_MAX_CAPTURE_BYTES = 16 * 1_048_576; +export const DEFAULT_MAX_CAPTURE_REQUESTS = 128; +const OTLP_CONTENT_TYPE = "application/x-protobuf"; +const FORBIDDEN_EXPORTER_HEADERS = new Set([ + "authorization", + "cookie", + "grpc-metadata-authorization", + "proxy-authorization", + "x-api-key", +]); + +export type OtlpCaptureMetadata = { + accepted: boolean; + contentType: typeof OTLP_CONTENT_TYPE | null; + method: "POST" | null; + path: "/v1/traces" | null; + port: number; + rejection: string | null; +}; + +export type OtlpCaptureServerOptions = { + allowLoopback?: boolean; + bindIp: string; + captureDir: string; + collectorPort: number; + decoyPort: number; + maxCaptureBytes?: number; + maxCaptureRequests?: number; + maxBodyBytes?: number; +}; + +export type StartedOtlpCaptureServers = { + close(): Promise; + collectorPort: number; + decoyPort: number; + snapshot(): { capturedBytes: number; requestCount: number; reservedBytes: number }; +}; + +function parseIpv4(value: string): number[] | null { + const parts = value.split("."); + if (parts.length !== 4) return null; + const octets = parts.map(Number); + if ( + octets.some( + (octet, index) => + !Number.isInteger(octet) || octet < 0 || octet > 255 || String(octet) !== parts[index], + ) + ) { + return null; + } + return octets; +} + +export function isPrivateBridgeIpv4(value: string, allowLoopback = false): boolean { + const octets = parseIpv4(value); + if (!octets) return false; + if (allowLoopback && octets[0] === 127) return true; + return ( + octets[0] === 10 || + (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) || + (octets[0] === 192 && octets[1] === 168) + ); +} + +function configuredPort(server: Server): number { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("OTLP capture server did not expose an IPv4 listener"); + } + return address.port; +} + +function listen(server: Server, bindIp: string, port: number): Promise { + return new Promise((resolve, reject) => { + const onError = (error: Error) => { + server.off("listening", onListening); + reject(error); + }; + const onListening = () => { + server.off("error", onError); + resolve(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(port, bindIp); + }); +} + +function close(server: Server): Promise { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +function writeResponse(response: ServerResponse, statusCode: number): void { + if (response.destroyed || response.writableEnded) return; + response.writeHead(statusCode, { "content-length": "0", connection: "close" }); + response.end(); +} + +function numericContentLength(request: IncomingMessage): number | null { + const raw = request.headers["content-length"]; + if (typeof raw !== "string" || !/^[1-9][0-9]*$/.test(raw)) return null; + const value = Number(raw); + return Number.isSafeInteger(value) ? value : null; +} + +function captureFilePath( + captureDir: string, + sequence: number, + port: number, + extension: "body" | "json", +): string { + if (!Number.isSafeInteger(sequence) || sequence < 1) { + throw new Error("capture sequence must be a positive safe integer"); + } + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + throw new Error("capture port must be a valid TCP port"); + } + const stem = `${String(sequence).padStart(4, "0")}-${port}`; + const target = path.resolve(captureDir, `${stem}.${extension}`); + if (path.dirname(target) !== captureDir) { + throw new Error("capture file escaped the configured directory"); + } + return target; +} + +function hasForbiddenExporterHeader(request: IncomingMessage): boolean { + return Object.keys(request.headers).some((name) => FORBIDDEN_EXPORTER_HEADERS.has(name)); +} + +export async function startOtlpCaptureServers( + options: OtlpCaptureServerOptions, +): Promise { + const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_OTLP_BODY_BYTES; + const maxCaptureBytes = options.maxCaptureBytes ?? DEFAULT_MAX_CAPTURE_BYTES; + const maxCaptureRequests = options.maxCaptureRequests ?? DEFAULT_MAX_CAPTURE_REQUESTS; + if (!Number.isSafeInteger(maxBodyBytes) || maxBodyBytes < 1) { + throw new Error("maxBodyBytes must be a positive safe integer"); + } + if (!Number.isSafeInteger(maxCaptureBytes) || maxCaptureBytes < maxBodyBytes) { + throw new Error("maxCaptureBytes must be a safe integer at least as large as maxBodyBytes"); + } + if (!Number.isSafeInteger(maxCaptureRequests) || maxCaptureRequests < 1) { + throw new Error("maxCaptureRequests must be a positive safe integer"); + } + if (!isPrivateBridgeIpv4(options.bindIp, options.allowLoopback === true)) { + throw new Error(`refusing non-private OTLP capture bind address: ${options.bindIp}`); + } + const requestedCaptureDir = path.resolve(options.captureDir); + const captureStat = fs.lstatSync(requestedCaptureDir); + if (captureStat.isSymbolicLink() || !captureStat.isDirectory()) { + throw new Error(`OTLP capture path is not a real directory: ${requestedCaptureDir}`); + } + const captureDir = fs.realpathSync.native(requestedCaptureDir); + + let sequence = 0; + let capturedBytes = 0; + let reservedBytes = 0; + const servers: Server[] = []; + + const start = async (port: number): Promise => { + let server: Server; + server = http.createServer((request, response) => { + if (request.method === "GET" && request.url === "/health") { + writeResponse(response, 200); + return; + } + + let finalized = false; + let observedBytes = 0; + let reservation = 0; + const chunks: Buffer[] = []; + const declaredBytes = numericContentLength(request); + sequence += 1; + const requestSequence = sequence; + + const finalize = (accepted: boolean, rejection: string | null, statusCode: number): void => { + if (finalized) return; + finalized = true; + reservedBytes -= reservation; + reservation = 0; + if (requestSequence > maxCaptureRequests + 1) { + writeResponse(response, 429); + return; + } + const requestLimitExceeded = requestSequence > maxCaptureRequests; + const captureAccepted = accepted && !requestLimitExceeded; + const body = captureAccepted ? Buffer.concat(chunks, observedBytes) : Buffer.alloc(0); + capturedBytes += body.length; + const port = configuredPort(server); + const metadata: OtlpCaptureMetadata = { + accepted: captureAccepted, + contentType: captureAccepted ? OTLP_CONTENT_TYPE : null, + method: captureAccepted ? "POST" : null, + path: captureAccepted ? "/v1/traces" : null, + port, + rejection: requestLimitExceeded ? "capture request count exceeds bound" : rejection, + }; + const bodyPath = captureFilePath(captureDir, requestSequence, port, "body"); + const metadataPath = captureFilePath(captureDir, requestSequence, port, "json"); + fs.writeFileSync(bodyPath, body, { flag: "wx", mode: 0o600 }); + fs.writeFileSync(metadataPath, JSON.stringify(metadata), { + flag: "wx", + mode: 0o600, + }); + writeResponse(response, requestLimitExceeded ? 429 : statusCode); + }; + + if (requestSequence > maxCaptureRequests) { + finalize(false, "capture request count exceeds bound", 429); + request.destroy(); + return; + } + + if (request.method !== "POST") { + finalize(false, "unexpected request method", 405); + request.destroy(); + return; + } + if (request.url !== "/v1/traces") { + finalize(false, "unexpected request path", 404); + request.destroy(); + return; + } + if (hasForbiddenExporterHeader(request)) { + finalize(false, "forbidden exporter header", 400); + request.destroy(); + return; + } + if (request.headers["content-type"] !== OTLP_CONTENT_TYPE) { + finalize(false, "unexpected content type", 415); + request.destroy(); + return; + } + + if (declaredBytes === null) { + finalize(false, "missing or invalid content-length", 411); + request.destroy(); + return; + } + if (declaredBytes > maxBodyBytes) { + finalize(false, "declared body exceeds capture bound", 413); + request.destroy(); + return; + } + if (capturedBytes + reservedBytes + declaredBytes > maxCaptureBytes) { + finalize(false, "aggregate captured bodies exceed bound", 507); + request.destroy(); + return; + } + reservation = declaredBytes; + reservedBytes += reservation; + + request.setTimeout(15_000, () => { + finalize(false, "request body timed out", 408); + request.destroy(); + }); + request.on("data", (chunk: Buffer) => { + if (finalized) return; + observedBytes += chunk.length; + if (observedBytes > maxBodyBytes || observedBytes > declaredBytes) { + finalize(false, "streamed body exceeds capture bound", 413); + request.destroy(); + return; + } + chunks.push(chunk); + }); + request.on("end", () => { + if (finalized) return; + if (observedBytes !== declaredBytes) { + finalize(false, "request body length mismatch", 400); + return; + } + finalize(true, null, 200); + }); + request.on("error", () => { + if (request.complete) finalize(false, "request body stream failed", 400); + }); + request.on("close", () => { + if (!request.complete) finalize(false, "request body aborted", 400); + }); + }); + await listen(server, options.bindIp, port); + servers.push(server); + return server; + }; + + try { + const collector = await start(options.collectorPort); + const decoy = await start(options.decoyPort); + return { + collectorPort: configuredPort(collector), + decoyPort: configuredPort(decoy), + close: async () => { + await Promise.all(servers.map(close)); + }, + snapshot: () => ({ capturedBytes, requestCount: sequence, reservedBytes }), + }; + } catch (error) { + await Promise.allSettled(servers.map(close)); + throw error; + } +} + +async function main(): Promise { + const [captureDir, bindIp, collectorPortRaw, decoyPortRaw] = process.argv.slice(2); + const collectorPort = Number(collectorPortRaw); + const decoyPort = Number(decoyPortRaw); + if ( + !captureDir || + !bindIp || + !Number.isInteger(collectorPort) || + collectorPort < 1 || + !Number.isInteger(decoyPort) || + decoyPort < 1 + ) { + throw new Error( + "usage: deepagents-otlp-capture-server.ts ", + ); + } + const started = await startOtlpCaptureServers({ + bindIp, + captureDir, + collectorPort, + decoyPort, + }); + process.stdout.write( + `CAPTURE_READY:${JSON.stringify({ bindIp, collectorPort: started.collectorPort, decoyPort: started.decoyPort })}\n`, + ); + + let closing = false; + const shutdown = async () => { + if (closing) return; + closing = true; + const timeout = setTimeout(() => process.exit(1), 5_000); + timeout.unref(); + await started.close(); + clearTimeout(timeout); + process.exit(0); + }; + process.on("SIGTERM", () => void shutdown()); + process.on("SIGINT", () => void shutdown()); +} + +const invokedAsScript = + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; +if (invokedAsScript) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exit(1); + }); +} diff --git a/test/e2e/live/otlp-trace-decoder.ts b/test/e2e/live/otlp-trace-decoder.ts new file mode 100644 index 0000000000..c178bf0bac --- /dev/null +++ b/test/e2e/live/otlp-trace-decoder.ts @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface OtlpAttributeArray extends ReadonlyArray {} + +export interface OtlpAttributeMap { + readonly [key: string]: OtlpAttributeValue; +} + +export type OtlpAttributeValue = + | string + | number + | boolean + | null + | OtlpAttributeArray + | OtlpAttributeMap; + +export type DecodedOtlpSpan = { + attributes: Readonly>; + name: string; + resourceAttributes: Readonly>; +}; + +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const MAX_ANY_VALUE_DEPTH = 16; +const FORBIDDEN_ATTRIBUTE_KEYS = new Set(["__proto__", "constructor", "prototype"]); + +function emptyAttributeMap(): Record { + return Object.create(null) as Record; +} + +class WireReader { + private offset = 0; + + constructor(private readonly bytes: Uint8Array) {} + + get done(): boolean { + return this.offset === this.bytes.length; + } + + readTag(): { field: number; wireType: number } { + const tag = this.readVarint(); + const field = Number(tag >> 3n); + const wireType = Number(tag & 0x07n); + if (!Number.isSafeInteger(field) || field < 1) throw new Error("invalid protobuf field tag"); + return { field, wireType }; + } + + readVarint(): bigint { + let result = 0n; + for (let index = 0; index < 10; index += 1) { + if (this.offset >= this.bytes.length) throw new Error("truncated protobuf varint"); + const byte = this.bytes[this.offset]; + this.offset += 1; + result |= BigInt(byte & 0x7f) << BigInt(index * 7); + if ((byte & 0x80) === 0) return result; + } + throw new Error("protobuf varint exceeds 10 bytes"); + } + + readBytes(): Uint8Array { + const length = Number(this.readVarint()); + if (!Number.isSafeInteger(length) || length < 0 || this.offset + length > this.bytes.length) { + throw new Error("truncated protobuf length-delimited field"); + } + const result = this.bytes.subarray(this.offset, this.offset + length); + this.offset += length; + return result; + } + + readString(): string { + return textDecoder.decode(this.readBytes()); + } + + readDouble(): number { + if (this.offset + 8 > this.bytes.length) throw new Error("truncated protobuf double"); + const view = new DataView(this.bytes.buffer, this.bytes.byteOffset + this.offset, 8); + const result = view.getFloat64(0, true); + this.offset += 8; + return result; + } + + skip(wireType: number): void { + switch (wireType) { + case 0: + this.readVarint(); + return; + case 1: + this.skipBytes(8); + return; + case 2: + this.readBytes(); + return; + case 5: + this.skipBytes(4); + return; + default: + throw new Error(`unsupported protobuf wire type ${wireType}`); + } + } + + private skipBytes(length: number): void { + if (this.offset + length > this.bytes.length) throw new Error("truncated protobuf fixed field"); + this.offset += length; + } +} + +function expectWireType(actual: number, expected: number, field: string): void { + if (actual !== expected) { + throw new Error(`${field} uses protobuf wire type ${actual}, expected ${expected}`); + } +} + +function visitFields( + bytes: Uint8Array, + visit: (reader: WireReader, field: number, wireType: number) => boolean, +): void { + const reader = new WireReader(bytes); + while (!reader.done) { + const { field, wireType } = reader.readTag(); + if (!visit(reader, field, wireType)) reader.skip(wireType); + } +} + +function signedInt64(value: bigint): number | string { + const signed = value >= 1n << 63n ? value - (1n << 64n) : value; + const number = Number(signed); + return Number.isSafeInteger(number) ? number : signed.toString(); +} + +function decodeArrayValue(bytes: Uint8Array, depth: number): OtlpAttributeValue[] { + const values: OtlpAttributeValue[] = []; + visitFields(bytes, (reader, field, wireType) => { + if (field !== 1) return false; + expectWireType(wireType, 2, "ArrayValue.values"); + values.push(decodeAnyValue(reader.readBytes(), depth + 1)); + return true; + }); + return values; +} + +function addAttribute( + attributes: Record, + [key, value]: readonly [string, OtlpAttributeValue], +): void { + if (FORBIDDEN_ATTRIBUTE_KEYS.has(key)) throw new Error(`forbidden OTLP attribute key ${key}`); + if (Object.hasOwn(attributes, key)) throw new Error(`duplicate OTLP attribute key ${key}`); + attributes[key] = value; +} + +function decodeKeyValueList(bytes: Uint8Array, depth: number): Record { + const attributes = emptyAttributeMap(); + visitFields(bytes, (reader, field, wireType) => { + if (field !== 1) return false; + expectWireType(wireType, 2, "KeyValueList.values"); + addAttribute(attributes, decodeKeyValue(reader.readBytes(), depth + 1)); + return true; + }); + return attributes; +} + +function decodeAnyValue(bytes: Uint8Array, depth = 0): OtlpAttributeValue { + if (depth > MAX_ANY_VALUE_DEPTH) throw new Error("OTLP AnyValue nesting exceeds 16 levels"); + let value: OtlpAttributeValue | undefined; + visitFields(bytes, (reader, field, wireType) => { + if (field < 1 || field > 7) return false; + if (value !== undefined) throw new Error("OTLP AnyValue contains multiple value variants"); + switch (field) { + case 1: + expectWireType(wireType, 2, "AnyValue.string_value"); + value = reader.readString(); + return true; + case 2: + expectWireType(wireType, 0, "AnyValue.bool_value"); + value = reader.readVarint() !== 0n; + return true; + case 3: + expectWireType(wireType, 0, "AnyValue.int_value"); + value = signedInt64(reader.readVarint()); + return true; + case 4: + expectWireType(wireType, 1, "AnyValue.double_value"); + value = reader.readDouble(); + return true; + case 5: + expectWireType(wireType, 2, "AnyValue.array_value"); + value = decodeArrayValue(reader.readBytes(), depth); + return true; + case 6: + expectWireType(wireType, 2, "AnyValue.kvlist_value"); + value = decodeKeyValueList(reader.readBytes(), depth); + return true; + case 7: + expectWireType(wireType, 2, "AnyValue.bytes_value"); + value = Buffer.from(reader.readBytes()).toString("base64"); + return true; + default: + return false; + } + }); + return value ?? null; +} + +function decodeKeyValue(bytes: Uint8Array, depth = 0): readonly [string, OtlpAttributeValue] { + let key: string | undefined; + let value: OtlpAttributeValue = null; + visitFields(bytes, (reader, field, wireType) => { + if (field === 1) { + expectWireType(wireType, 2, "KeyValue.key"); + key = reader.readString(); + return true; + } + if (field === 2) { + expectWireType(wireType, 2, "KeyValue.value"); + value = decodeAnyValue(reader.readBytes(), depth); + return true; + } + return false; + }); + if (key === undefined || key.length === 0) throw new Error("OTLP KeyValue is missing its key"); + return [key, value]; +} + +function decodeAttributes( + bytes: Uint8Array, + fieldNumber: number, +): Record { + const attributes = emptyAttributeMap(); + visitFields(bytes, (reader, field, wireType) => { + if (field !== fieldNumber) return false; + expectWireType(wireType, 2, "repeated KeyValue attribute"); + addAttribute(attributes, decodeKeyValue(reader.readBytes())); + return true; + }); + return attributes; +} + +function decodeSpan(bytes: Uint8Array): Omit { + let name = ""; + visitFields(bytes, (reader, field, wireType) => { + if (field !== 5) return false; + expectWireType(wireType, 2, "Span.name"); + name = reader.readString(); + return true; + }); + return { name, attributes: decodeAttributes(bytes, 9) }; +} + +function decodeScopeSpans(bytes: Uint8Array): Omit[] { + const spans: Omit[] = []; + visitFields(bytes, (reader, field, wireType) => { + if (field !== 2) return false; + expectWireType(wireType, 2, "ScopeSpans.spans"); + spans.push(decodeSpan(reader.readBytes())); + return true; + }); + return spans; +} + +function decodeResourceSpans(bytes: Uint8Array): DecodedOtlpSpan[] { + let resourceAttributes = emptyAttributeMap(); + const spans: Omit[] = []; + visitFields(bytes, (reader, field, wireType) => { + if (field === 1) { + expectWireType(wireType, 2, "ResourceSpans.resource"); + resourceAttributes = decodeAttributes(reader.readBytes(), 1); + return true; + } + if (field === 2) { + expectWireType(wireType, 2, "ResourceSpans.scope_spans"); + spans.push(...decodeScopeSpans(reader.readBytes())); + return true; + } + return false; + }); + return spans.map((span) => ({ ...span, resourceAttributes })); +} + +export function decodeExportTraceServiceRequest(bytes: Uint8Array): DecodedOtlpSpan[] { + const spans: DecodedOtlpSpan[] = []; + visitFields(bytes, (reader, field, wireType) => { + if (field !== 1) return false; + expectWireType(wireType, 2, "ExportTraceServiceRequest.resource_spans"); + spans.push(...decodeResourceSpans(reader.readBytes())); + return true; + }); + if (spans.length === 0) throw new Error("OTLP request contains no spans"); + return spans; +} + +export function otlpValueContains(value: OtlpAttributeValue | undefined, marker: string): boolean { + if (typeof value === "string") return value.includes(marker); + if (Array.isArray(value)) return value.some((item) => otlpValueContains(item, marker)); + if (value !== null && typeof value === "object") { + return Object.values(value).some((item) => otlpValueContains(item, marker)); + } + return false; +} diff --git a/test/e2e/manifests/langchain-deepagents-code-nvidia.yaml b/test/e2e/manifests/langchain-deepagents-code-nvidia.yaml index 826c674cd1..d2c1cf0018 100644 --- a/test/e2e/manifests/langchain-deepagents-code-nvidia.yaml +++ b/test/e2e/manifests/langchain-deepagents-code-nvidia.yaml @@ -25,6 +25,7 @@ spec: terminalRuntime: true interactiveCommand: dcode headlessCommand: dcode -n + observability: true remoteDeepAgentsSandboxes: false mcpAutoLoad: false state: diff --git a/test/e2e/support/deepagents-observability-contract-fixtures.ts b/test/e2e/support/deepagents-observability-contract-fixtures.ts new file mode 100644 index 0000000000..904b9085e0 --- /dev/null +++ b/test/e2e/support/deepagents-observability-contract-fixtures.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import http from "node:http"; + +import type { StartedOtlpCaptureServers } from "../live/deepagents-otlp-capture-server.ts"; +import type { OtlpAttributeValue } from "../live/otlp-trace-decoder.ts"; + +export const SERVICE_NAME = "nemoclaw-langchain-deepagents-code"; + +export type TestSpan = { + attributes: Record; + name: string; +}; + +function varint(value: number): Buffer { + let remaining = BigInt(value); + const bytes: number[] = []; + do { + let byte = Number(remaining & 0x7fn); + remaining >>= 7n; + if (remaining > 0n) byte |= 0x80; + bytes.push(byte); + } while (remaining > 0n); + return Buffer.from(bytes); +} + +function field(fieldNumber: number, wireType: number, value: Buffer): Buffer { + return Buffer.concat([varint((fieldNumber << 3) | wireType), value]); +} + +function bytesField(fieldNumber: number, value: Buffer): Buffer { + return field(fieldNumber, 2, Buffer.concat([varint(value.length), value])); +} + +function stringField(fieldNumber: number, value: string): Buffer { + return bytesField(fieldNumber, Buffer.from(value)); +} + +function anyValue(value: OtlpAttributeValue): Buffer { + if (value === null) return Buffer.alloc(0); + if (typeof value === "string") return stringField(1, value); + if (typeof value === "boolean") return field(2, 0, varint(value ? 1 : 0)); + if (typeof value === "number") { + if (Number.isSafeInteger(value) && value >= 0) return field(3, 0, varint(value)); + const double = Buffer.alloc(8); + double.writeDoubleLE(value); + return field(4, 1, double); + } + if (Array.isArray(value)) { + const array = Buffer.concat(value.map((item) => bytesField(1, anyValue(item)))); + return bytesField(5, array); + } + const entries = Object.entries(value).map(([key, item]) => bytesField(1, keyValue(key, item))); + return bytesField(6, Buffer.concat(entries)); +} + +function keyValue(key: string, value: OtlpAttributeValue): Buffer { + return Buffer.concat([stringField(1, key), bytesField(2, anyValue(value))]); +} + +function attributes(fieldNumber: number, values: Record): Buffer[] { + return Object.entries(values).map(([key, value]) => + bytesField(fieldNumber, keyValue(key, value)), + ); +} + +function spanBytes(span: TestSpan): Buffer { + return Buffer.concat([stringField(5, span.name), ...attributes(9, span.attributes)]); +} + +export function traceRequest(spans: readonly TestSpan[], serviceName = SERVICE_NAME): Buffer { + const resource = Buffer.concat(attributes(1, { "service.name": serviceName })); + const scopeSpans = Buffer.concat(spans.map((span) => bytesField(2, spanBytes(span)))); + const resourceSpans = Buffer.concat([bytesField(1, resource), bytesField(2, scopeSpans)]); + return bytesField(1, resourceSpans); +} + +export function request( + port: number, + headers: Record, + body = "", +): Promise { + return new Promise((resolve) => { + const client = http.request( + { host: "127.0.0.1", method: "POST", path: "/v1/traces", port, headers }, + (response) => { + response.resume(); + response.on("end", () => resolve(response.statusCode ?? null)); + }, + ); + client.on("error", () => resolve(null)); + if (body) client.write(body); + client.end(); + }); +} + +export function pendingRequest( + port: number, + contentLength: number, +): { + complete(body: string): void; + destroy(): void; + status: Promise; +} { + let finish: (status: number | null) => void = () => {}; + const status = new Promise((resolve) => { + finish = resolve; + }); + const client = http.request( + { + host: "127.0.0.1", + method: "POST", + path: "/v1/traces", + port, + headers: { + "content-length": String(contentLength), + "content-type": "application/x-protobuf", + }, + }, + (response) => { + response.resume(); + response.on("end", () => finish(response.statusCode ?? null)); + }, + ); + client.on("error", () => finish(null)); + client.flushHeaders(); + return { + complete: (body) => client.end(body), + destroy: () => client.destroy(), + status, + }; +} + +export async function waitForMetadata(captureDir: string, count: number): Promise { + for (let attempt = 0; attempt < 50; attempt += 1) { + if (fs.readdirSync(captureDir).filter((name) => name.endsWith(".json")).length >= count) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`capture server did not write ${count} metadata files`); +} + +export async function waitForReservedBytes( + started: Pick, + expectedBytes: number, +): Promise { + for (let attempt = 0; attempt < 50; attempt += 1) { + if (started.snapshot().reservedBytes === expectedBytes) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + if (started.snapshot().reservedBytes === expectedBytes) return; + throw new Error(`capture server did not reserve ${expectedBytes} bytes`); +} diff --git a/test/e2e/support/deepagents-observability-contract.test.ts b/test/e2e/support/deepagents-observability-contract.test.ts new file mode 100644 index 0000000000..f664f10432 --- /dev/null +++ b/test/e2e/support/deepagents-observability-contract.test.ts @@ -0,0 +1,441 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + assertDeepAgentsTraceContract, + hasConfirmedOpenShellPolicyDenial, + observabilityPresetState, +} from "../live/deepagents-observability-contract.ts"; +import { + isPrivateBridgeIpv4, + startOtlpCaptureServers, +} from "../live/deepagents-otlp-capture-server.ts"; +import { + decodeExportTraceServiceRequest, + type OtlpAttributeValue, +} from "../live/otlp-trace-decoder.ts"; +import { + pendingRequest, + request, + SERVICE_NAME, + type TestSpan, + traceRequest, + waitForMetadata, + waitForReservedBytes, +} from "./deepagents-observability-contract-fixtures.ts"; + +const DIRECT_PROMPT = "DIRECT_PROMPT"; +const DIRECT_RESPONSE = "DIRECT_RESPONSE"; +const LOGIN_PROMPT = "LOGIN_PROMPT"; +const LOGIN_RESPONSE = "LOGIN_RESPONSE"; +const TOOL_NAME = "nemoclaw_otlp_e2e_tool"; +const TOOL_ARGUMENT = "TOOL_ARGUMENT"; +const TOOL_RESULT = "TOOL_RESULT"; +const AMBIENT_CANARY = "AMBIENT_CANARY"; + +function validSpans(): TestSpan[] { + return [ + { + name: "direct model", + attributes: { + "openinference.span.kind": "LLM", + "input.value": JSON.stringify({ prompt: DIRECT_PROMPT, requested: DIRECT_RESPONSE }), + "output.value": JSON.stringify({ content: DIRECT_RESPONSE }), + }, + }, + { + name: "login model", + attributes: { + "openinference.span.kind": "LLM", + "llm.input_messages": JSON.stringify([{ content: LOGIN_PROMPT }]), + "llm.output_messages": JSON.stringify([{ content: LOGIN_RESPONSE }]), + }, + }, + { + name: "deterministic tool", + attributes: { + "openinference.span.kind": "TOOL", + "tool.name": TOOL_NAME, + "tool.parameters": JSON.stringify({ command: TOOL_ARGUMENT }), + "output.value": JSON.stringify({ stdout: TOOL_RESULT }), + }, + }, + ]; +} + +const expectations = { + ambientCanary: AMBIENT_CANARY, + serviceName: SERVICE_NAME, + llmExchanges: [ + { label: "direct", promptMarker: DIRECT_PROMPT, responseMarker: DIRECT_RESPONSE }, + { label: "login", promptMarker: LOGIN_PROMPT, responseMarker: LOGIN_RESPONSE }, + ], + tool: { argumentMarker: TOOL_ARGUMENT, name: TOOL_NAME, resultMarker: TOOL_RESULT }, +} as const; + +describe("Deep Agents OTLP trace contract", () => { + it("decodes the stable OTLP trace fields and recursive AnyValue shapes", () => { + const body = traceRequest([ + { + name: "nested attributes", + attributes: { + "openinference.span.kind": "LLM", + nested: { enabled: true, items: ["one", 2, { leaf: "three" }] }, + }, + }, + ]); + + expect(decodeExportTraceServiceRequest(body)).toEqual([ + { + name: "nested attributes", + resourceAttributes: { "service.name": SERVICE_NAME }, + attributes: { + "openinference.span.kind": "LLM", + nested: { enabled: true, items: ["one", 2, { leaf: "three" }] }, + }, + }, + ]); + }); + + it("requires input and output markers on the same managed LLM and TOOL spans", () => { + expect(assertDeepAgentsTraceContract([traceRequest(validSpans())], expectations)).toEqual({ + requestCount: 1, + spanCount: 3, + }); + + const misplacedResponse = validSpans(); + misplacedResponse[0] = { + ...misplacedResponse[0], + attributes: { + ...misplacedResponse[0].attributes, + "output.value": "unrelated output", + }, + }; + expect(() => + assertDeepAgentsTraceContract([traceRequest(misplacedResponse)], expectations), + ).toThrow(/direct prompt and response markers were not associated on one managed LLM span/); + + const misplacedToolArgument = validSpans(); + misplacedToolArgument[0].attributes["input.value"] = JSON.stringify({ + prompt: DIRECT_PROMPT, + requested: DIRECT_RESPONSE, + unrelatedToolArgument: TOOL_ARGUMENT, + }); + misplacedToolArgument[2] = { + ...misplacedToolArgument[2], + attributes: { ...misplacedToolArgument[2].attributes, "tool.parameters": "unrelated" }, + }; + expect(() => + assertDeepAgentsTraceContract([traceRequest(misplacedToolArgument)], expectations), + ).toThrow(/not associated on one managed TOOL span/); + }); + + it("fails closed on malformed requests, wrong service identity, and ambient canaries", () => { + expect(() => + assertDeepAgentsTraceContract([Buffer.from([0x0a, 0x05, 0x01])], expectations), + ).toThrow(/not a valid ExportTraceServiceRequest/); + expect(() => + assertDeepAgentsTraceContract( + [traceRequest(validSpans(), "unmanaged-service")], + expectations, + ), + ).toThrow(/not associated on one managed LLM span/); + expect(() => + assertDeepAgentsTraceContract( + [traceRequest([...validSpans(), { name: AMBIENT_CANARY, attributes: {} }])], + expectations, + ), + ).toThrow(/ambient exporter configuration reached OTLP/); + }); + + it("rejects prototype-sensitive attribute keys and excessive AnyValue nesting", () => { + const hostileAttributes = Object.fromEntries([["__proto__", "hostile"]]) as Record< + string, + OtlpAttributeValue + >; + expect(() => + decodeExportTraceServiceRequest( + traceRequest([{ name: "hostile attribute", attributes: hostileAttributes }]), + ), + ).toThrow(/forbidden OTLP attribute key __proto__/); + + let nested: OtlpAttributeValue = "leaf"; + for (let depth = 0; depth < 18; depth += 1) nested = [nested]; + expect(() => + decodeExportTraceServiceRequest( + traceRequest([{ name: "deep attribute", attributes: { nested } }]), + ), + ).toThrow(/AnyValue nesting exceeds 16 levels/); + }); +}); + +describe("Deep Agents observability policy proof", () => { + it("accepts only the exact active policy-list state", () => { + expect( + observabilityPresetState( + " ● observability-otlp-local [from balanced tier] — host-local OTLP export\n", + ), + ).toBe("active"); + expect( + observabilityPresetState( + " ○ observability-otlp-local — host-local OTLP export (recorded locally, not active on gateway)\n", + ), + ).toBe("drift"); + expect(observabilityPresetState("observability-otlp-local is documented here\n")).toBe( + "missing", + ); + }); + + it("distinguishes confirmed OpenShell denials from DNS and transport failures", () => { + expect( + hasConfirmedOpenShellPolicyDenial( + "[1783046573.602] [sandbox] [OCSF ] NET:OPEN [MED] DENIED /usr/bin/curl(1) -> example.com:443 [reason:not allowed by any policy]", + ), + ).toBe(true); + expect( + hasConfirmedOpenShellPolicyDenial( + 'proxy: {"error":"policy_denied","detail":"CONNECT example.com:443 not allowed by any policy"}', + ), + ).toBe(true); + expect( + hasConfirmedOpenShellPolicyDenial( + "nemoclaw: recent network policy denial detected for example.com:443 inside sandbox 'dcode-test'.", + ), + ).toBe(true); + expect(hasConfirmedOpenShellPolicyDenial("URLError: Name or service not known")).toBe(false); + expect(hasConfirmedOpenShellPolicyDenial("curl: (7) Connection refused")).toBe(false); + expect(hasConfirmedOpenShellPolicyDenial("curl: (28) Operation timed out")).toBe(false); + }); + + it("runs the policy parser and denial classifier through the live tsx command path", () => { + const tsx = path.join(process.cwd(), "node_modules", ".bin", "tsx"); + const helper = path.join( + process.cwd(), + "test", + "e2e", + "live", + "deepagents-observability-contract.ts", + ); + const active = spawnSync(tsx, [helper, "policy-state"], { + encoding: "utf8", + env: { PATH: process.env.PATH }, + input: " ● observability-otlp-local [from balanced tier] — local OTLP\n", + }); + expect(active.status, active.stderr).toBe(0); + expect(active.stdout.trim()).toBe("active"); + + const denial = spawnSync(tsx, [helper, "denial-state"], { + encoding: "utf8", + env: { PATH: process.env.PATH }, + input: + "[1.0] [sandbox] [OCSF ] NET:OPEN [MED] DENIED /usr/bin/curl(1) -> example.com:443 [reason:not allowed by any policy]\n", + }); + expect(denial.status, denial.stderr).toBe(0); + expect(denial.stdout.trim()).toBe("policy-denied"); + + const proxyDenial = spawnSync(tsx, [helper, "denial-state"], { + encoding: "utf8", + env: { PATH: process.env.PATH }, + input: + 'FAILED:HTTPError:HTTP Error 403: Forbidden:{"error":"policy_denied","detail":"POST example.com:4318/v1/traces not permitted by policy"}\n', + }); + expect(proxyDenial.status, proxyDenial.stderr).toBe(0); + expect(proxyDenial.stdout.trim()).toBe("policy-denied"); + }); +}); + +describe("bounded private OTLP capture server", () => { + it("accepts only private bridge addresses unless a hermetic test opts into loopback", () => { + expect(isPrivateBridgeIpv4("10.1.2.3")).toBe(true); + expect(isPrivateBridgeIpv4("172.31.0.1")).toBe(true); + expect(isPrivateBridgeIpv4("192.168.1.1")).toBe(true); + expect(isPrivateBridgeIpv4("127.0.0.1")).toBe(false); + expect(isPrivateBridgeIpv4("127.0.0.1", true)).toBe(true); + expect(isPrivateBridgeIpv4("0.0.0.0", true)).toBe(false); + expect(isPrivateBridgeIpv4("8.8.8.8", true)).toBe(false); + }); + + it("bounds per-request, aggregate, and request-count capture volume", async () => { + const captureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-otlp-capture-test-")); + const started = await startOtlpCaptureServers({ + allowLoopback: true, + bindIp: "127.0.0.1", + captureDir, + collectorPort: 0, + decoyPort: 0, + maxCaptureBytes: 16, + maxCaptureRequests: 4, + maxBodyBytes: 16, + }); + try { + expect( + await request( + started.collectorPort, + { "content-length": "4", "content-type": "application/x-protobuf" }, + "test", + ), + ).toBe(200); + const forbiddenHeaderStatus = await request( + started.collectorPort, + { + authorization: "Bearer MUST_NOT_REACH_CAPTURE_METADATA", + "content-length": "4", + "content-type": "application/x-protobuf", + }, + "test", + ); + expect([400, null]).toContain(forbiddenHeaderStatus); + const aggregateStatus = await request( + started.collectorPort, + { "content-length": "13", "content-type": "application/x-protobuf" }, + "1234567890123", + ); + expect([507, null]).toContain(aggregateStatus); + const oversizedStatus = await request(started.collectorPort, { + "content-length": "17", + "content-type": "application/x-protobuf", + }); + expect([413, null]).toContain(oversizedStatus); + const overCountStatus = await request( + started.collectorPort, + { "content-length": "4", "content-type": "application/x-protobuf" }, + "test", + ); + expect([429, null]).toContain(overCountStatus); + await waitForMetadata(captureDir, 5); + + const metadata = fs + .readdirSync(captureDir) + .filter((name) => name.endsWith(".json")) + .sort() + .map((name) => JSON.parse(fs.readFileSync(path.join(captureDir, name), "utf8"))); + expect(metadata).toMatchObject([ + { + accepted: true, + contentType: "application/x-protobuf", + method: "POST", + path: "/v1/traces", + rejection: null, + }, + { + accepted: false, + contentType: null, + method: null, + path: null, + rejection: "forbidden exporter header", + }, + { + accepted: false, + rejection: "aggregate captured bodies exceed bound", + }, + { + accepted: false, + rejection: "declared body exceeds capture bound", + }, + { + accepted: false, + rejection: "capture request count exceeds bound", + }, + ]); + expect(JSON.stringify(metadata)).not.toContain("MUST_NOT_REACH_CAPTURE_METADATA"); + const bodyFiles = fs + .readdirSync(captureDir) + .filter((name) => name.endsWith(".body")) + .sort(); + expect(bodyFiles.map((name) => fs.statSync(path.join(captureDir, name)).size)).toEqual([ + 4, 0, 0, 0, 0, + ]); + } finally { + await started.close(); + fs.rmSync(captureDir, { force: true, recursive: true }); + } + }); + + it("reserves declared bytes before admitting concurrent request bodies", async () => { + const captureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-otlp-reservation-test-")); + const started = await startOtlpCaptureServers({ + allowLoopback: true, + bindIp: "127.0.0.1", + captureDir, + collectorPort: 0, + decoyPort: 0, + maxBodyBytes: 16, + maxCaptureBytes: 16, + maxCaptureRequests: 10, + }); + const first = pendingRequest(started.collectorPort, 12); + try { + await waitForReservedBytes(started, 12); + expect(started.snapshot()).toMatchObject({ capturedBytes: 0, reservedBytes: 12 }); + + const rejectedStatus = await request( + started.collectorPort, + { "content-length": "12", "content-type": "application/x-protobuf" }, + "abcdefghijkl", + ); + expect([507, null]).toContain(rejectedStatus); + expect(started.snapshot()).toMatchObject({ capturedBytes: 0, reservedBytes: 12 }); + + first.complete("abcdefghijkl"); + expect(await first.status).toBe(200); + await waitForMetadata(captureDir, 2); + expect(started.snapshot()).toMatchObject({ capturedBytes: 12, reservedBytes: 0 }); + } finally { + first.destroy(); + await started.close(); + fs.rmSync(captureDir, { force: true, recursive: true }); + } + }); + + it("releases reserved bytes when a client disconnects before its body completes (#3915)", async () => { + const captureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-otlp-abort-test-")); + const started = await startOtlpCaptureServers({ + allowLoopback: true, + bindIp: "127.0.0.1", + captureDir, + collectorPort: 0, + decoyPort: 0, + maxBodyBytes: 16, + maxCaptureBytes: 16, + maxCaptureRequests: 10, + }); + const partial = pendingRequest(started.collectorPort, 12); + try { + await waitForReservedBytes(started, 12); + partial.destroy(); + await waitForMetadata(captureDir, 1); + + expect(started.snapshot()).toEqual({ + capturedBytes: 0, + requestCount: 1, + reservedBytes: 0, + }); + const metadata = fs + .readdirSync(captureDir) + .filter((name) => name.endsWith(".json")) + .map((name) => JSON.parse(fs.readFileSync(path.join(captureDir, name), "utf8"))); + expect(metadata).toEqual([ + { + accepted: false, + contentType: null, + method: null, + path: null, + port: started.collectorPort, + rejection: "request body aborted", + }, + ]); + const bodyFiles = fs.readdirSync(captureDir).filter((name) => name.endsWith(".body")); + expect(bodyFiles.map((name) => fs.statSync(path.join(captureDir, name)).size)).toEqual([0]); + } finally { + partial.destroy(); + await started.close(); + fs.rmSync(captureDir, { force: true, recursive: true }); + } + }); +}); diff --git a/test/e2e/support/e2e-manifests.test.ts b/test/e2e/support/e2e-manifests.test.ts index 32462bea43..8a2a3bc719 100644 --- a/test/e2e/support/e2e-manifests.test.ts +++ b/test/e2e/support/e2e-manifests.test.ts @@ -79,4 +79,14 @@ describe("NemoClawInstance manifests", () => { expect(manifest.spec.onboarding.agent).toBe("openclaw"); expect(manifest.spec.onboarding.provider).toBe("nvidia"); }); + + it("declares observability on the canonical Deep Agents Code live target", () => { + const target = listTargets().find( + (entry) => entry.id === "ubuntu-repo-cloud-langchain-deepagents-code", + ); + + expect(target).toBeTruthy(); + const manifest = loadManifest(path.join(REPO_ROOT, target!.manifestPath as string)).document; + expect(manifest.spec.onboarding.features?.observability).toBe(true); + }); }); diff --git a/test/e2e/support/e2e-phase-onboarding.test.ts b/test/e2e/support/e2e-phase-onboarding.test.ts index 6c3ac80fcc..b42cf4d267 100644 --- a/test/e2e/support/e2e-phase-onboarding.test.ts +++ b/test/e2e/support/e2e-phase-onboarding.test.ts @@ -154,6 +154,41 @@ describe("onboarding phase fixture", () => { ]); }); + it("opts the canonical Deep Agents Code target into composed observability", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const secrets = new FakeSecrets({ NVIDIA_INFERENCE_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from(ready({ onboarding: "cloud-langchain-deepagents-code" }), { + sandboxName: "e2e-ubuntu-repo-cloud-langchain-deepagents-code", + }); + + expect(instance).toMatchObject({ + agent: "langchain-deepagents-code", + sandboxName: "e2e-ubuntu-repo-cloud-langchain-deepagents-code", + }); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: [ + "onboard", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + "--observability", + ], + options: { + artifactName: "onboard-cloud-langchain-deepagents-code", + env: expect.objectContaining({ + NEMOCLAW_AGENT: "langchain-deepagents-code", + NVIDIA_INFERENCE_API_KEY: "secret-token", + }), + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }); + }); + it("fails cloud OpenClaw onboarding on non-zero exit", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(42, "provider rejected credential")); diff --git a/test/e2e/support/platform-parity-cloud-experimental.test.ts b/test/e2e/support/platform-parity-cloud-experimental.test.ts index 0fa4e277a5..639e5bcd17 100644 --- a/test/e2e/support/platform-parity-cloud-experimental.test.ts +++ b/test/e2e/support/platform-parity-cloud-experimental.test.ts @@ -3,9 +3,17 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; + +const execSandboxMock = vi.hoisted(() => vi.fn(async () => {})); +vi.mock("../../../src/lib/actions/sandbox/exec", () => ({ + execSandbox: execSandboxMock, +})); + +import SandboxExecCommand from "../../../src/commands/sandbox/exec.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS } from "../live/cloud-experimental-check-list.ts"; import { @@ -32,6 +40,135 @@ function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeR } describe("P0-E cloud-experimental parity guardrails", () => { + it("preserves the repeated env-unset pairs from the failed observability invocation", async () => { + await SandboxExecCommand.run( + [ + "deepagents-sandbox", + "--", + "env", + "-u", + "ALL_PROXY", + "-u", + "HTTPS_PROXY", + "-u", + "HTTP_PROXY", + "-u", + "all_proxy", + "-u", + "https_proxy", + "-u", + "http_proxy", + "/opt/venv/bin/python3", + "-I", + "-c", + "pass", + ], + process.cwd(), + ); + + expect(execSandboxMock).toHaveBeenCalledWith( + "deepagents-sandbox", + [ + "env", + "-u", + "ALL_PROXY", + "-u", + "HTTPS_PROXY", + "-u", + "HTTP_PROXY", + "-u", + "all_proxy", + "-u", + "https_proxy", + "-u", + "http_proxy", + "/opt/venv/bin/python3", + "-I", + "-c", + "pass", + ], + { workdir: undefined, tty: null, timeoutSeconds: undefined }, + ); + }); + + it("routes the live OTLP probe through managed Python and the OpenShell proxy", () => { + const script = fs.readFileSync( + path.join( + process.cwd(), + "test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh", + ), + "utf8", + ); + + expect(script).toMatch( + /grep -Fq 'CAPTURE_READY:'[\s\S]*COLLECTOR_PORT}\/health[\s\S]*DECOY_PORT}\/health/, + ); + expect(script).toContain("urllib.request.urlopen(request, timeout=10)"); + expect(script).toContain("except urllib.error.HTTPError as error:"); + expect(script).toContain('body = error.read(512).decode("utf-8", "replace")'); + expect(script).not.toContain("urllib.request.ProxyHandler({})"); + expect(script).not.toContain("os.environ.pop"); + expect(script).toMatch(/\"\$CLI\" \"\$SANDBOX_NAME\" exec -- \\\n\s+\/opt\/venv\/bin\/python3/); + expect(script).not.toContain("env -u ALL_PROXY"); + expect(script.match(/--noproxy '\*'/g)).toHaveLength(2); + expect(script).toContain("/usr/bin/curl --fail-with-body -sS"); + expect(script).toMatch( + /run_deterministic_tool_trace\(\)[\s\S]*"\$CLI" "\$SANDBOX_NAME" exec --[\s\S]*\/opt\/venv\/bin\/python3/, + ); + }); + + it("skips the DCode observability probe before host prerequisites on other agents", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-observability-skip-")); + try { + const invocationLog = path.join(tempDir, "openshell-args.txt"); + const openshell = path.join(tempDir, "openshell"); + fs.writeFileSync( + openshell, + '#!/bin/sh\nprintf \'%s\\n\' "$@" > "$NEMOCLAW_FAKE_OPENSHELL_LOG"\nexit 1\n', + { mode: 0o755 }, + ); + const result = spawnSync( + "bash", + [ + path.join( + process.cwd(), + "test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh", + ), + ], + { + encoding: "utf8", + env: { + NEMOCLAW_CLI_BIN: path.join(tempDir, "missing-nemoclaw"), + NEMOCLAW_FAKE_OPENSHELL_LOG: invocationLog, + PATH: `${tempDir}:${process.env.PATH ?? "/usr/bin:/bin"}`, + REPO: path.join(tempDir, "missing-repo"), + SANDBOX_NAME: "openclaw-sandbox", + }, + }, + ); + + expect(result.status).toBe(0); + expect(`${result.stdout}\n${result.stderr}`).toContain( + "11-deepagents-code-observability: SKIP: sandbox openclaw-sandbox is not a Deep Agents Code sandbox", + ); + expect(fs.readFileSync(invocationLog, "utf8")).toBe( + [ + "sandbox", + "exec", + "--name", + "openclaw-sandbox", + "--", + "bash", + "-c", + "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1", + "", + ].join("\n"), + ); + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); + it("fails required Deep Agents cloud-experimental checks when scripts print SKIP", () => { expect(() => assertRequiredCloudExperimentalResult( @@ -141,6 +278,7 @@ describe("P0-E cloud-experimental parity guardrails", () => { "test/e2e/e2e-cloud-experimental/checks/08-deepagents-code-secret-boundary.sh", "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", + "test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh", ]); for (const scriptPath of DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS) { @@ -160,6 +298,11 @@ describe("P0-E cloud-experimental parity guardrails", () => { "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", ), ).toBe(180_000); + expect( + cloudExperimentalCheckTimeoutMs( + "test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh", + ), + ).toBe(8 * 60_000); }); it("documents Deep Agents check scripts in generated launch/QA evidence", () => { diff --git a/test/fixtures/deepagents-observability-harness.py b/test/fixtures/deepagents-observability-harness.py new file mode 100644 index 0000000000..65bba7bf52 --- /dev/null +++ b/test/fixtures/deepagents-observability-harness.py @@ -0,0 +1,1258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Isolated contract harness for managed Deep Agents Code observability.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import inspect +import io +import json +import logging +import os +import sys +import types +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +SECRET = "NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL" +DROPPED_MODEL_SETTINGS = "NEMOCLAW-DROPPED-MODEL-SETTINGS" +DROPPED_RESPONSE_FORMAT = "NEMOCLAW-DROPPED-RESPONSE-FORMAT" +DROPPED_TOOL_SCHEMA = "NEMOCLAW-DROPPED-TOOL-SCHEMA" +UNSAFE_RELAY_FALLBACK = "NEMOCLAW-UNSAFE-RELAY-FALLBACK" +LOG_HEADER_SECRET = "NEMOCLAW-OTEL-HEADER-CANARY" +LOG_CERTIFICATE_SECRET = "NEMOCLAW-OTEL-CERTIFICATE-CANARY" +LOG_CLIENT_KEY_SECRET = "NEMOCLAW-OTEL-CLIENT-KEY-CANARY" +_RELAY_OBSERVED_ERRORS: list[dict[str, Any]] = [] +_RELAY_OBSERVED_MODEL_NAMES: list[str] = [] +_RELAY_OBSERVED_TOOL_NAMES: list[str] = [] +_RELAY_LLM_FAILURE_MODE: list[str | None] = [None] +_RELAY_TOOL_FAILURE_MODE: list[str | None] = [None] + + +def _validate_relay_json(value: Any) -> None: + """Match the native JSON value domain in the pinned nemo-relay 0.4.0.""" + if value is None or type(value) is bool: + return + if type(value) is int: + if -(1 << 63) <= value <= (1 << 64) - 1: + return + raise ValueError("Relay JSON integer is out of range") + if type(value) is float: + return + if type(value) is str: + value.encode("utf-8", errors="strict") + return + if type(value) is list: + for item in value: + _validate_relay_json(item) + return + if type(value) is dict: + for key, item in value.items(): + if type(key) is not str: + raise ValueError("Relay JSON object key is not a string") + key.encode("utf-8", errors="strict") + _validate_relay_json(item) + return + raise ValueError("Relay JSON value has an unsupported type") + + +class _Guardrails: + def __init__(self) -> None: + self.registered: dict[str, Any] = {} + self.deregistered: list[str] = [] + + def _register(self, kind: str, name: str, priority: int, callback: Any) -> None: + self.registered[kind] = { + "name": name, + "priority": priority, + "callback": callback, + } + + def register_llm_sanitize_request( + self, name: str, priority: int, callback: Any + ) -> None: + self._register("llm_request", name, priority, callback) + + def register_llm_sanitize_response( + self, name: str, priority: int, callback: Any + ) -> None: + self._register("llm_response", name, priority, callback) + + def register_tool_sanitize_request( + self, name: str, priority: int, callback: Any + ) -> None: + self._register("tool_request", name, priority, callback) + + def register_tool_sanitize_response( + self, name: str, priority: int, callback: Any + ) -> None: + self._register("tool_response", name, priority, callback) + + def _deregister(self, kind: str, name: str) -> bool: + self.deregistered.append(f"{kind}:{name}") + return True + + def deregister_llm_sanitize_request(self, name: str) -> bool: + return self._deregister("llm_request", name) + + def deregister_llm_sanitize_response(self, name: str) -> bool: + return self._deregister("llm_response", name) + + def deregister_tool_sanitize_request(self, name: str) -> bool: + return self._deregister("tool_request", name) + + def deregister_tool_sanitize_response(self, name: str) -> bool: + return self._deregister("tool_response", name) + + +class _SubscriberCollection: + def __init__(self, *, fail_flush: bool) -> None: + self.fail_flush = fail_flush + self.flush_calls = 0 + + def flush(self) -> None: + self.flush_calls += 1 + if self.fail_flush: + raise RuntimeError("collector flush unavailable") + + +class _OpenInferenceConfig: + def __init__(self) -> None: + self.transport = None + self.endpoint = os.environ.get( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" + ) or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + self.headers = { + "ambient": os.environ.get("OTEL_EXPORTER_OTLP_TRACES_HEADERS") + or os.environ.get("OTEL_EXPORTER_OTLP_HEADERS", "") + } + self.service_name = None + self.timeout_millis = None + + +class _OpenInferenceSubscriber: + instances: list[_OpenInferenceSubscriber] = [] + fail_force_flush = False + fail_construct = False + fail_register = False + + def __init__(self, config: _OpenInferenceConfig) -> None: + if self.fail_construct: + raise RuntimeError("collector construction unavailable") + self.config = config + self.registered: list[str] = [] + self.force_flush_calls = 0 + self.deregistered: list[str] = [] + self.shutdown_calls = 0 + self.instances.append(self) + + def register(self, name: str) -> None: + if self.fail_register: + raise RuntimeError( + "subscriber registration failed: " + f"{os.environ.get('OTEL_EXPORTER_OTLP_HEADERS', '')}|" + f"{os.environ.get('OTEL_EXPORTER_OTLP_CERTIFICATE', '')}|" + f"{os.environ.get('OTEL_EXPORTER_OTLP_CLIENT_KEY', '')}" + ) + self.registered.append(name) + + def force_flush(self) -> None: + self.force_flush_calls += 1 + if self.fail_force_flush: + raise RuntimeError("collector unavailable") + + def deregister(self, name: str) -> None: + self.deregistered.append(name) + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + +class _LLMRequest: + def __init__(self, headers: dict[str, str], content: dict[str, Any]) -> None: + self.headers = headers + self.content = content + + +class _Scope: + def __init__(self) -> None: + self.records: list[dict[str, Any]] = [] + + def push(self, name: str, category: str, **kwargs: Any) -> str: + self.records.append( + {"operation": "push", "name": name, "category": category, **kwargs} + ) + return f"handle-{len(self.records)}" + + def pop(self, handle: str, **kwargs: Any) -> None: + self.records.append({"operation": "pop", "handle": handle, **kwargs}) + + def event(self, name: str, **kwargs: Any) -> None: + self.records.append({"operation": "event", "name": name, **kwargs}) + + +class _GraphCallbackHandler: + def __init__(self) -> None: + self.base_initialized = True + + +class _CallbackManager: + def __init__( + self, + handlers: list[Any], + inheritable_handlers: list[Any] | None = None, + parent_run_id: Any | None = None, + *, + tags: list[str] | None = None, + inheritable_tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + inheritable_metadata: dict[str, Any] | None = None, + ) -> None: + self.handlers = list(handlers) + self.inheritable_handlers = list(inheritable_handlers or ()) + self.parent_run_id = parent_run_id + self.tags = list(tags or ()) + self.inheritable_tags = list(inheritable_tags or ()) + self.metadata = dict(metadata or {}) + self.inheritable_metadata = dict(inheritable_metadata or {}) + + def copy(self) -> _CallbackManager: + return self.__class__( + handlers=self.handlers.copy(), + inheritable_handlers=self.inheritable_handlers.copy(), + parent_run_id=self.parent_run_id, + tags=self.tags.copy(), + inheritable_tags=self.inheritable_tags.copy(), + metadata=self.metadata.copy(), + inheritable_metadata=self.inheritable_metadata.copy(), + ) + + def add_handler(self, handler: Any, inherit: bool = True) -> None: + if handler not in self.handlers: + self.handlers.append(handler) + if inherit and handler not in self.inheritable_handlers: + self.inheritable_handlers.append(handler) + + +class _RelayWrappedError(RuntimeError): + pass + + +def _relay_wrapped_error(error: Exception) -> _RelayWrappedError: + _RELAY_OBSERVED_ERRORS.append( + { + "type": type(error).__name__, + "message": str(error), + "context_is_none": error.__context__ is None, + "cause_is_none": error.__cause__ is None, + } + ) + return _RelayWrappedError("relay wrapped callback failure") + + +async def _tool_execute(*, name: str, args: Any, func: Any, **_kwargs: Any) -> Any: + _RELAY_OBSERVED_TOOL_NAMES.append(name) + _validate_relay_json(args) + if _RELAY_TOOL_FAILURE_MODE[0] == "before": + raise RuntimeError("injected Relay tool failure before callback") + try: + result = func(args) + result = await result if inspect.isawaitable(result) else result + except Exception as error: + raise _relay_wrapped_error(error) from error + _validate_relay_json(result) + if _RELAY_TOOL_FAILURE_MODE[0] == "after": + raise RuntimeError("injected Relay tool failure after callback") + return result + + +def _run_sync(awaitable: Any) -> Any: + return asyncio.run(awaitable) + + +class _NemoRelayMiddleware: + def __init__(self, *, name: str) -> None: + self.name = name + + async def _llm_execute( + self, + model_name: str, + request: Any, + codec: Any, + response_codec: Any, + func: Any, + **_kwargs: Any, + ) -> Any: + _RELAY_OBSERVED_MODEL_NAMES.append(model_name) + del codec, response_codec + validate_payload = type(request) is _LLMRequest + if validate_payload: + _validate_relay_json(request.headers) + _validate_relay_json(request.content) + if _RELAY_LLM_FAILURE_MODE[0] == "before": + raise RuntimeError("injected Relay model failure before callback") + try: + result = await func(request) + except Exception as error: + raise _relay_wrapped_error(error) from error + if validate_payload: + _validate_relay_json(result) + if _RELAY_LLM_FAILURE_MODE[0] == "after": + raise RuntimeError("injected Relay model failure after callback") + return result + + def wrap_model_call(self, request: Any, handler: Any) -> Any: + async def call(inner_request: Any) -> Any: + return handler(inner_request) + + return _run_sync(self._llm_execute("model", request, None, None, call)) + + async def awrap_model_call(self, request: Any, handler: Any) -> Any: + async def call(inner_request: Any) -> Any: + return await handler(inner_request) + + return await self._llm_execute("model", request, None, None, call) + + def _prepare_tool_call(self, request: Any) -> tuple[Any, Any, str, Any]: + return None, object(), request.tool_call["name"], request.tool_call.get("args") or {} + + +class _AIMessage: + def __init__(self, *, content: str) -> None: + self.content = content + + +def _messages_to_dict(messages: list[_AIMessage]) -> list[dict[str, Any]]: + return [ + { + "type": "ai", + "data": { + "content": message.content, + "additional_kwargs": {}, + "response_metadata": {}, + "tool_calls": [], + "invalid_tool_calls": [], + }, + } + for message in messages + ] + + +def _install_stubs( + *, + fail_flush: bool = False, + fail_force_flush: bool = False, + fail_construct: bool = False, + fail_register: bool = False, +) -> tuple[types.ModuleType, _Guardrails, _SubscriberCollection, _Scope]: + _RELAY_OBSERVED_ERRORS.clear() + _RELAY_OBSERVED_MODEL_NAMES.clear() + _RELAY_OBSERVED_TOOL_NAMES.clear() + _RELAY_LLM_FAILURE_MODE[0] = None + _RELAY_TOOL_FAILURE_MODE[0] = None + guardrails = _Guardrails() + subscribers = _SubscriberCollection(fail_flush=fail_flush) + scope = _Scope() + _OpenInferenceSubscriber.instances = [] + _OpenInferenceSubscriber.fail_force_flush = fail_force_flush + _OpenInferenceSubscriber.fail_construct = fail_construct + _OpenInferenceSubscriber.fail_register = fail_register + + relay = types.ModuleType("nemo_relay") + relay.LLMRequest = _LLMRequest + relay.OpenInferenceConfig = _OpenInferenceConfig + relay.OpenInferenceSubscriber = _OpenInferenceSubscriber + relay.ScopeType = SimpleNamespace(Agent="agent") + relay.guardrails = guardrails + relay.subscribers = subscribers + relay.scope = scope + relay.tools = SimpleNamespace(execute=_tool_execute) + relay.typed = SimpleNamespace(tool_execute=_tool_execute) + + integrations = types.ModuleType("nemo_relay.integrations") + langchain_integration = types.ModuleType("nemo_relay.integrations.langchain") + langchain_integration.NemoRelayMiddleware = _NemoRelayMiddleware + relay_utils = types.ModuleType("nemo_relay.utils") + relay_utils.run_sync = _run_sync + relay.integrations = integrations + + langgraph = types.ModuleType("langgraph") + langgraph_callbacks = types.ModuleType("langgraph.callbacks") + langgraph_callbacks.GraphCallbackHandler = _GraphCallbackHandler + + langchain_core = types.ModuleType("langchain_core") + langchain_callbacks = types.ModuleType("langchain_core.callbacks") + langchain_callbacks.CallbackManager = _CallbackManager + langchain_messages = types.ModuleType("langchain_core.messages") + langchain_messages.AIMessage = _AIMessage + langchain_messages.messages_to_dict = _messages_to_dict + + sys.modules.update( + { + "nemo_relay": relay, + "nemo_relay.integrations": integrations, + "nemo_relay.integrations.langchain": langchain_integration, + "nemo_relay.utils": relay_utils, + "langgraph": langgraph, + "langgraph.callbacks": langgraph_callbacks, + "langchain_core": langchain_core, + "langchain_core.callbacks": langchain_callbacks, + "langchain_core.messages": langchain_messages, + } + ) + return relay, guardrails, subscribers, scope + + +def _load_module(path: Path) -> types.ModuleType: + spec = importlib.util.spec_from_file_location("nemoclaw_observability_test", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _SensitiveOperationError(RuntimeError): + pass + + +_HOSTILE_TYPE_NAME_READS = [0] +_HOSTILE_EXCEPTION_DISPATCHES = [0] + + +class _HostileCaptureMeta(type): + def __getattribute__(cls, name: str) -> Any: + if name == "__name__": + _HOSTILE_TYPE_NAME_READS[0] += 1 + raise AttributeError("hostile type name is intentionally unavailable") + return super().__getattribute__(name) + + +class _HostileCaptureObject(metaclass=_HostileCaptureMeta): + def __repr__(self) -> str: + raise AssertionError("observability evaluated an opaque repr") + + def __str__(self) -> str: + raise AssertionError("observability evaluated an opaque string") + + +class _HostileIdentifier(str): + def __str__(self) -> str: + raise AssertionError("observability coerced a hostile identifier") + + +class _HostileDispatchError(_SensitiveOperationError): + @property + def __traceback__(self) -> Any: + _HOSTILE_EXCEPTION_DISPATCHES[0] += 1 + raise _SensitiveOperationError(f"hostile-traceback:{SECRET}") + + def with_traceback(self, _traceback: Any) -> Any: + _HOSTILE_EXCEPTION_DISPATCHES[0] += 1 + raise _SensitiveOperationError(f"hostile-restore:{SECRET}") + + +class _ToolCallRequest: + def __init__(self, tool_call: dict[str, Any]) -> None: + self.tool_call = tool_call + + def override(self, *, tool_call: dict[str, Any]) -> _ToolCallRequest: + return _ToolCallRequest(tool_call) + + +class _FailingToolCallRequest: + @property + def tool_call(self) -> Any: + raise RuntimeError("injected tool request-build failure") + + +def _preserved_exception(error: Exception, caught: Exception) -> dict[str, Any]: + return { + "same_instance": caught is error, + "type": type(caught).__name__, + "message": str(caught), + } + + +def _exercise_hostile_exception(module: types.ModuleType, middleware: Any) -> dict[str, Any]: + explicit_cause = ValueError(f"explicit-cause:{SECRET}") + hostile_error = _HostileDispatchError(f"hostile-original:{SECRET}") + hostile_error.__cause__ = explicit_cause + + def handler(_request: Any) -> Any: + raise hostile_error + + try: + middleware.wrap_model_call(object(), handler) + except Exception as caught: + return { + **_preserved_exception(hostile_error, caught), + "cause_preserved": BaseException.__getattribute__( + caught, "__cause__" + ) + is explicit_cause, + "subclass_dispatches": _HOSTILE_EXCEPTION_DISPATCHES[0], + } + raise AssertionError("hostile application exception did not escape") + + +def _exercise_middleware_errors(module: types.ModuleType) -> dict[str, Any]: + middleware = module.new_relay_middleware() + preserved: dict[str, Any] = {} + + sync_model_error = _SensitiveOperationError(f"sync-model:{SECRET}") + + def sync_model_handler(_request: Any) -> Any: + raise sync_model_error + + try: + middleware.wrap_model_call(object(), sync_model_handler) + except Exception as caught: + preserved["sync_model"] = _preserved_exception(sync_model_error, caught) + + sync_tool_error = _SensitiveOperationError(f"sync-tool:{SECRET}") + + def sync_tool_handler(_request: Any) -> Any: + raise sync_tool_error + + tool_request = _ToolCallRequest( + {"name": "execute", "args": {"command": SECRET}} + ) + try: + middleware.wrap_tool_call(tool_request, sync_tool_handler) + except Exception as caught: + preserved["sync_tool"] = _preserved_exception(sync_tool_error, caught) + + async def exercise_async() -> None: + async_model_error = _SensitiveOperationError(f"async-model:{SECRET}") + + async def async_model_handler(_request: Any) -> Any: + raise async_model_error + + try: + await middleware.awrap_model_call(object(), async_model_handler) + except Exception as caught: + preserved["async_model"] = _preserved_exception( + async_model_error, caught + ) + + async_tool_error = _SensitiveOperationError(f"async-tool:{SECRET}") + + async def async_tool_handler(_request: Any) -> Any: + raise async_tool_error + + try: + await middleware.awrap_tool_call(tool_request, async_tool_handler) + except Exception as caught: + preserved["async_tool"] = _preserved_exception(async_tool_error, caught) + + asyncio.run(exercise_async()) + + relay_errors_before_control_flow = len(_RELAY_OBSERVED_ERRORS) + keyboard_interrupt = KeyboardInterrupt("operator interrupt") + + def interrupted_model_handler(_request: Any) -> Any: + raise keyboard_interrupt + + try: + middleware.wrap_model_call(object(), interrupted_model_handler) + except KeyboardInterrupt as caught: + control_flow = { + "same_instance": caught is keyboard_interrupt, + "relay_observed": len(_RELAY_OBSERVED_ERRORS) + != relay_errors_before_control_flow, + } + else: + raise AssertionError("KeyboardInterrupt did not escape the observability boundary") + + return { + "preserved": preserved, + "hostile": _exercise_hostile_exception(module, middleware), + "control_flow": control_flow, + "relay_observed": list(_RELAY_OBSERVED_ERRORS), + "secret_present_in_relay_errors": SECRET + in json.dumps(_RELAY_OBSERVED_ERRORS, sort_keys=True), + } + + +def _exercise_relay_fail_open(module: types.ModuleType) -> dict[str, Any]: + middleware = module.new_relay_middleware() + cases: dict[str, Any] = {} + + def sync_model_case(mode: str) -> dict[str, Any]: + calls = 0 + expected = object() + + def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + return expected + + _RELAY_LLM_FAILURE_MODE[0] = mode + try: + result = middleware.wrap_model_call(object(), handler) + finally: + _RELAY_LLM_FAILURE_MODE[0] = None + return {"calls": calls, "same_result": result is expected} + + def sync_tool_case(mode: str) -> dict[str, Any]: + calls = 0 + expected = object() + request = _ToolCallRequest({"name": "execute", "args": {"mode": mode}}) + + def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + return expected + + _RELAY_TOOL_FAILURE_MODE[0] = mode + try: + result = middleware.wrap_tool_call(request, handler) + finally: + _RELAY_TOOL_FAILURE_MODE[0] = None + return {"calls": calls, "same_result": result is expected} + + cases["sync_model_before"] = sync_model_case("before") + cases["sync_model_after"] = sync_model_case("after") + cases["sync_tool_before"] = sync_tool_case("before") + cases["sync_tool_after"] = sync_tool_case("after") + + async def exercise_async() -> None: + async def model_case(mode: str) -> dict[str, Any]: + calls = 0 + expected = object() + + async def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + return expected + + _RELAY_LLM_FAILURE_MODE[0] = mode + try: + result = await middleware.awrap_model_call(object(), handler) + finally: + _RELAY_LLM_FAILURE_MODE[0] = None + return {"calls": calls, "same_result": result is expected} + + async def tool_case(mode: str) -> dict[str, Any]: + calls = 0 + expected = object() + request = _ToolCallRequest( + {"name": "execute", "args": {"mode": mode}} + ) + + async def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + return expected + + _RELAY_TOOL_FAILURE_MODE[0] = mode + try: + result = await middleware.awrap_tool_call(request, handler) + finally: + _RELAY_TOOL_FAILURE_MODE[0] = None + return {"calls": calls, "same_result": result is expected} + + cases["async_model_before"] = await model_case("before") + cases["async_model_after"] = await model_case("after") + cases["async_tool_before"] = await tool_case("before") + cases["async_tool_after"] = await tool_case("after") + + asyncio.run(exercise_async()) + + original_args = { + "huge_negative": -(10**1000), + "huge_positive": 10**1000, + "lone_surrogate": "before\ud800after", + } + original_result = { + "huge_result": 10**1000, + "lone_surrogate_result": "before\udfffafter", + } + value_calls = 0 + value_request = _ToolCallRequest({"name": "execute", "args": original_args}) + + def value_handler(request: _ToolCallRequest) -> Any: + nonlocal value_calls + value_calls += 1 + if request.tool_call["args"] is not original_args: + raise AssertionError("observability mutated application tool arguments") + return original_result + + value_result = middleware.wrap_tool_call(value_request, value_handler) + normalized = module._bounded_capture({**original_args, **original_result}) + _validate_relay_json(normalized) + + return { + "failure_cases": cases, + "unsafe_python_values": { + "calls": value_calls, + "same_result": value_result is original_result, + "normalized": normalized, + }, + } + + +def _new_contextual_error( + error_type: type[BaseException], label: str +) -> tuple[BaseException, BaseException, BaseException]: + cause = ValueError(f"{label}-cause") + context = LookupError(f"{label}-context") + error = error_type(f"{label}-application-error") + error.__cause__ = cause + error.__context__ = context + return error, cause, context + + +def _fallback_error_result( + *, + calls: int, + caught: BaseException, + expected: BaseException, + cause: BaseException, + context: BaseException, +) -> dict[str, Any]: + return { + "calls": calls, + "same_instance": caught is expected, + "cause_preserved": BaseException.__cause__.__get__(caught, BaseException) + is cause, + "context_preserved": BaseException.__context__.__get__( + caught, BaseException + ) + is context, + "type": type(caught).__name__, + } + + +def _exercise_fallback_exception_transparency( + module: types.ModuleType, +) -> dict[str, Any]: + middleware = module.new_relay_middleware() + results: dict[str, Any] = {} + + def sync_case( + name: str, + error_type: type[BaseException], + invoke: Any, + ) -> None: + calls = 0 + error, cause, context = _new_contextual_error(error_type, name) + + def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + raise error + + try: + invoke(handler) + except BaseException as caught: + results[name] = _fallback_error_result( + calls=calls, + caught=caught, + expected=error, + cause=cause, + context=context, + ) + else: + raise AssertionError(f"{name} application error did not escape") + + original_model_builder = module._bounded_model_call_request + + def failing_model_builder(_request: Any) -> Any: + raise RuntimeError("injected model request-build failure") + + module._bounded_model_call_request = failing_model_builder + try: + sync_case( + "sync_model_build", + RuntimeError, + lambda handler: middleware.wrap_model_call(object(), handler), + ) + finally: + module._bounded_model_call_request = original_model_builder + + _RELAY_LLM_FAILURE_MODE[0] = "before" + try: + sync_case( + "sync_model_relay", + KeyboardInterrupt, + lambda handler: middleware.wrap_model_call(object(), handler), + ) + finally: + _RELAY_LLM_FAILURE_MODE[0] = None + + sync_case( + "sync_tool_build", + SystemExit, + lambda handler: middleware.wrap_tool_call(_FailingToolCallRequest(), handler), + ) + + _RELAY_TOOL_FAILURE_MODE[0] = "before" + try: + sync_case( + "sync_tool_relay", + RuntimeError, + lambda handler: middleware.wrap_tool_call( + _ToolCallRequest({"name": "execute", "args": {}}), handler + ), + ) + finally: + _RELAY_TOOL_FAILURE_MODE[0] = None + + async def exercise_async() -> None: + async def async_case( + name: str, + error_type: type[BaseException], + invoke: Any, + ) -> None: + calls = 0 + error, cause, context = _new_contextual_error(error_type, name) + + async def handler(_request: Any) -> Any: + nonlocal calls + calls += 1 + raise error + + try: + await invoke(handler) + except BaseException as caught: + results[name] = _fallback_error_result( + calls=calls, + caught=caught, + expected=error, + cause=cause, + context=context, + ) + else: + raise AssertionError(f"{name} application error did not escape") + + module._bounded_model_call_request = failing_model_builder + try: + await async_case( + "async_model_build", + asyncio.CancelledError, + lambda handler: middleware.awrap_model_call(object(), handler), + ) + finally: + module._bounded_model_call_request = original_model_builder + + _RELAY_LLM_FAILURE_MODE[0] = "before" + try: + await async_case( + "async_model_relay", + RuntimeError, + lambda handler: middleware.awrap_model_call(object(), handler), + ) + finally: + _RELAY_LLM_FAILURE_MODE[0] = None + + await async_case( + "async_tool_build", + RuntimeError, + lambda handler: middleware.awrap_tool_call( + _FailingToolCallRequest(), handler + ), + ) + + _RELAY_TOOL_FAILURE_MODE[0] = "before" + try: + await async_case( + "async_tool_relay", + asyncio.CancelledError, + lambda handler: middleware.awrap_tool_call( + _ToolCallRequest({"name": "execute", "args": {}}), handler + ), + ) + finally: + _RELAY_TOOL_FAILURE_MODE[0] = None + + asyncio.run(exercise_async()) + return results + + +def _exercise_control_flow_suppression(module: types.ModuleType) -> dict[str, Any]: + results: dict[str, Any] = {} + for name, control_flow in ( + ("KeyboardInterrupt", KeyboardInterrupt("operator interrupt")), + ("SystemExit", SystemExit("process exit")), + ("CancelledError", asyncio.CancelledError("task cancellation")), + ): + boundary = module._RelayExceptionBoundary() + boundary.capture(RuntimeError("captured application error")) + try: + with boundary.suppress_relay_exception(): + raise control_flow + except BaseException as caught: + results[name] = caught is control_flow + return results + + +def _exercise_identifier_boundaries( + module: types.ModuleType, scope: _Scope +) -> dict[str, Any]: + controls = "\r\n\t\x00\u202e" + overlong = "x" * 200 + truncation_sentinel = "-MUST-NOT-REACH-RELAY" + middleware = module.new_relay_middleware() + + async def model_call(request: Any) -> Any: + return request + + asyncio.run( + middleware._llm_execute( + f"model{controls}{overlong}{truncation_sentinel}", + object(), + None, + None, + model_call, + ) + ) + + sync_tool_request = _ToolCallRequest( + { + "name": f"tool{controls}{overlong}{truncation_sentinel}", + "args": {}, + } + ) + middleware.wrap_tool_call(sync_tool_request, lambda _request: None) + + async_tool_request = _ToolCallRequest( + { + "name": f"async-tool{controls}{overlong}{truncation_sentinel}", + "args": {}, + } + ) + + async def async_tool_handler(_request: Any) -> None: + return None + + asyncio.run(middleware.awrap_tool_call(async_tool_request, async_tool_handler)) + + callback = module.new_metadata_only_callback_handler() + scope_record_offset = len(scope.records) + callback.on_chain_start( + None, + {}, + run_id="hostile-name-run", + name=f"graph{controls}{overlong}{truncation_sentinel}", + ) + callback.on_chain_end({}, run_id="hostile-name-run") + graph_records = scope.records[scope_record_offset:] + graph_name = next( + record["name"] + for record in graph_records + if record["operation"] == "push" + ) + + return { + "model": _RELAY_OBSERVED_MODEL_NAMES[-1], + "sync_tool": _RELAY_OBSERVED_TOOL_NAMES[-2], + "async_tool": _RELAY_OBSERVED_TOOL_NAMES[-1], + "graph": graph_name, + } + + +def _exercise_callback_manager_boundary(module: types.ModuleType) -> dict[str, Any]: + class _HostileCallback: + pass + + hostile = _HostileCallback() + manager = module.new_metadata_only_callback_manager() + managed_handler = manager.handlers[0] + manager.add_handler(hostile) + copied = manager.copy() + manager.set_handler(hostile) + manager.set_handlers([hostile]) + manager.remove_handler(managed_handler) + + hostile_manager = _CallbackManager( + handlers=[hostile], + inheritable_handlers=[hostile], + tags=["invocation-tag"], + inheritable_tags=["invocation-inheritable-tag"], + metadata={"invocation": "preserved"}, + inheritable_metadata={"inheritable": "preserved"}, + ) + merged = manager.merge(hostile_manager) + merged.add_handler(hostile) + + return { + "bound_handlers": len(manager.handlers), + "bound_metadata_only": manager.handlers == [managed_handler], + "copy_handlers": len(copied.handlers), + "copy_metadata_only": copied.handlers == [managed_handler], + "merged_handlers": len(merged.handlers), + "merged_metadata_only": merged.handlers == [managed_handler], + "merged_tags": merged.tags, + "merged_inheritable_tags": merged.inheritable_tags, + "merged_metadata": merged.metadata, + "merged_inheritable_metadata": merged.inheritable_metadata, + } + + +def _privacy_scenario(path: Path) -> dict[str, Any]: + ambient_otel = { + "OTEL_EXPORTER_OTLP_ENDPOINT": f"https://attacker.invalid/{SECRET}", + "OTEL_EXPORTER_OTLP_HEADERS": f"authorization={SECRET}", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": ( + f"https://traces.attacker.invalid/{SECRET}" + ), + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": f"x-api-key={SECRET}", + "OTEL_RESOURCE_ATTRIBUTES": f"service.name={SECRET}", + "OTEL_EXPORTER_OTLP_CERTIFICATE": f"/nonexistent/{SECRET}/ca.pem", + } + os.environ.update(ambient_otel) + _, guardrails, subscribers, scope = _install_stubs() + module = _load_module(path) + + exact_opt_in = { + value: module.observability_requested({"NEMOCLAW_OBSERVABILITY": value}) + for value in ("1", "true", "TRUE", " 1", "0") + } + os.environ["NEMOCLAW_OBSERVABILITY"] = "1" + initialized = module.initialize_observability() + initialized_again = module.initialize_observability() + ambient_environment_restored = all( + os.environ.get(name) == value for name, value in ambient_otel.items() + ) + subscriber = _OpenInferenceSubscriber.instances[0] + + request_guardrail = guardrails.registered["llm_request"]["callback"] + response_guardrail = guardrails.registered["llm_response"]["callback"] + tool_request_guardrail = guardrails.registered["tool_request"]["callback"] + tool_response_guardrail = guardrails.registered["tool_response"]["callback"] + + request = request_guardrail( + _LLMRequest( + {"authorization": SECRET}, + { + "model": "managed-model", + "messages": [{"content": SECRET}], + "tools": [{"description": DROPPED_TOOL_SCHEMA}], + "model_settings": {"api_key": DROPPED_MODEL_SETTINGS}, + "response_format": {"schema": DROPPED_RESPONSE_FORMAT}, + }, + ) + ) + response = response_guardrail({"content": SECRET, "error": SECRET}) + tool_request = tool_request_guardrail("execute", {"command": SECRET}) + tool_response = tool_response_guardrail("execute", {"stdout": SECRET}) + bounded_redaction = tool_request_guardrail( + "execute", + { + "APIKey": SECRET, + "APIToken": SECRET, + "AWS_SECRET_ACCESS_KEY": SECRET, + "AWSSecretAccessKey": SECRET, + "accessToken": SECRET, + "api_key": SECRET, + "apiKey": SECRET, + "auth": SECRET, + "authentication": SECRET, + "bearer": SECRET, + "clientSecret": SECRET, + "credential": SECRET, + "header": SECRET, + "nested": {"checkpoint_id": SECRET, "command": "allowed"}, + "opaque": _HostileCaptureObject(), + "oversized": "x" * 9000, + "passwd": SECRET, + "privateKey": SECRET, + "token": SECRET, + }, + ) + if _HOSTILE_TYPE_NAME_READS[0] != 0: + raise AssertionError("observability evaluated a hostile type name") + oversized_capture = tool_request_guardrail( + "execute", {f"item_{index}": "y" * 8000 for index in range(10)} + ) + unsafe_relay_serialization = { + "pickle": tool_request_guardrail( + "execute", + {"__nv_pickle__": "opaque.Artifact", "data": UNSAFE_RELAY_FALLBACK}, + ), + "fallback_string": tool_request_guardrail( + "execute", + { + "__nv_fallback_str__": "opaque.Artifact", + "data": UNSAFE_RELAY_FALLBACK, + }, + ), + } + cyclic_capture_input: list[Any] = [] + cyclic_capture_input.append(cyclic_capture_input) + cyclic_capture = tool_request_guardrail("execute", cyclic_capture_input) + shared_capture_input: list[Any] = ["leaf"] + for _ in range(module._MAX_CAPTURE_DEPTH + 1): + shared_capture_input = [shared_capture_input] * module._MAX_CAPTURE_ITEMS + shared_capture = tool_request_guardrail("execute", shared_capture_input) + hostile_identifier = module._safe_identifier(_HostileIdentifier(SECRET), "fallback") + + callback = module.new_metadata_only_callback_handler() + callback.on_chain_start( + {"serialized": SECRET}, + {"messages": [SECRET]}, + run_id="run-1", + name="model", + metadata={"arbitrary": SECRET}, + tags=[SECRET], + ) + callback.on_chain_error( + RuntimeError(SECRET), + run_id="run-1", + metadata={"arbitrary": SECRET}, + ) + callback.on_interrupt( + SimpleNamespace( + status=SECRET, + checkpoint_id=SECRET, + interrupts=[{"value": SECRET}], + ) + ) + callback.on_resume(SimpleNamespace(status=SECRET, checkpoint_id=SECRET)) + + first_middleware = module.new_relay_middleware() + second_middleware = module.new_relay_middleware() + callback_manager_boundary = _exercise_callback_manager_boundary(module) + error_boundary = _exercise_middleware_errors(module) + relay_fail_open = _exercise_relay_fail_open(module) + fallback_exception_transparency = _exercise_fallback_exception_transparency(module) + control_flow_suppression = _exercise_control_flow_suppression(module) + emitted = { + "request": {"headers": request.headers, "content": request.content}, + "response": response, + "tool_request": tool_request, + "tool_response": tool_response, + "bounded_redaction": bounded_redaction, + "oversized_capture": oversized_capture, + "unsafe_relay_serialization": unsafe_relay_serialization, + "cyclic_capture": cyclic_capture, + "shared_capture": shared_capture, + "hostile_identifier": hostile_identifier, + "callback_records": list(scope.records), + } + identifier_boundaries = _exercise_identifier_boundaries(module, scope) + module.shutdown_observability() + module.shutdown_observability() + + return { + "exact_opt_in": exact_opt_in, + "initialized": initialized, + "initialized_again": initialized_again, + "ambient_environment_restored": ambient_environment_restored, + "subscriber_count": len(_OpenInferenceSubscriber.instances), + "config": { + "transport": subscriber.config.transport, + "endpoint": subscriber.config.endpoint, + "headers": subscriber.config.headers, + "service_name": subscriber.config.service_name, + "timeout_millis": subscriber.config.timeout_millis, + }, + "guardrail_priorities": { + name: registration["priority"] + for name, registration in guardrails.registered.items() + }, + "emitted": emitted, + "secret_present": SECRET in json.dumps(emitted, sort_keys=True), + "middleware_distinct": first_middleware is not second_middleware, + "middleware_name": first_middleware.name, + "callback_manager_boundary": callback_manager_boundary, + "error_boundary": error_boundary, + "relay_fail_open": relay_fail_open, + "fallback_exception_transparency": fallback_exception_transparency, + "control_flow_suppression": control_flow_suppression, + "identifier_boundaries": identifier_boundaries, + "flush_calls": subscribers.flush_calls, + "force_flush_calls": subscriber.force_flush_calls, + "deregistered": subscriber.deregistered, + "shutdown_calls": subscriber.shutdown_calls, + "guardrails_deregistered": len(guardrails.deregistered), + } + + +def _outage_scenario(path: Path, *, fail_construct: bool = False) -> dict[str, Any]: + _, guardrails, subscribers, _ = _install_stubs( + fail_flush=True, + fail_force_flush=True, + fail_construct=fail_construct, + ) + module = _load_module(path) + os.environ["NEMOCLAW_OBSERVABILITY"] = "1" + initialized = module.initialize_observability() + module.shutdown_observability() + + subscriber = ( + _OpenInferenceSubscriber.instances[0] + if _OpenInferenceSubscriber.instances + else None + ) + return { + "initialized": initialized, + "flush_calls": subscribers.flush_calls, + "force_flush_calls": subscriber.force_flush_calls if subscriber else 0, + "deregistered": subscriber.deregistered if subscriber else [], + "shutdown_calls": subscriber.shutdown_calls if subscriber else 0, + "guardrails_deregistered": len(guardrails.deregistered), + } + + +def _logging_failure_scenario(path: Path) -> dict[str, Any]: + ambient_otel = { + "OTEL_EXPORTER_OTLP_HEADERS": f"authorization={LOG_HEADER_SECRET}", + "OTEL_EXPORTER_OTLP_CERTIFICATE": f"/nonexistent/{LOG_CERTIFICATE_SECRET}/ca.pem", + "OTEL_EXPORTER_OTLP_CLIENT_KEY": f"/nonexistent/{LOG_CLIENT_KEY_SECRET}/client.key", + } + os.environ.update(ambient_otel) + _, guardrails, _, _ = _install_stubs(fail_register=True) + module = _load_module(path) + os.environ["NEMOCLAW_OBSERVABILITY"] = "1" + log_output = io.StringIO() + handler = logging.StreamHandler(log_output) + handler.setFormatter(logging.Formatter("%(levelname)s:%(message)s")) + module.logger.setLevel(logging.DEBUG) + module.logger.addHandler(handler) + module.logger.propagate = False + try: + initialized = module.initialize_observability() + finally: + module.logger.removeHandler(handler) + handler.close() + + subscriber = _OpenInferenceSubscriber.instances[0] + return { + "initialized": initialized, + "logs": log_output.getvalue(), + "ambient_environment_restored": all( + os.environ.get(name) == value for name, value in ambient_otel.items() + ), + "shutdown_calls": subscriber.shutdown_calls, + "guardrails_deregistered": len(guardrails.deregistered), + } + + +def main() -> None: + if len(sys.argv) != 3: + raise SystemExit( + "usage: harness.py " + ) + scenario, raw_path = sys.argv[1:] + path = Path(raw_path) + if scenario == "privacy": + result = _privacy_scenario(path) + elif scenario == "outage": + result = _outage_scenario(path) + elif scenario == "construction": + result = _outage_scenario(path, fail_construct=True) + elif scenario == "logging": + result = _logging_failure_scenario(path) + else: + raise SystemExit(f"unknown scenario: {scenario}") + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 4c89e1dbe0..96e5911518 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -94,6 +94,8 @@ export type RebuildFlowOverrides = { sessionSandboxName?: string; sandboxListOutput?: string; backupPolicyPresets?: string[]; + gatewayPresets?: string[]; + verificationUnavailableAfterPresetRemoval?: boolean; preDeleteSandboxEntry?: Record; preDeleteDefaultSandbox?: string | null; preDeleteLatestManifest?: Record | null; @@ -139,6 +141,7 @@ export type RebuildFlowHarness = { preflightMessagingConflictsSpy: MockInstance; preflightDcodeRouteSpy: MockInstance; prepareManagedDcodeRebuildImageSpy: MockInstance; + removePresetSpy: MockInstance; removeSandboxRegistryEntrySpy: MockInstance; registryUpdateSpy: MockInstance; releaseOnboardLockSpy: MockInstance; @@ -496,15 +499,77 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): (options ?? {}) as Record, ); }); + const livePolicyPresets = new Set(overrides.gatewayPresets ?? []); + const managedObservabilityPreset = "observability-otlp-local"; + const managedObservabilityContent = + "network_policies:\n observability-otlp-local:\n name: observability-otlp-local\n"; + let liveManagedObservabilityContent = livePolicyPresets.has(managedObservabilityPreset) + ? managedObservabilityContent + : null; + let policyRemovalObserved = false; const applyPresetSpy = vi .spyOn(policies, "applyPreset") .mockImplementation((_sandboxName: unknown, presetName: unknown) => { const normalizedPresetName = String(presetName); - if (overrides.applyPreset) return overrides.applyPreset(normalizedPresetName); - if (normalizedPresetName === "throw") throw new Error("preset boom"); - return normalizedPresetName === "npm"; + let applied: boolean; + if (overrides.applyPreset) { + applied = overrides.applyPreset(normalizedPresetName); + } else if (normalizedPresetName === "throw") { + throw new Error("preset boom"); + } else { + applied = normalizedPresetName === "npm"; + } + if (applied) { + livePolicyPresets.add(normalizedPresetName); + if (normalizedPresetName === managedObservabilityPreset) { + liveManagedObservabilityContent = managedObservabilityContent; + } + } + return applied; + }); + const applyPresetContentSpy = vi + .spyOn(policies, "applyPresetContent") + .mockImplementation((_sandboxName: unknown, presetName: unknown, presetContent: unknown) => { + livePolicyPresets.add(String(presetName)); + const content = String(presetContent); + if (policies.parsePresetPolicyKeys(content).includes(managedObservabilityPreset)) { + liveManagedObservabilityContent = content; + } + return true; + }); + vi.spyOn(policies, "loadPresetForSandbox").mockImplementation( + (_sandboxName: unknown, presetName: unknown) => + String(presetName) === managedObservabilityPreset ? managedObservabilityContent : null, + ); + vi.spyOn(policies, "getPresetContentGatewayState").mockImplementation( + (_sandboxName: unknown, presetContent: unknown) => { + if (overrides.verificationUnavailableAfterPresetRemoval && policyRemovalObserved) return null; + const content = String(presetContent); + if (!policies.parsePresetPolicyKeys(content).includes(managedObservabilityPreset)) { + return "absent"; + } + if (liveManagedObservabilityContent === null) return "absent"; + return liveManagedObservabilityContent === content ? "match" : "drift"; + }, + ); + vi.spyOn(policies, "getGatewayPresets").mockImplementation(() => + overrides.verificationUnavailableAfterPresetRemoval && policyRemovalObserved + ? null + : [...livePolicyPresets], + ); + const removePresetSpy = vi + .spyOn(policies, "removePreset") + .mockImplementation((_sandboxName: unknown, presetName: unknown) => { + const removed = livePolicyPresets.delete(String(presetName)); + if ( + String(presetName) === managedObservabilityPreset && + liveManagedObservabilityContent === managedObservabilityContent + ) { + liveManagedObservabilityContent = null; + } + if (removed) policyRemovalObserved = true; + return removed; }); - const applyPresetContentSpy = vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); const executeSandboxCommandSpy = vi .spyOn(processRecovery, "executeSandboxCommand") .mockImplementation( @@ -569,6 +634,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): preflightMessagingConflictsSpy, preflightDcodeRouteSpy, prepareManagedDcodeRebuildImageSpy, + removePresetSpy, removeSandboxRegistryEntrySpy, registryUpdateSpy, releaseOnboardLockSpy, diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index ac236685ad..9c788dfcf1 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -388,14 +388,26 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): overrides.ensureValidatedBraveSearchCredential ?? (async () => "web-search-key"), ); + const livePolicyPresets = new Set(); const applyPresetSpy = vi .spyOn(policies, "applyPreset") .mockImplementation((_sandboxName: unknown, presetName: unknown) => { const normalizedPresetName = String(presetName); - if (overrides.applyPreset) return overrides.applyPreset(normalizedPresetName); - if (normalizedPresetName === "throw") throw new Error("preset boom"); - return normalizedPresetName === "npm"; + let applied: boolean; + if (overrides.applyPreset) { + applied = overrides.applyPreset(normalizedPresetName); + } else if (normalizedPresetName === "throw") { + throw new Error("preset boom"); + } else { + applied = normalizedPresetName === "npm"; + } + if (applied) livePolicyPresets.add(normalizedPresetName); + return applied; }); + vi.spyOn(policies, "getGatewayPresets").mockImplementation(() => [...livePolicyPresets]); + vi.spyOn(policies, "removePreset").mockImplementation( + (_sandboxName: unknown, presetName: unknown) => livePolicyPresets.delete(String(presetName)), + ); const executeSandboxCommandSpy = vi .spyOn(processRecovery, "executeSandboxCommand") .mockImplementation( diff --git a/test/langchain-deepagents-code-direct-module-patch.test.ts b/test/langchain-deepagents-code-direct-module-patch.test.ts index d6e023f10c..eba3ee181b 100644 --- a/test/langchain-deepagents-code-direct-module-patch.test.ts +++ b/test/langchain-deepagents-code-direct-module-patch.test.ts @@ -651,7 +651,6 @@ describe("LangChain Deep Agents Code managed package patch", () => { const source = fs.readFileSync(path.join(packageDir, relativePath), "utf8"); expect(source.match(/NemoClaw-managed Deep Agents Code hardening v2\./g)).toHaveLength(1); } - const main = fs.readFileSync(path.join(packageDir, "main.py"), "utf8"); for (const expected of [ 'args.sandbox = "none"', @@ -1154,7 +1153,7 @@ spec.loader.exec_module(progressive_disclosure_harness) progressive_disclosure_harness._install_stubs() from deepagents_code import agent, app, auth_store, config, hooks, main as dcode_main, model_config, non_interactive, server, subagents, update_check -from deepagents_code import _nemoclaw_managed +from deepagents_code import _nemoclaw_managed, nemoclaw_observability from deepagents_code import config_manifest from deepagents_code.integrations import openai_codex from deepagents_code.widgets.auth import AuthManagerScreen, AuthPromptScreen, AuthResult @@ -1165,6 +1164,7 @@ from types import SimpleNamespace async def validate(): + assert nemoclaw_observability.initialize_observability() is False instance = app.DeepAgentsApp() for command in ( "/update", diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index 290565d50c..c940721f42 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -329,6 +329,7 @@ describe("LangChain Deep Agents Code image contracts", () => { } for (const s of [ "managed-dcode-runtime.py", + "nemoclaw_observability.py", "patch-managed-deepagents-code.py", "DEEPAGENTS_CODE_LANGSMITH_TRACING=false", "LANGSMITH_TRACING=false", @@ -351,6 +352,11 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(dockerfile).toContain( "rm -f /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py", ); + expect(dockerfile).toContain("COPY agents/langchain-deepagents-code/validate-observability.py"); + expect(dockerfile).toContain( + "/opt/venv/bin/python3 -I /opt/nemoclaw-deepagents-code/validate-observability.py", + ); + expect(dockerfile).toContain("rm -f /opt/nemoclaw-deepagents-code/validate-observability.py"); expect(dockerfile).toContain("ARG NEMOCLAW_TOOL_DISCLOSURE=progressive"); expect(dockerfile).toContain("NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE}"); expect(dockerfile).toContain("progressive|direct)"); @@ -656,6 +662,7 @@ describe("LangChain Deep Agents Code image contracts", () => { "test/e2e/e2e-cloud-experimental/checks/08-deepagents-code-secret-boundary.sh", "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", "test/e2e/e2e-cloud-experimental/checks/10-deepagents-code-tui-startup.sh", + "test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh", ]); }); @@ -785,6 +792,7 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(baseDockerfile).not.toContain("deepagents-code[nvidia]==${DEEPAGENTS_CODE_VERSION}"); expect(requirementsLock).toContain("uv==0.11.15 \\"); expect(requirementsLock).toContain("deepagents-code==0.1.30 \\"); + expect(requirementsLock).toContain("nemo-relay==0.4.0 \\"); expect(requirementsLock).toContain("langchain-nvidia-ai-endpoints==1.4.3 \\"); expect(requirementsLock).toContain("aiohttp==3.14.1 \\"); expect(requirementsLock).toContain("langchain-nvidia-ai-endpoints=="); @@ -795,10 +803,10 @@ describe("LangChain Deep Agents Code image contracts", () => { const review = readAgentFile("dependency-review.md"); expect(review).toContain("requirements.lock"); - expect(review).toContain("229efec862ec10e6b128525e95c8fb8b44cdef8285a6cee78e3a7c73af780a9b"); - expect(review).toContain("Audit date: 2026-07-03"); + expect(review).toContain("6fde7b3188137ab5669898a552d5b12c7def2560cb4c861e8ed3563d35a5bcb9"); + expect(review).toContain("Audit date: 2026-07-06"); expect(review).toContain( - "uvx --python 3.13 pip-audit -r agents/langchain-deepagents-code/requirements.lock --progress-spinner off", + "uv tool run --python 3.13 pip-audit -r agents/langchain-deepagents-code/requirements.lock --progress-spinner off", ); expect(review).toContain("No known vulnerabilities found"); }); diff --git a/test/langchain-deepagents-code-observability.test.ts b/test/langchain-deepagents-code-observability.test.ts new file mode 100644 index 0000000000..9bc1ec808e --- /dev/null +++ b/test/langchain-deepagents-code-observability.test.ts @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const repoRoot = path.resolve(import.meta.dirname, ".."); +const modulePath = path.join( + repoRoot, + "agents", + "langchain-deepagents-code", + "nemoclaw_observability.py", +); +const harnessPath = path.join(repoRoot, "test", "fixtures", "deepagents-observability-harness.py"); + +function runScenario(scenario: "privacy" | "outage" | "construction" | "logging") { + const result = spawnSync("python3", [harnessPath, scenario, modulePath], { + encoding: "utf8", + env: { PATH: process.env.PATH }, + }); + expect(result.status, result.stderr).toBe(0); + return JSON.parse(result.stdout) as Record; +} + +describe("managed Deep Agents Code observability", () => { + it("exports bounded content through the fixed credential-free local boundary", () => { + const result = runScenario("privacy"); + + expect(result.exact_opt_in).toEqual({ + "1": true, + true: false, + TRUE: false, + " 1": false, + "0": false, + }); + expect(result.initialized).toBe(true); + expect(result.initialized_again).toBe(true); + expect(result.ambient_environment_restored).toBe(true); + expect(result.subscriber_count).toBe(1); + expect(result.config).toEqual({ + transport: "http_binary", + endpoint: "http://host.openshell.internal:4318/v1/traces", + headers: {}, + service_name: "nemoclaw-langchain-deepagents-code", + timeout_millis: 1000, + }); + expect(result.guardrail_priorities).toEqual({ + llm_request: 0, + llm_response: 0, + tool_request: 0, + tool_response: 0, + }); + expect(result.secret_present).toBe(true); + expect(result.emitted).toMatchObject({ + request: { + headers: {}, + content: { + messages: [{ content: "NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL" }], + model: "managed-model", + }, + }, + response: { + content: "NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + error: "NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + }, + tool_request: { command: "NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL" }, + tool_response: { stdout: "NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL" }, + bounded_redaction: { + APIKey: "", + APIToken: "", + AWS_SECRET_ACCESS_KEY: "", + AWSSecretAccessKey: "", + accessToken: "", + api_key: "", + apiKey: "", + auth: "", + authentication: "", + bearer: "", + clientSecret: "", + credential: "", + header: "", + nested: { checkpoint_id: "", command: "allowed" }, + opaque: { _omitted_type: "opaque" }, + passwd: "", + privateKey: "", + token: "", + }, + oversized_capture: { + _truncated: true, + _omitted_type: "opaque", + }, + unsafe_relay_serialization: { + fallback_string: { _omitted_type: "opaque" }, + pickle: { _omitted_type: "opaque" }, + }, + cyclic_capture: [{ _omitted_reference: "shared_or_cycle" }], + hostile_identifier: "fallback", + callback_records: [ + { + operation: "push", + name: "model", + category: "agent", + }, + { + operation: "pop", + metadata: { integration: "langgraph", "otel.status_code": "ERROR" }, + }, + { + operation: "event", + name: "Graph Interrupt", + metadata: { integration: "langgraph" }, + }, + { + operation: "event", + name: "Graph Resume", + metadata: { integration: "langgraph" }, + }, + ], + }); + const resultWithBounds = result.emitted as { + bounded_redaction: { + oversized: string; + }; + oversized_capture: { preview: string }; + }; + const boundedRedaction = resultWithBounds.bounded_redaction; + expect(boundedRedaction.oversized).toMatch(/^x{8000}\.\.\.\[truncated 1000 chars\]$/); + expect(resultWithBounds.oversized_capture.preview).toHaveLength(16_000); + expect( + JSON.stringify((result.emitted as { shared_capture: unknown }).shared_capture), + ).toContain("shared_or_cycle"); + expect( + JSON.stringify( + (result.emitted as { unsafe_relay_serialization: unknown }).unsafe_relay_serialization, + ), + ).not.toContain("NEMOCLAW-UNSAFE-RELAY-FALLBACK"); + expect(JSON.stringify((result.emitted as { request: unknown }).request)).not.toMatch( + /NEMOCLAW-DROPPED-(MODEL-SETTINGS|RESPONSE-FORMAT|TOOL-SCHEMA)/, + ); + expect( + JSON.stringify((result.emitted as { callback_records: unknown[] }).callback_records), + ).not.toContain("NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL"); + expect(result.middleware_distinct).toBe(true); + expect(result.middleware_name).toBe("NemoClawObservabilityMiddleware"); + expect(result.callback_manager_boundary).toEqual({ + bound_handlers: 1, + bound_metadata_only: true, + copy_handlers: 1, + copy_metadata_only: true, + merged_handlers: 1, + merged_metadata_only: true, + merged_tags: ["invocation-tag"], + merged_inheritable_tags: ["invocation-inheritable-tag"], + merged_metadata: { invocation: "preserved" }, + merged_inheritable_metadata: { inheritable: "preserved" }, + }); + expect(result.identifier_boundaries).toEqual({ + model: `model_${"x".repeat(118)}`, + sync_tool: `tool_${"x".repeat(119)}`, + async_tool: `async-tool_${"x".repeat(113)}`, + graph: `graph_${"x".repeat(118)}`, + }); + expect(result.error_boundary).toEqual({ + control_flow: { + same_instance: true, + relay_observed: true, + }, + hostile: { + same_instance: true, + type: "_HostileDispatchError", + message: "hostile-original:NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + cause_preserved: true, + subclass_dispatches: 0, + }, + preserved: { + sync_model: { + same_instance: true, + type: "_SensitiveOperationError", + message: "sync-model:NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + }, + sync_tool: { + same_instance: true, + type: "_SensitiveOperationError", + message: "sync-tool:NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + }, + async_model: { + same_instance: true, + type: "_SensitiveOperationError", + message: "async-model:NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + }, + async_tool: { + same_instance: true, + type: "_SensitiveOperationError", + message: "async-tool:NEMOCLAW-OBSERVABILITY-SECRET-SENTINEL", + }, + }, + relay_observed: Array.from({ length: 6 }, () => ({ + type: "RuntimeError", + message: "NEMOCLAW_DCODE_OPERATION_FAILED: managed operation failed (details redacted)", + context_is_none: true, + cause_is_none: true, + })), + secret_present_in_relay_errors: false, + }); + expect(result.relay_fail_open).toEqual({ + failure_cases: { + sync_model_before: { calls: 1, same_result: true }, + sync_model_after: { calls: 1, same_result: true }, + sync_tool_before: { calls: 1, same_result: true }, + sync_tool_after: { calls: 1, same_result: true }, + async_model_before: { calls: 1, same_result: true }, + async_model_after: { calls: 1, same_result: true }, + async_tool_before: { calls: 1, same_result: true }, + async_tool_after: { calls: 1, same_result: true }, + }, + unsafe_python_values: { + calls: 1, + same_result: true, + normalized: { + huge_negative: "", + huge_positive: "", + huge_result: "", + lone_surrogate: "before\ufffdafter", + lone_surrogate_result: "before\ufffdafter", + }, + }, + }); + expect(result.control_flow_suppression).toEqual({ + KeyboardInterrupt: true, + SystemExit: true, + CancelledError: true, + }); + const transparentFallback = (type: string) => ({ + calls: 1, + same_instance: true, + cause_preserved: true, + context_preserved: true, + type, + }); + expect(result.fallback_exception_transparency).toEqual({ + sync_model_build: transparentFallback("RuntimeError"), + sync_model_relay: transparentFallback("KeyboardInterrupt"), + sync_tool_build: transparentFallback("SystemExit"), + sync_tool_relay: transparentFallback("RuntimeError"), + async_model_build: transparentFallback("CancelledError"), + async_model_relay: transparentFallback("RuntimeError"), + async_tool_build: transparentFallback("RuntimeError"), + async_tool_relay: transparentFallback("CancelledError"), + }); + expect(result.flush_calls).toBe(1); + expect(result.force_flush_calls).toBe(1); + expect(result.shutdown_calls).toBe(1); + expect(result.guardrails_deregistered).toBe(4); + }); + + it("keeps agent shutdown fail-open when the collector cannot flush", () => { + expect(runScenario("outage")).toEqual({ + initialized: true, + flush_calls: 1, + force_flush_calls: 1, + deregistered: ["nemoclaw-dcode-openinference"], + shutdown_calls: 1, + guardrails_deregistered: 4, + }); + }); + + it("rolls back source sanitizers when exporter construction fails", () => { + expect(runScenario("construction")).toEqual({ + initialized: false, + flush_calls: 0, + force_flush_calls: 0, + deregistered: [], + shutdown_calls: 0, + guardrails_deregistered: 4, + }); + }); + + it("does not log ambient OTEL values when observability initialization fails", () => { + const result = runScenario("logging"); + const logs = String(result.logs); + + expect(result).toMatchObject({ + initialized: false, + ambient_environment_restored: true, + shutdown_calls: 1, + guardrails_deregistered: 4, + }); + expect(logs).toContain( + "WARNING:Managed observability could not be initialized; continuing without tracing", + ); + expect(logs).not.toMatch( + /NEMOCLAW-OTEL-(HEADER|CERTIFICATE|CLIENT-KEY)-CANARY|RuntimeError|Traceback|registration failed/, + ); + }); +}); diff --git a/test/langchain-deepagents-code-progressive-tool-disclosure.test.ts b/test/langchain-deepagents-code-progressive-tool-disclosure.test.ts index ce1d5b1327..11f268360c 100644 --- a/test/langchain-deepagents-code-progressive-tool-disclosure.test.ts +++ b/test/langchain-deepagents-code-progressive-tool-disclosure.test.ts @@ -10,6 +10,7 @@ import { describe, expect, it } from "vitest"; const repoRoot = path.resolve(import.meta.dirname, ".."); const agentDir = path.join(repoRoot, "agents", "langchain-deepagents-code"); const middlewarePath = path.join(agentDir, "progressive_tool_disclosure.py"); +const observabilityPath = path.join(agentDir, "nemoclaw_observability.py"); const patcherPath = path.join(agentDir, "patch-managed-deepagents-code.py"); const harnessPath = path.join( repoRoot, @@ -22,6 +23,7 @@ const MAIN_ANCHOR = " args = parser.parse_args()\n"; const ENTRYPOINT_ANCHOR = "from deepagents_code.main import cli_main\n"; const HARDENING_MARKER = "NemoClaw-managed Deep Agents Code hardening v2."; const DISCLOSURE_MARKER = "NemoClaw-managed progressive tool disclosure."; +const OBSERVABILITY_MARKER = "NemoClaw-managed backend-neutral observability."; const PACKAGE_SOURCES: Record = { "__init__.py": `"""Deep Agents Code 0.1.30 test package."""`, @@ -88,6 +90,38 @@ class ModelConfig: `, "agent.py": `from __future__ import annotations +class FakeGraph: + def __init__(self, main, subagents): + self.main = main + self.subagents = subagents + self.config = { + "tags": ["managed-tag"], + "metadata": {"managed": "preserved"}, + } + + def with_config(self, config): + merged = {**self.config, **config} + existing_callbacks = self.config.get("callbacks") + incoming_callbacks = config.get("callbacks") + if existing_callbacks is not None and incoming_callbacks is not None: + if isinstance(incoming_callbacks, list): + if isinstance(existing_callbacks, list): + merged["callbacks"] = existing_callbacks + incoming_callbacks + else: + manager = existing_callbacks.copy() + for callback in incoming_callbacks: + manager.add_handler(callback) + merged["callbacks"] = manager + elif isinstance(existing_callbacks, list): + manager = incoming_callbacks.copy() + for callback in existing_callbacks: + manager.add_handler(callback) + merged["callbacks"] = manager + else: + merged["callbacks"] = existing_callbacks.merge(incoming_callbacks) + self.config = merged + return self + def create_deep_agent(*args, **kwargs): del args main = list(kwargs.get("middleware") or ()) @@ -95,7 +129,7 @@ def create_deep_agent(*args, **kwargs): list(subagent.get("middleware") or ()) for subagent in kwargs.get("subagents") or () ] - return main, subagents + return FakeGraph(main, subagents) def _resolve_ptc_option(*args, **kwargs): return None def load_async_subagents(config_path=None): return [] @@ -105,11 +139,15 @@ def create_cli_agent(model, assistant_id, *args, **kwargs): kwargs.pop("mcp_server_info", None) kwargs.pop("rubric_model", None) kwargs.pop("async_subagents", None) - return create_deep_agent( + graph_config = kwargs.pop("graph_config", None) + graph = create_deep_agent( middleware=[], subagents=[{"name": "first", "middleware": []}, {"name": "second", "middleware": []}], **kwargs, ) + if graph_config is not None: + graph.config = {**graph.config, **graph_config} + return graph, "fixture-backend" `, "update_check.py": `from __future__ import annotations @@ -222,6 +260,7 @@ interface PatchFixture { mainPath: string; agentPath: string; modulePath: string; + observabilityModulePath: string; helperPath: string; sourcePaths: string[]; } @@ -248,6 +287,7 @@ function makePatchFixture(version = "0.1.30"): PatchFixture { const mainPath = path.join(packageDir, "main.py"); const agentPath = path.join(packageDir, "agent.py"); const modulePath = path.join(packageDir, "progressive_tool_disclosure.py"); + const observabilityModulePath = path.join(packageDir, "nemoclaw_observability.py"); const helperPath = path.join(packageDir, "_nemoclaw_managed.py"); return { root, @@ -256,6 +296,7 @@ function makePatchFixture(version = "0.1.30"): PatchFixture { mainPath, agentPath, modulePath, + observabilityModulePath, helperPath, sourcePaths, }; @@ -278,12 +319,57 @@ import importlib.util import json import os import sys +import types spec = importlib.util.spec_from_file_location("disclosure_harness", ${JSON.stringify(harnessPath)}) harness = importlib.util.module_from_spec(spec) spec.loader.exec_module(harness) harness._install_stubs() sys.path.insert(0, ${JSON.stringify(fixture.root)}) + +observability = types.ModuleType("deepagents_code.nemoclaw_observability") + +class RelayMiddleware: + pass + +class MetadataOnlyCallback: + pass + +class MetadataOnlyCallbackManager: + def __init__(self): + self.handlers = [MetadataOnlyCallback()] + + def copy(self): + return self + + def add_handler(self, handler): + del handler + + def merge(self, other): + del other + return self + +class HostileCallback: + pass + +class NormalCallbackManager: + def __init__(self, handlers): + self.handlers = handlers + + def copy(self): + return NormalCallbackManager(list(self.handlers)) + + def add_handler(self, handler): + self.handlers.append(handler) + + def merge(self, other): + return NormalCallbackManager([*self.handlers, *other.handlers]) + +observability.initialize_observability = lambda: os.environ.get("NEMOCLAW_OBSERVABILITY") == "1" +observability.new_relay_middleware = RelayMiddleware +observability.new_metadata_only_callback_manager = MetadataOnlyCallbackManager +sys.modules["deepagents_code.nemoclaw_observability"] = observability + agent = importlib.import_module("deepagents_code.agent") middleware = importlib.import_module("deepagents_code.progressive_tool_disclosure") @@ -297,7 +383,9 @@ class NamedTool: self.name = name def counts(result): - main, subagents = result + graph, backend = result + assert backend == "fixture-backend" + main, subagents = graph.main, graph.subagents middleware_type = middleware.ProgressiveToolDisclosureMiddleware instances = [item for item in main if isinstance(item, middleware_type)] instances.extend( @@ -305,6 +393,32 @@ def counts(result): ) return len(instances), len({id(item) for item in instances}) +def observability_counts(result): + graph, backend = result + assert backend == "fixture-backend" + instances = [item for item in graph.main if isinstance(item, RelayMiddleware)] + instances.extend( + item + for stack in graph.subagents + for item in stack + if isinstance(item, RelayMiddleware) + ) + callback_manager = graph.config.get("callbacks") + callbacks = callback_manager.handlers if callback_manager is not None else [] + return { + "instances": len(instances), + "distinct": len({id(item) for item in instances}), + "callbacks": len(callbacks), + "callback_manager": isinstance( + callback_manager, MetadataOnlyCallbackManager + ) if callback_manager is not None else False, + "metadata_only_callback": all( + isinstance(callback, MetadataOnlyCallback) for callback in callbacks + ), + "tags": graph.config.get("tags"), + "metadata": graph.config.get("metadata"), + } + os.environ.pop("NEMOCLAW_TOOL_DISCLOSURE", None) no_mcp = counts(agent.create_cli_agent(None, "assistant")) empty_mcp = counts(agent.create_cli_agent(None, "assistant", mcp_server_info=[Info(())])) @@ -312,6 +426,30 @@ active = counts(agent.create_cli_agent(None, "assistant", mcp_server_info=[Info( os.environ["NEMOCLAW_TOOL_DISCLOSURE"] = "direct" direct = counts(agent.create_cli_agent(None, "assistant", mcp_server_info=[Info(("mcp_echo",))])) +os.environ["NEMOCLAW_OBSERVABILITY"] = "true" +observability_noncanonical = observability_counts( + agent.create_cli_agent(None, "assistant") +) +os.environ["NEMOCLAW_OBSERVABILITY"] = "1" +observability_active = observability_counts( + agent.create_cli_agent(None, "assistant") +) +observability_prebound_list = observability_counts( + agent.create_cli_agent( + None, + "assistant", + graph_config={"callbacks": [HostileCallback()]}, + ) +) +observability_prebound_manager = observability_counts( + agent.create_cli_agent( + None, + "assistant", + graph_config={"callbacks": NormalCallbackManager([HostileCallback()])}, + ) +) +os.environ.pop("NEMOCLAW_OBSERVABILITY", None) + original_factory = agent._nemoclaw_original_create_cli_agent reached_original = [] @@ -378,6 +516,10 @@ print(json.dumps({ "direct_collisions": direct_collisions, "reached_original": reached_original, "direct": direct, + "observability_noncanonical": observability_noncanonical, + "observability_active": observability_active, + "observability_prebound_list": observability_prebound_list, + "observability_prebound_manager": observability_prebound_manager, "invalid": invalid, })) `; @@ -473,7 +615,12 @@ describe("Deep Agents 0.1.30 progressive-disclosure build patch", () => { const first = runPatcher(fixture); expect(first.status, first.stderr).toBe(0); - const managedPaths = [...fixture.sourcePaths, fixture.modulePath, fixture.helperPath]; + const managedPaths = [ + ...fixture.sourcePaths, + fixture.modulePath, + fixture.observabilityModulePath, + fixture.helperPath, + ]; const firstBytes = snapshot(managedPaths); const second = runPatcher(fixture); expect(second.status, second.stderr).toBe(0); @@ -498,6 +645,16 @@ describe("Deep Agents 0.1.30 progressive-disclosure build patch", () => { firstBytes[fixture.agentPath].match(/ProgressiveToolDisclosureMiddleware\(\)/g), ).toHaveLength(2); expect(firstBytes[fixture.modulePath]).toBe(fs.readFileSync(middlewarePath, "utf8")); + expect(firstBytes[fixture.observabilityModulePath]).toBe( + fs.readFileSync(observabilityPath, "utf8"), + ); + expect(firstBytes[fixture.agentPath]).toContain( + '"callbacks": new_metadata_only_callback_manager()', + ); + expect(firstBytes[fixture.agentPath]).toContain("agent.config = {"); + expect(firstBytes[fixture.agentPath]).not.toContain( + 'with_config({"callbacks": new_metadata_only_callback_manager()})', + ); const wiring = runWiring(fixture); expect(wiring).toMatchObject({ @@ -508,6 +665,26 @@ describe("Deep Agents 0.1.30 progressive-disclosure build patch", () => { reached_original: [], invalid: "NEMOCLAW_TOOL_DISCLOSURE must be 'progressive' or 'direct'", }); + expect(wiring.observability_noncanonical).toEqual({ + instances: 0, + distinct: 0, + callbacks: 0, + callback_manager: false, + metadata_only_callback: true, + tags: ["managed-tag"], + metadata: { managed: "preserved" }, + }); + expect(wiring.observability_active).toEqual({ + instances: 3, + distinct: 3, + callbacks: 1, + callback_manager: true, + metadata_only_callback: true, + tags: ["managed-tag"], + metadata: { managed: "preserved" }, + }); + expect(wiring.observability_prebound_list).toEqual(wiring.observability_active); + expect(wiring.observability_prebound_manager).toEqual(wiring.observability_active); expect(wiring.progressive_collisions).toEqual({ regular_regular: expect.stringContaining("multiple registered implementations"), regular_mcp: expect.stringContaining("MCP metadata owners"), @@ -587,6 +764,38 @@ describe("Deep Agents 0.1.30 progressive-disclosure build patch", () => { expect(fs.existsSync(fixture.modulePath)).toBe(false); }); + it.each([ + ["progressive-disclosure", DISCLOSURE_MARKER], + ["observability", OBSERVABILITY_MARKER], + ])("rejects a fully installed package missing its %s marker", (boundary, marker) => { + const fixture = makePatchFixture(); + const first = runPatcher(fixture); + expect(first.status, first.stderr).toBe(0); + fs.writeFileSync( + fixture.agentPath, + fs.readFileSync(fixture.agentPath, "utf8").replace(`# ${marker}`, "# marker removed"), + "utf8", + ); + const before = snapshot([ + ...fixture.sourcePaths, + fixture.modulePath, + fixture.observabilityModulePath, + fixture.helperPath, + ]); + + const result = runPatcher(fixture); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(`Managed package ${boundary} patch is partial`); + expect( + snapshot([ + ...fixture.sourcePaths, + fixture.modulePath, + fixture.observabilityModulePath, + fixture.helperPath, + ]), + ).toEqual(before); + }); + it("rejects a partial package install with the middleware missing", () => { const fixture = makePatchFixture(); const first = runPatcher(fixture); @@ -601,6 +810,24 @@ describe("Deep Agents 0.1.30 progressive-disclosure build patch", () => { expect(fs.existsSync(fixture.modulePath)).toBe(false); }); + it("rejects a partial package install with the observability module missing", () => { + const fixture = makePatchFixture(); + const first = runPatcher(fixture); + expect(first.status, first.stderr).toBe(0); + fs.rmSync(fixture.observabilityModulePath); + const before = snapshot([...fixture.sourcePaths, fixture.modulePath, fixture.helperPath]); + + const result = runPatcher(fixture); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "Managed package patch is partial: observability module is missing", + ); + expect(snapshot([...fixture.sourcePaths, fixture.modulePath, fixture.helperPath])).toEqual( + before, + ); + expect(fs.existsSync(fixture.observabilityModulePath)).toBe(false); + }); + it("refuses to overwrite a conflicting installed middleware module", () => { const fixture = makePatchFixture(); fs.writeFileSync(fixture.modulePath, "# unexpected module\n", "utf8"); @@ -612,4 +839,16 @@ describe("Deep Agents 0.1.30 progressive-disclosure build patch", () => { expect(snapshot(fixture.sourcePaths)).toEqual(before); expect(fs.readFileSync(fixture.modulePath, "utf8")).toBe("# unexpected module\n"); }); + + it("refuses to overwrite a conflicting installed observability module", () => { + const fixture = makePatchFixture(); + fs.writeFileSync(fixture.observabilityModulePath, "# unexpected module\n", "utf8"); + const before = snapshot(fixture.sourcePaths); + const result = runPatcher(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Refusing to overwrite unexpected observability module"); + expect(snapshot(fixture.sourcePaths)).toEqual(before); + expect(fs.readFileSync(fixture.observabilityModulePath, "utf8")).toBe("# unexpected module\n"); + }); }); diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index ad20559834..7b82f47950 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -66,16 +66,21 @@ function makeLauncherProxyProbeFixture( const probePath = path.join(tempDir, "managed-dcode-probe.sh"); const probe = [ "#!/bin/bash -p", - "for name in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy OPENAI_PROXY NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT; do", + "for name in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy OPENAI_PROXY NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT NEMOCLAW_OBSERVABILITY; do", ' printf \'LAUNCHER_%s=%s\\n\' "$name" "${!name-__unset__}"', "done", "", ].join("\n"); const fixture = replaceManagedProxyFileConstants( - readAgentFile("dcode-launcher.sh").replace( - 'readonly MANAGED_DCODE_WRAPPER="/usr/local/lib/nemoclaw/dcode-wrapper.sh"', - `readonly MANAGED_DCODE_WRAPPER="${probePath}"`, - ), + readAgentFile("dcode-launcher.sh") + .replace( + 'readonly MANAGED_DCODE_WRAPPER="/usr/local/lib/nemoclaw/dcode-wrapper.sh"', + `readonly MANAGED_DCODE_WRAPPER="${probePath}"`, + ) + .replace( + 'readonly MANAGED_OBSERVABILITY_MARKER="/tmp/nemoclaw-observability-enabled"', + `readonly MANAGED_OBSERVABILITY_MARKER="${path.join(tempDir, "observability-enabled")}"`, + ), tempDir, ); fs.writeFileSync(probePath, probe, "utf8"); @@ -89,19 +94,25 @@ function makeLauncherProxyProbeFixture( function makeStartProxyProbeFixture( tempDir: string, managedProxy: { host: string; port: string } = DEFAULT_MANAGED_PROXY, -): { envFile: string; scriptPath: string } { +): { envFile: string; markerFile: string; scriptPath: string } { const envFile = path.join(tempDir, "proxy-env.sh"); + const markerFile = path.join(tempDir, "observability-enabled"); const scriptPath = path.join(tempDir, "start.sh"); const fixture = replaceManagedProxyFileConstants(readAgentFile("start.sh"), tempDir) .replace("local target=/tmp/nemoclaw-proxy-env.sh", `local target="${envFile}"`) .replace( 'tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"', `tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`, + ) + .replace("local target=/tmp/nemoclaw-observability-enabled", `local target="${markerFile}"`) + .replace( + 'tmp="$(mktemp /tmp/nemoclaw-observability-enabled.XXXXXX)"', + `tmp="$(mktemp "${tempDir}/nemoclaw-observability-enabled.XXXXXX")"`, ); fs.writeFileSync(scriptPath, fixture, "utf8"); writeManagedProxyFiles(tempDir, managedProxy); fs.chmodSync(scriptPath, 0o755); - return { envFile, scriptPath }; + return { envFile, markerFile, scriptPath }; } function runLauncher( @@ -198,6 +209,62 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(output).not.toContain("all-password"); }); + it("recovers only the exact entrypoint observability bit for raw dcode exec", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-observability-")); + const launcherPath = makeLauncherProxyProbeFixture(tempDir); + const { markerFile, scriptPath } = makeStartProxyProbeFixture(tempDir); + + const noncanonicalStart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + NEMOCLAW_OBSERVABILITY: "true", + }, + encoding: "utf8", + }); + const noncanonicalLaunch = runLauncher(launcherPath, [], { + NEMOCLAW_OBSERVABILITY: "1", + }); + expect(noncanonicalStart.status, noncanonicalStart.stderr).toBe(0); + expect(fs.existsSync(markerFile)).toBe(false); + expect(noncanonicalLaunch.status, noncanonicalLaunch.stderr).toBe(0); + expect(noncanonicalLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=__unset__"); + + const enabledStart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + NEMOCLAW_OBSERVABILITY: "1", + }, + encoding: "utf8", + }); + const enabledLaunch = runLauncher(launcherPath, [], {}); + expect(enabledStart.status, enabledStart.stderr).toBe(0); + expect(fs.readFileSync(markerFile, "utf8")).toBe("1\n"); + expect(fs.statSync(markerFile).mode & 0o777).toBe(0o444); + expect(enabledLaunch.status, enabledLaunch.stderr).toBe(0); + expect(enabledLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=1"); + }); + + it("ignores tampered and non-regular observability markers", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-observability-")); + const launcherPath = makeLauncherProxyProbeFixture(tempDir); + const markerFile = path.join(tempDir, "observability-enabled"); + + fs.writeFileSync(markerFile, "true\n", { encoding: "utf8", mode: 0o644 }); + const tamperedLaunch = runLauncher(launcherPath, [], { + NEMOCLAW_OBSERVABILITY: "1", + }); + expect(tamperedLaunch.status, tamperedLaunch.stderr).toBe(0); + expect(tamperedLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=__unset__"); + + fs.rmSync(markerFile); + fs.mkdirSync(markerFile); + const nonRegularLaunch = runLauncher(launcherPath, [], { + NEMOCLAW_OBSERVABILITY: "1", + }); + expect(nonRegularLaunch.status, nonRegularLaunch.stderr).toBe(0); + expect(nonRegularLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=__unset__"); + }); + it("pins validated proxy overrides into direct dcode execution paths (#6191)", () => { const dockerfile = readAgentFile("Dockerfile"); const launcher = readAgentFile("dcode-launcher.sh"); diff --git a/test/observability-otlp-policy-preset.test.ts b/test/observability-otlp-policy-preset.test.ts new file mode 100644 index 0000000000..864a91eef7 --- /dev/null +++ b/test/observability-otlp-policy-preset.test.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { filterSetupPolicyPresetsForAgent } from "../src/lib/onboard/agent-policy-presets"; +import * as policies from "../src/lib/policy"; + +type RestRule = { allow?: { method?: string; path?: string } }; +type RestEndpoint = { + host?: string; + port?: number; + rules?: RestRule[]; +}; +type ObservabilityPreset = { + network_policies?: Record< + string, + { endpoints?: RestEndpoint[]; binaries?: Array<{ path: string }> } + >; +}; + +function loadObservabilityPreset(): ObservabilityPreset { + return YAML.parse(String(policies.loadPreset("observability-otlp-local"))); +} + +function allows(endpoint: RestEndpoint, host: string, method: string, path: string): boolean { + return ( + endpoint.host === host && + endpoint.rules?.some((rule) => rule.allow?.method === method && rule.allow.path === path) === + true + ); +} + +describe("backend-neutral OTLP observability policy preset", () => { + it("keeps the built-in preset catalog complete", () => { + expect( + policies + .listPresets() + .map((preset) => preset.name) + .sort(), + ).toEqual([ + "brave", + "brew", + "claude-code", + "discord", + "github", + "huggingface", + "jira", + "local-inference", + "nous-audio", + "nous-browser", + "nous-code", + "nous-image", + "nous-web", + "npm", + "observability-otlp-local", + "openclaw-diagnostics-otel-local", + "openclaw-pricing", + "outlook", + "public-reference", + "pypi", + "slack", + "tavily", + "teams", + "telegram", + "weather", + "wechat", + "whatsapp", + ]); + }); + + it("is available only to LangChain Deep Agents Code", () => { + const namesFor = (agent: string) => + filterSetupPolicyPresetsForAgent(policies.listPresets(), agent).map((preset) => preset.name); + + expect(namesFor("langchain-deepagents-code")).toContain("observability-otlp-local"); + expect(namesFor("openclaw")).not.toContain("observability-otlp-local"); + expect(namesFor("hermes")).not.toContain("observability-otlp-local"); + }); + + it("permits only trace POSTs from managed Python", () => { + const parsed = loadObservabilityPreset(); + const policy = parsed.network_policies?.["observability-otlp-local"]; + + expect(policy?.endpoints).toEqual([ + { + host: "host.openshell.internal", + port: 4318, + protocol: "rest", + enforcement: "enforce", + allowed_ips: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + rules: [{ allow: { method: "POST", path: "/v1/traces" } }], + }, + ]); + expect(policy?.binaries).toEqual([{ path: "/opt/venv/bin/python3*" }]); + }); + + it.each([ + ["non-POST method", "host.openshell.internal", "GET", "/v1/traces"], + ["alternate path", "host.openshell.internal", "POST", "/v1/logs"], + ["path suffix", "host.openshell.internal", "POST", "/v1/traces/extra"], + ["alternate host", "collector.example", "POST", "/v1/traces"], + ])("denies %s (#3915)", (_label, host, method, path) => { + const parsed = loadObservabilityPreset(); + const endpoint = parsed.network_policies?.["observability-otlp-local"]?.endpoints?.[0]; + + expect(endpoint).toBeDefined(); + expect(allows(endpoint ?? {}, host, method, path)).toBe(false); + }); + + it("contains no exporter credential or header configuration (#3915)", () => { + const parsed = loadObservabilityPreset(); + const endpoint = parsed.network_policies?.["observability-otlp-local"]?.endpoints?.[0]; + + expect(allows(endpoint ?? {}, "host.openshell.internal", "POST", "/v1/traces")).toBe(true); + expect(JSON.stringify(parsed)).not.toMatch( + /authorization|cookie|credential|headers?|langsmith|secret|token/i, + ); + }); +}); diff --git a/test/onboard-mcp-observability-redirect.test.ts b/test/onboard-mcp-observability-redirect.test.ts new file mode 100644 index 0000000000..0b25b8037f --- /dev/null +++ b/test/onboard-mcp-observability-redirect.test.ts @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, it } from "vitest"; + +const repoRoot = path.join(import.meta.dirname, ".."); + +describe("onboard managed MCP recreation redirect", () => { + it("prints the explicit observability opt-out in the transactional rebuild command", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-observability-redirect-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "redirect.js"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); + const mocksPath = JSON.stringify( + path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), + ); + const script = String.raw` +const runner = require(${runnerPath}); +const registry = require(${registryPath}); +const normalize = (command) => (Array.isArray(command) ? command.join(" ") : String(command)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = (command) => { + const value = normalize(command); + if (value.includes("sandbox get alpha")) return "alpha"; + if (value.includes("sandbox list")) return "alpha Ready"; + const mocked = require(${mocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok" }); + return mocked === null ? "" : mocked; +}; +registry.getSandbox = () => ({ + name: "alpha", + agent: "langchain-deepagents-code", + model: "model", + provider: "provider", + preferredInferenceApi: "openai-completions", + toolDisclosure: "progressive", + observabilityEnabled: true, + mcp: { + version: 1, + bridges: { + search: { + server: "search", + agent: "langchain-deepagents-code", + url: "https://mcp.example.test", + env: [], + policyName: "mcp-bridge-search", + addedAt: "2026-07-07T00:00:00.000Z" + } + } + } +}); +registry.getDefault = () => null; +const { createSandbox } = require(${onboardPath}); +createSandbox( + null, "model", "provider", "openai-completions", "alpha", null, null, null, + { name: "langchain-deepagents-code" }, null, null, null, [], null, + { + recreate: true, + toolDisclosure: "progressive", + observabilityEnabled: false, + observabilityRequestedExplicitly: true + } +).then(() => process.exit(91)).catch((error) => { console.error(error); process.exit(92); }); +`; + fs.writeFileSync(scriptPath, script); + + try { + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", + }, + }); + + assert.equal(result.status, 1, result.stderr); + assert.match(result.stderr, /Refusing the generic onboard recreation path/); + assert.match( + result.stderr, + /nemoclaw alpha rebuild --yes --tool-disclosure progressive --no-observability/, + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/onboard-policy-suggestions.test.ts b/test/onboard-policy-suggestions.test.ts index 579420be65..b228dac898 100644 --- a/test/onboard-policy-suggestions.test.ts +++ b/test/onboard-policy-suggestions.test.ts @@ -19,6 +19,7 @@ const { computeSetupPresetSuggestions, filterSetupPolicyPresets, getSuggestedPol knownPresetNames: string[]; provider?: string | null; agent?: string | null; + observabilityEnabled?: boolean | null; webSearchConfig?: { fetchEnabled?: boolean; provider?: string | null } | null; webSearchSupported?: boolean | null; hermesToolGateways?: string[] | null; @@ -33,6 +34,7 @@ const { computeSetupPresetSuggestions, filterSetupPolicyPresets, getSuggestedPol enabledChannels?: string[] | null; provider?: string | null; agent?: string | null; + observabilityEnabled?: boolean | null; env?: NodeJS.ProcessEnv; webSearchConfig?: { fetchEnabled?: boolean; provider?: "brave" | "tavily" } | null; }) => string[]; @@ -45,6 +47,7 @@ const { mergeRequiredSetupPolicyPresets, suppressedAgentRequiredPresets } = enabledChannels?: string[] | null; hermesToolGateways?: string[] | null; agent?: string | null; + observabilityEnabled?: boolean | null; knownPresetNames?: string[] | Set | null; env?: NodeJS.ProcessEnv; tierName?: string | null; @@ -269,6 +272,24 @@ describe("onboard policy preset suggestions", () => { } }); + it("suggests local observability only for enabled Deep Agents Code", () => { + expect( + getSuggestedPolicyPresets({ + agent: "langchain-deepagents-code", + observabilityEnabled: true, + }), + ).toContain("observability-otlp-local"); + expect( + getSuggestedPolicyPresets({ + agent: "langchain-deepagents-code", + observabilityEnabled: false, + }), + ).not.toContain("observability-otlp-local"); + expect( + getSuggestedPolicyPresets({ agent: "openclaw", observabilityEnabled: true }), + ).not.toContain("observability-otlp-local"); + }); + it("balanced OpenClaw with web search returns exactly brave brew huggingface npm openclaw-pricing pypi and excludes weather", () => { const knownWithPricing = [...known, "openclaw-pricing"]; const suggestions = computeSetupPresetSuggestions("balanced", { @@ -357,6 +378,34 @@ describe("onboard policy preset suggestions", () => { expect(disabledSuggestions).not.toContain("openclaw-diagnostics-otel-local"); }); + it("adds the DCode observability preset only when enabled and non-restricted", () => { + const knownWithObservability = [...known, "observability-otlp-local"]; + expect( + computeSetupPresetSuggestions("balanced", { + enabledChannels: [], + knownPresetNames: knownWithObservability, + agent: "langchain-deepagents-code", + observabilityEnabled: true, + }), + ).toContain("observability-otlp-local"); + expect( + computeSetupPresetSuggestions("balanced", { + enabledChannels: [], + knownPresetNames: knownWithObservability, + agent: "langchain-deepagents-code", + observabilityEnabled: false, + }), + ).not.toContain("observability-otlp-local"); + expect( + computeSetupPresetSuggestions("restricted", { + enabledChannels: [], + knownPresetNames: knownWithObservability, + agent: "langchain-deepagents-code", + observabilityEnabled: true, + }), + ).not.toContain("observability-otlp-local"); + }); + it("returns balanced tier defaults without messaging presets when no channels enabled", () => { const suggestions = computeSetupPresetSuggestions("balanced", { enabledChannels: [], @@ -477,6 +526,7 @@ describe("onboard policy preset suggestions", () => { { name: "weather" }, { name: "openclaw-pricing" }, { name: "openclaw-diagnostics-otel-local" }, + { name: "observability-otlp-local" }, { name: "nous-web" }, { name: "nous-image" }, ]; @@ -491,6 +541,9 @@ describe("onboard policy preset suggestions", () => { "openclaw-pricing", "openclaw-diagnostics-otel-local", ]); + expect( + filterSetupPolicyPresetsForAgent(allPresets, "langchain-deepagents-code").map((p) => p.name), + ).toEqual(["weather", "observability-otlp-local"]); }); it("does not add explicitly requested Hermes Nous presets to OpenClaw suggestions", () => { @@ -709,6 +762,34 @@ describe("onboard policy preset suggestions", () => { }); describe("mergeRequiredSetupPolicyPresets tier plumbing", () => { + it("adds enabled DCode observability and removes it when disabled or restricted", () => { + const options = { + agent: "langchain-deepagents-code", + knownPresetNames: ["npm", "observability-otlp-local"], + }; + expect( + mergeRequiredSetupPolicyPresets(["npm"], { + ...options, + observabilityEnabled: true, + tierName: "balanced", + }), + ).toEqual(["npm", "observability-otlp-local"]); + expect( + mergeRequiredSetupPolicyPresets(["npm", "observability-otlp-local"], { + ...options, + observabilityEnabled: false, + tierName: "balanced", + }), + ).toEqual(["npm"]); + expect( + mergeRequiredSetupPolicyPresets(["npm"], { + ...options, + observabilityEnabled: true, + tierName: "restricted", + }), + ).toEqual(["npm"]); + }); + it("suppresses openclaw-pricing only when tierName is restricted", () => { expect( mergeRequiredSetupPolicyPresets(["npm", "openclaw-pricing"], { diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 2ec8117241..bfdcfe1d77 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -101,6 +101,7 @@ registry.getSandbox = () => gpuEnabled: false, agent: "langchain-deepagents-code", dashboardPort: 18789, + observabilityEnabled: false, toolDisclosure: "progressive", } : null; diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 6889188576..c1b064e259 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -2626,7 +2626,7 @@ const registry = require(${registryPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); -const commands = []; +const commands = []; let registeredSandbox = null; runner.run = (command, opts = {}) => { commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; @@ -2644,7 +2644,7 @@ runner.runCapture = (command) => { return ""; }; registry.getSandbox = () => ({ name: "my-assistant", gpuEnabled: false }); -registry.registerSandbox = () => true; +registry.registerSandbox = (entry) => { registeredSandbox = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; @@ -2673,7 +2673,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); - console.log(JSON.stringify({ sandboxName, commands })); + console.log(JSON.stringify({ sandboxName, commands, registeredSandbox })); })().catch((error) => { console.error(error); process.exit(1); @@ -2689,6 +2689,7 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_POLICY_TIER: "restricted", }, }); @@ -2701,17 +2702,16 @@ const { createSandbox } = require(${onboardPath}); .find((line) => line.startsWith("{") && line.endsWith("}")); assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); const payload = JSON.parse(payloadLine); - assert.ok( payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox delete")), "should delete existing sandbox when --recreate-sandbox is set", ); assert.ok( - payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox create")), - "should create a new sandbox when --recreate-sandbox is set", + payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox create")) && + payload.registeredSandbox?.policyTier === "restricted", + "should create a sandbox and persist its tier before policy finalization", ); }); - it("recreate-sandbox flag backs up and restores workspace state", { timeout: 60_000, }, async () => { diff --git a/test/policies.test.ts b/test/policies.test.ts index 872c46b7ac..1d261ded26 100644 --- a/test/policies.test.ts +++ b/test/policies.test.ts @@ -145,42 +145,6 @@ describe("policies", () => { expect(whatsapp?.description).toBe("WhatsApp Web WebSocket and media access"); expect(whatsapp?.description).not.toContain("network_policies:"); }); - - it("returns expected preset names", () => { - const names = policies - .listPresets() - .map((p: { name: string }) => p.name) - .sort(); - const expected = [ - "brave", - "brew", - "claude-code", - "discord", - "github", - "huggingface", - "jira", - "local-inference", - "nous-audio", - "nous-browser", - "nous-code", - "nous-image", - "nous-web", - "npm", - "openclaw-diagnostics-otel-local", - "openclaw-pricing", - "outlook", - "public-reference", - "pypi", - "slack", - "tavily", - "teams", - "telegram", - "weather", - "wechat", - "whatsapp", - ]; - expect(names).toEqual(expected); - }); }); describe("loadPreset", () => { diff --git a/test/policy-tiers-onboard.test.ts b/test/policy-tiers-onboard.test.ts index fc1fa1596e..18280a6ccd 100644 --- a/test/policy-tiers-onboard.test.ts +++ b/test/policy-tiers-onboard.test.ts @@ -93,6 +93,7 @@ type SetupHarnessOptions = { policyPresets?: string; currentApplied?: string[]; customPresets?: TestPreset[]; + customOwnsObservability?: boolean; recordedPolicyTier?: string | null; env?: NodeJS.ProcessEnv; }; @@ -103,6 +104,7 @@ function createSetupHarness({ policyPresets = "", currentApplied = [], customPresets = [], + customOwnsObservability = false, recordedPolicyTier = null, env = {}, }: SetupHarnessOptions = {}) { @@ -116,6 +118,7 @@ function createSetupHarness({ const appliedCalls: string[] = []; const removedCalls: string[] = []; const tierUpdates: Array<{ sandboxName: string; policyTier: string }> = []; + const removedBuiltinAttributions: string[] = []; const deps: SetupPolicySelectionDeps = { policies: { @@ -128,6 +131,10 @@ function createSetupHarness({ ...customPresets, ], listCustomPresets: () => customPresets, + customPresetOwnsNetworkPolicyKey: () => customOwnsObservability, + removeBuiltinPresetAttribution: (_sandboxName, presetName) => { + removedBuiltinAttributions.push(presetName); + }, getAppliedPresets: () => [...currentApplied], clampSetupPolicyPresetNames: policy.clampSetupPolicyPresetNames, }, @@ -166,7 +173,15 @@ function createSetupHarness({ }, }; - return { appliedCalls, deps, notes, removedCalls, syncCalls, tierUpdates }; + return { + appliedCalls, + deps, + notes, + removedBuiltinAttributions, + removedCalls, + syncCalls, + tierUpdates, + }; } async function runPolicySetup( @@ -552,6 +567,57 @@ describe("policy tier setup", () => { assert.deepEqual(result.removedCalls, []); }); + it("treats exact custom OTLP ownership as attribution-only during non-interactive re-onboard", async () => { + const result = await runPolicySetup( + { + currentApplied: ["observability-otlp-local", "corp-otel"], + customPresets: [{ name: "corp-otel", description: "custom preset" }], + customOwnsObservability: true, + }, + { agent: "langchain-deepagents-code", observabilityEnabled: true }, + ); + + assert.ok(result.applied.includes("corp-otel")); + assert.ok(!result.applied.includes("observability-otlp-local")); + assert.ok(!result.removedCalls.includes("observability-otlp-local")); + assert.deepEqual(result.removedBuiltinAttributions, ["observability-otlp-local"]); + }); + + it("keeps exact custom OTLP ownership during selected resume without live built-in removal", async () => { + const result = await runPolicySetup( + { + currentApplied: ["observability-otlp-local", "corp-otel"], + customPresets: [{ name: "corp-otel", description: "custom preset" }], + customOwnsObservability: true, + }, + { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + selectedPresets: ["observability-otlp-local", "corp-otel"], + }, + ); + + assert.deepEqual(result.applied, ["corp-otel"]); + assert.deepEqual(result.removedCalls, []); + assert.deepEqual(result.removedBuiltinAttributions, ["observability-otlp-local"]); + }); + + it("does not let stale declared custom OTLP content suppress the required built-in", async () => { + const result = await runPolicySetup( + { + currentApplied: ["corp-otel"], + customPresets: [{ name: "corp-otel", description: "custom preset" }], + customOwnsObservability: false, + }, + { agent: "langchain-deepagents-code", observabilityEnabled: true }, + ); + + assert.ok(result.applied.includes("corp-otel")); + assert.ok(result.applied.includes("observability-otlp-local")); + assert.ok(result.appliedCalls.includes("observability-otlp-local")); + assert.deepEqual(result.removedBuiltinAttributions, []); + }); + it("falls back to tier suggestions when NEMOCLAW_POLICY_MODE is unknown (#2429)", async () => { const warnings = vi.spyOn(console, "warn").mockImplementation(() => undefined); const result = await runPolicySetup({ tierName: "balanced", policyMode: "restricted" }); @@ -659,6 +725,22 @@ describe("policy tier setup", () => { assert.ok(!result.appliedCalls.includes("openclaw-pricing")); }); + it("never applies DCode observability while an authoritative restricted rebuild tier is pending registration", async () => { + const result = await runPolicySetup( + { recordedPolicyTier: null }, + { + agent: "langchain-deepagents-code", + observabilityEnabled: true, + selectedPresets: ["observability-otlp-local"], + tierName: " Restricted ", + }, + ); + + assert.deepEqual(result.applied, []); + assert.ok(!result.appliedCalls.includes("observability-otlp-local")); + assert.deepEqual(result.syncCalls[0]?.selected, []); + }); + it("removes previously-applied OpenClaw pricing during a restricted resume", async () => { const result = await runPolicySetup( { recordedPolicyTier: "restricted", currentApplied: ["openclaw-pricing"] }, diff --git a/test/registry.test.ts b/test/registry.test.ts index c9141bf860..84d036308a 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -94,17 +94,24 @@ describe("registry", () => { name: "alpha", webSearchEnabled: true, toolDisclosure: "direct", + observabilityEnabled: true, fromDockerfile: "/tmp/Dockerfile.custom", hermesAuthMethod: "oauth", }); expect(registry.getSandbox("alpha")).toMatchObject({ webSearchEnabled: true, toolDisclosure: "direct", + observabilityEnabled: true, fromDockerfile: "/tmp/Dockerfile.custom", hermesAuthMethod: "oauth", }); }); + it("does not invent observability intent for legacy registry rows", () => { + registry.registerSandbox({ name: "legacy" }); + expect(registry.getSandbox("legacy").observabilityEnabled).toBeUndefined(); + }); + it("preserves missing tool-disclosure state on reconstructed legacy rows", () => { registry.registerSandbox({ name: "legacy" });