diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c1761a91a3e..51b1fb8af9b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2987,6 +2987,34 @@ jobs: set -euo pipefail printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" >> "${GITHUB_ENV}" + # invalidState: the cloud-onboard DCode TUI check requires a PTY driver, + # but the fixed GitHub-hosted runner image does not provide expect. + # sourceBoundary: this trusted workflow owns host setup; the repository + # check only verifies and consumes expect without privilege. + # whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host + # image, and caching privileged dpkg state between clean runners is not + # supported. + # regressionTest: workflow-boundary tests pin the ordering and exact + # one-package apt allowlist. + # removalCondition: remove when the hosted runner supplies expect or the + # cloud-onboard acceptance check no longer requires a PTY. + - name: Install cloud-onboard DCode TUI host dependencies + shell: bash + run: | + set -euo pipefail + for attempt in 1 2 3; do + if sudo apt-get update; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::apt-get update failed after 3 attempts." >&2 + exit 1 + fi + echo "::warning::apt-get update attempt ${attempt} failed; retrying." >&2 + sleep $((attempt * 5)) + done + sudo apt-get install -y --no-install-recommends expect + - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 26ce1accb88..b9604dcd5b2 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -57,7 +57,7 @@ RUN chmod 444 /opt/nemoclaw-deepagents-code/generate-config.ts /opt/nemoclaw-dee && cmp -s /usr/local/lib/nemoclaw/dcode-launcher.sh /usr/local/lib/nemoclaw/dcode-managed-exec \ && chmod -R a+rX /opt/nemoclaw-blueprint \ && test "$(find /opt/nemoclaw-deepagents-profile-plugin -type f -print | LC_ALL=C sort)" = "$(printf '%s\n' '/opt/nemoclaw-deepagents-profile-plugin/pyproject.toml' '/opt/nemoclaw-deepagents-profile-plugin/src/nemoclaw_deepagents_profile/__init__.py')" \ - && printf '%s %s\n' '75ff7e7a5142cad4305126ccb1b8fc756306e82d4c559ddbc624012fb54ebfc4' '/opt/nemoclaw-deepagents-profile-plugin/src/nemoclaw_deepagents_profile/__init__.py' '7ba7b77bd6f889cc861eddbe3e38fc1f4433a85b7bc2a9b516e19a19a37a7686' '/opt/nemoclaw-deepagents-profile-plugin/pyproject.toml' | sha256sum -c - \ + && printf '%s %s\n' '1cee6afafcbe545f5d095c94cb0ad81ff2a1512f84ad9d128a69a9b3d72b3def' '/opt/nemoclaw-deepagents-profile-plugin/src/nemoclaw_deepagents_profile/__init__.py' '7ba7b77bd6f889cc861eddbe3e38fc1f4433a85b7bc2a9b516e19a19a37a7686' '/opt/nemoclaw-deepagents-profile-plugin/pyproject.toml' | sha256sum -c - \ && /opt/venv/bin/pip3 install --no-index --no-cache-dir --no-deps --no-build-isolation /opt/nemoclaw-deepagents-profile-plugin \ && /opt/venv/bin/python3 -I -c 'import nemoclaw_deepagents_profile; print("NEMOCLAW_DCODE_PROFILE_" + "IMPORT_GATE", flush=True); import deepagents; import deepagents_code' \ && /opt/venv/bin/pip3 check \ diff --git a/agents/langchain-deepagents-code/dcode-launcher.sh b/agents/langchain-deepagents-code/dcode-launcher.sh index 7cbc98fd125..225c2f3e58a 100755 --- a/agents/langchain-deepagents-code/dcode-launcher.sh +++ b/agents/langchain-deepagents-code/dcode-launcher.sh @@ -13,7 +13,7 @@ unset _nemoclaw_auto_approval_env readonly MANAGED_DCODE_WRAPPER="/usr/local/lib/nemoclaw/dcode-wrapper.sh" readonly MANAGED_EXEC_LAUNCHER="/usr/local/lib/nemoclaw/dcode-managed-exec" -readonly MANAGED_OBSERVABILITY_MARKER="/tmp/nemoclaw-observability-enabled" +readonly MANAGED_OBSERVABILITY_MARKER="/sandbox/.deepagents/.nemoclaw-observability-enabled" export HOME=/sandbox export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/agents/langchain-deepagents-code/dependency-review.md b/agents/langchain-deepagents-code/dependency-review.md index 668374dcbf2..956a65bb113 100644 --- a/agents/langchain-deepagents-code/dependency-review.md +++ b/agents/langchain-deepagents-code/dependency-review.md @@ -23,7 +23,7 @@ NemoClaw no longer vendors or overlays that source. - Native profile SHA-256: `c8e8dd2b0182334b54be4f46ff0c7b45fbb95dc13bd9a92c249eb47a14fa13d7` - Unmodified built-in bootstrap SHA-256: `005a91e7fc4ca6b21220673dd9d02d6686bf63e1e4f1102d124b01f96886efcf` - First-party adapter: `nemoclaw-deepagents-profile==0.1.0` -- Adapter module SHA-256: `75ff7e7a5142cad4305126ccb1b8fc756306e82d4c559ddbc624012fb54ebfc4` +- Adapter module SHA-256: `1cee6afafcbe545f5d095c94cb0ad81ff2a1512f84ad9d128a69a9b3d72b3def` - Adapter project metadata SHA-256: `7ba7b77bd6f889cc861eddbe3e38fc1f4433a85b7bc2a9b516e19a19a37a7686` - Adapter wheel license expression: `Apache-2.0` - Adapter dependency audit result: `No known vulnerabilities found`. Its only @@ -73,19 +73,77 @@ without consulting an index. Its `deepagents.harness_profiles` entry point runs after built-in profiles are registered, reads the reviewed canonical profile through one exact-version/hash-gated private registry lookup, and uses Deep Agents' public registration API to map it to the two exact `openai:` model -keys used by NemoClaw's managed OpenAI-compatible `ChatOpenAI` route. The +keys used by NemoClaw's managed OpenAI-compatible `ChatOpenAI` route. It layers +one first-party middleware onto those aliases that rejects only a +case-insensitive `[content]` value, with optional whitespace around the token +and brackets, passed as the complete `execute` command; +the canonical NVIDIA profile and unrelated models remain unchanged. The released SDK has no public profile getter or alias API. The adapter does not add a provider-wide OpenAI profile. +### Managed Ultra compatibility workarounds + +Two localized behaviors close separate invalid states on the managed Ultra +aliases. They are not a new provider profile and do not modify the reviewed +canonical NVIDIA profile. + +The two managed model IDs remain language-local constants in the TypeScript +config generator and the isolated Python image/plugin validators. Those +components run on opposite sides of the offline wheel-install boundary, so a +shared runtime data file would enlarge the installed trust surface solely to +deduplicate two immutable strings. The focused profile-plugin suite extracts +the identifiers from every production consumer and requires the exact sets to +match, preventing drift without adding another mutable build artifact. + +For `force_nonempty_content`, the invalid state originates in the NVIDIA Ultra +chat template/serving path: a Chat Completions response that combines reasoning +and tool calls can otherwise carry empty assistant content. That response shape +is outside NemoClaw; this repository owns only the generated DCode provider +configuration, so `generate-config.ts` supplies the model-specific template +argument at that request boundary. Fixing the serving template, model, or +third-party client in this repository would require vendoring an upstream +component and would violate the released-dependency boundary. The focused config +tests prove both managed Ultra IDs receive the argument and unrelated models do +not; the Deep Agents E2E verifies the installed request shape. Remove this +argument only after a reviewed serving-template or client update produces +nonempty assistant content for reasoning-plus-tool-call turns without it, and +the live DCode Ultra E2E passes for both managed model IDs with the override +deleted. + +For the `[content]` guard, the invalid state is a model-produced tool call whose +complete `execute.command` is the placeholder, ignoring case and whitespace +around the token and brackets. The released Deep Agents parser/profile can carry that +argument to normal tool middleware, where an unrestricted execute backend would +otherwise treat it as a shell command. The model/provider emission and the +hash-locked `deepagents==0.7.0a6` canonical profile are upstream boundaries; +NemoClaw owns the two managed aliases and the final middleware immediately before +dispatch. The adapter therefore rejects only that observed complete argument and +leaves concrete commands, other tools, the canonical NVIDIA profile, and +unrelated models unchanged. Focused fixture tests plus the isolated image +validator cover sync and async rejection, concrete and non-execute pass-through, +and graph dispatch with shell restrictions disabled; the Deep Agents E2E repeats +the installed guard contract. Remove the guard only after a reviewed model, +serving-template, and Deep Agents update no longer emits or converts `[content]` +into an execute call across native and repaired tool-call paths, and those tests +plus the live DCode Ultra E2E pass with the middleware removed. + The adapter verifies the exact DCode and Deep Agents versions plus the official native-profile and bootstrap source hashes. It also binds the imported Deep Agents package to the distribution that supplied the reviewed version. -Registration is atomic, idempotent, and rejects missing canonical, partial, or -conflicting alias state. The image validator runs under isolated Python, -verifies the installed entry-point metadata and adapter source hash before the -upstream source checks, checks both upstream files again after profile loading, -resolves the complete native middleware for both aliases, compiles a graph, -proves parser/native dispatch parity, and confirms an unrelated OpenAI model +Registration uses the Deep Agents registry itself as its only idempotency +source, serializes the multi-key transaction for concurrent plugin discovery +within one Python interpreter, and rejects missing canonical, partial, or +conflicting alias state. The Deep Agents registry is process-local, so separate +agent processes have separate registries and cannot interleave writes; a +filesystem lock would not protect shared state. Revisit that assumption if an +upstream release moves the registry out of process. The image validator runs +under isolated Python, verifies the installed entry-point metadata and adapter +source hash before the upstream source checks, checks both upstream files again +after profile loading, +resolves the complete native middleware plus the managed guard for both aliases, +proves the canonical middleware remains unchanged, compiles a graph, exercises +sync and async placeholder rejection, proves concrete-command and parser/native +dispatch parity through the actual graph, and confirms an unrelated OpenAI model receives no Ultra behavior. The Docker build separately imports the adapter, Deep Agents, and DCode under isolated Python immediately after installation; the validator then binds the installed module to its distribution and rechecks @@ -98,7 +156,8 @@ the fake-Docker unit suite separately pins its diagnostic failure branches. The reviewed native-profile and bootstrap files stay byte-for-byte unchanged. Focused fixtures cover the reviewed version/hash, missing-source, -missing-canonical, partial/conflicting, rollback, and idempotence states. The +missing-canonical, partial/conflicting, rollback, idempotence, exact placeholder +rejection, and unchanged concrete-command states. The deleted source-backport license path, `LICENSE.langchain-deepagents`, is not staged into the image, and image regression tests enforce that absence. @@ -115,3 +174,37 @@ this review to revalidate the managed adapter. Remove it instead of refreshing its hashes only if a future reviewed dependency already provides both exact mappings; no external contribution is required. Issue #6424 records the NemoClaw-owned replacement of the previous installed-bootstrap mutation. + +## Managed observability and ordered policy cleanup + +The managed observability marker closes a sandbox lifecycle gap rather than an +authorization gap. OpenShell policy replacement can clear ephemeral `/tmp`, and +independent sandbox exec/login processes do not inherit the entrypoint's +environment, while the host registry and the active OTLP network policy remain +enabled. OpenShell owns those lifecycle semantics; NemoClaw owns the DCode +startup and launcher boundary but does not modify OpenShell here. Create, +rebuild, and snapshot-clone paths pass an explicit `1` or `0`; an environment- +less policy restart preserves the validated durable state. The startup script +writes only the credential-free enable bit to persistent +`/sandbox/.deepagents/.nemoclaw-observability-enabled`. The launcher accepts +only a non-symlink regular marker containing exactly `1`, and the network policy +remains the authority for OTLP access. + +Focused launcher fixtures delete unrelated ephemeral state and prove the marker +survives, reject unsafe directory and marker types, and cover enabled and +disabled values. The ordered live checks prove Tavily removal restores the +deny-by-default policy while check 11 independently requires the host registry, +live policy, and durable sandbox marker to agree. Remove this marker and its +launcher recovery only when OpenShell propagates the selected observability bit +to every exec/login process across policy replacement, or when DCode no longer +needs the bit. + +Tavily cleanup persists across sandbox rebuilds because `policy-remove` first +applies the narrowed live policy and then removes the preset from the sandbox's +registry-backed policy list, which is the source used by rebuild. The +`policy-add-remove-session-sync` tests cover successful persisted removal, and +the snapshot regression `does not resurrect an earlier removed preset` guards +restore behavior. The E2E EXIT trap is still required for early probe failures +so the ordered suite cannot leave the current sandbox broader than the registry. +Remove that trap only when each check receives an isolated sandbox or no longer +mutates policy. diff --git a/agents/langchain-deepagents-code/generate-config.ts b/agents/langchain-deepagents-code/generate-config.ts index 897f41b2b73..46715343ac7 100644 --- a/agents/langchain-deepagents-code/generate-config.ts +++ b/agents/langchain-deepagents-code/generate-config.ts @@ -18,6 +18,11 @@ type Settings = { inferenceApi: string; }; +const NEMOTRON_ULTRA_MODEL_IDS = new Set([ + "nvidia/nemotron-3-ultra-550b-a55b", + "nvidia/nvidia/nemotron-3-ultra", +]); + function readSettings(env: NodeJS.ProcessEnv): Settings { const providerKey = normalizeCommentMetadata( env.NEMOCLAW_PROVIDER_KEY || "inference", @@ -94,6 +99,19 @@ function modelNameForOpenAiProvider(model: string): string { function buildConfig(settings: Settings): string { const model = modelNameForOpenAiProvider(settings.model); const defaultModel = `openai:${model}`; + // Source boundary: NVIDIA's Ultra serving template owns the empty assistant + // content behavior; this generator owns only the managed per-model request + // parameters. Keep the exact invalid state, regression proof, and separate + // removal conditions for this option and the dispatch guard in + // dependency-review.md under "Managed Ultra compatibility workarounds." + const modelParams = NEMOTRON_ULTRA_MODEL_IDS.has(model) + ? [ + "", + `[models.providers.openai.params.${tomlString(model)}]`, + "# Nemotron Ultra coding-agent requests need nonempty content when tool calls and reasoning are combined.", + "extra_body = { chat_template_kwargs = { force_nonempty_content = true } }", + ] + : []; return [ "# Generated by NemoClaw. This file contains no provider secrets.", `# NemoClaw provider route: ${settings.providerKey}; upstream provider: ${settings.upstreamProvider}; API: ${settings.inferenceApi}.`, @@ -111,6 +129,7 @@ function buildConfig(settings: Settings): string { "# NemoClaw-managed inference.local currently exposes Chat Completions.", "# Remove this override when that route supports OpenAI Responses API.", "use_responses_api = false", + ...modelParams, "", "[update]", "check = false", diff --git a/agents/langchain-deepagents-code/profile-plugin/src/nemoclaw_deepagents_profile/__init__.py b/agents/langchain-deepagents-code/profile-plugin/src/nemoclaw_deepagents_profile/__init__.py index 04835992f4a..220e844025c 100644 --- a/agents/langchain-deepagents-code/profile-plugin/src/nemoclaw_deepagents_profile/__init__.py +++ b/agents/langchain-deepagents-code/profile-plugin/src/nemoclaw_deepagents_profile/__init__.py @@ -7,7 +7,9 @@ import hashlib import importlib.metadata import importlib.util -from collections.abc import Callable, MutableMapping +import re +import threading +from collections.abc import Awaitable, Callable, MutableMapping from pathlib import Path from typing import Any @@ -25,12 +27,15 @@ "openai:nvidia/nemotron-3-ultra-550b-a55b", "openai:nvidia/nvidia/nemotron-3-ultra", ) +_INVALID_EXECUTE_COMMAND = re.compile(r"\[\s*content\s*\]", re.IGNORECASE) +_REGISTRATION_LOCK = threading.Lock() # invalidState: Deep Agents resolves pre-built ChatOpenAI models under `openai:` # keys, while its native Ultra profile is registered under an NVIDIA key. -# sourceBoundary: NemoClaw owns only these two managed inference aliases; the -# prompt, tool overrides, middleware, bootstrap, and canonical profile remain -# byte-identical Deep Agents artifacts. +# sourceBoundary: NemoClaw owns only these two managed inference aliases and one +# exact malformed-tool-call guard layered onto them; the prompt, tool overrides, +# bootstrap, canonical profile, and upstream source remain byte-identical Deep +# Agents artifacts. # whyPrivateRead: Deep Agents exposes public profile registration and plugin # hooks but no public getter/alias API. The exact version/source gates constrain # this single registry read; all writes use the public registration function. @@ -104,9 +109,73 @@ def _require_source(path: Path, label: str, expected_sha256: str) -> None: compile(source, str(path), "exec") +def _managed_profile_overlay() -> Any: + """Build the NemoClaw-only middleware layered onto managed Ultra aliases.""" + from deepagents.profiles.harness.harness_profiles import ( # noqa: PLC0415 + HarnessProfile, + ) + from langchain.agents.middleware.types import AgentMiddleware # noqa: PLC0415 + from langchain_core.messages import ToolMessage # noqa: PLC0415 + + class NemoClawExecutePlaceholderGuardMiddleware(AgentMiddleware): + """Reject Ultra's literal execute placeholder before shell dispatch.""" + + name = "NemoClawExecutePlaceholderGuardMiddleware" + + @staticmethod + def _reject(request: Any) -> ToolMessage | None: + tool_call = request.tool_call + name = tool_call.get("name") + if name != "execute": + return None + args = tool_call.get("args") + command = args.get("command") if isinstance(args, dict) else None + if not isinstance(command, str) or _INVALID_EXECUTE_COMMAND.fullmatch( + command.strip() + ) is None: + return None + return ToolMessage( + content=( + "Error: execute received the placeholder '[content]' instead of " + "a concrete shell command. Provide the complete command and do " + "not retry the placeholder." + ), + name=name, + tool_call_id=tool_call.get("id"), + status="error", + ) + + def wrap_tool_call( + self, + request: Any, + handler: Callable[[Any], Any], + ) -> Any: + """Reject the placeholder or delegate the original request unchanged.""" + rejected = self._reject(request) + if rejected is not None: + return rejected + return handler(request) + + async def awrap_tool_call( + self, + request: Any, + handler: Callable[[Any], Awaitable[Any]], + ) -> Any: + """Apply the same guard on the asynchronous tool-dispatch path.""" + rejected = self._reject(request) + if rejected is not None: + return rejected + return await handler(request) + + return HarnessProfile( + extra_middleware=[NemoClawExecutePlaceholderGuardMiddleware()] + ) + + def _register_aliases( registry: MutableMapping[str, Any], register_profile: Callable[[str, Any], None], + overlay: Any, ) -> None: native_profile = registry.get(CANONICAL_PROFILE_KEY) if native_profile is None: @@ -114,19 +183,46 @@ def _register_aliases( existing = tuple(key in registry for key in MANAGED_PROFILE_KEYS) if all(existing): - if all(registry[key] is native_profile for key in MANAGED_PROFILE_KEYS): + managed_profile = registry[MANAGED_PROFILE_KEYS[0]] + native_middleware = tuple(getattr(native_profile, "extra_middleware", ())) + managed_middleware = tuple(getattr(managed_profile, "extra_middleware", ())) + preserves_native_middleware = ( + len(managed_middleware) == len(native_middleware) + 1 + and all( + managed_item is native_item + for managed_item, native_item in zip( + managed_middleware, native_middleware, strict=False + ) + ) + ) + guard = managed_middleware[-1] if preserves_native_middleware else None + if ( + managed_profile is not native_profile + and all(registry[key] is managed_profile for key in MANAGED_PROFILE_KEYS) + and guard is not None + and type(guard).__name__ + == "NemoClawExecutePlaceholderGuardMiddleware" + and type(guard).__module__ == __name__ + ): return - raise _fail("managed aliases conflict with the reviewed canonical profile") + raise _fail("managed aliases conflict with the reviewed managed profile") if any(existing): raise _fail("managed aliases are in a partial registration state") try: - for key in MANAGED_PROFILE_KEYS: - register_profile(key, native_profile) - if not all(registry.get(key) is native_profile for key in MANAGED_PROFILE_KEYS): - raise _fail( - "managed alias registration did not preserve canonical identity" - ) + first_key, second_key = MANAGED_PROFILE_KEYS + register_profile(first_key, native_profile) + register_profile(first_key, overlay) + managed_profile = registry.get(first_key) + if managed_profile is None or managed_profile is native_profile: + raise _fail("managed profile overlay was not applied") + register_profile(second_key, managed_profile) + if registry.get(CANONICAL_PROFILE_KEY) is not native_profile: + raise _fail("canonical profile changed during managed registration") + if not all( + registry.get(key) is managed_profile for key in MANAGED_PROFILE_KEYS + ): + raise _fail("managed alias registration did not preserve managed identity") except Exception: for key in MANAGED_PROFILE_KEYS: registry.pop(key, None) @@ -154,7 +250,15 @@ def register() -> None: _HARNESS_PROFILES, ) - _register_aliases(_HARNESS_PROFILES, register_harness_profile) + # Plugin discovery can race when several managed agents initialize at once. + # The registry is the source of truth; this lock only makes its multi-key + # transaction atomic and stores no parallel registration state. + with _REGISTRATION_LOCK: + _register_aliases( + _HARNESS_PROFILES, + register_harness_profile, + _managed_profile_overlay(), + ) __all__ = ["register"] diff --git a/agents/langchain-deepagents-code/start.sh b/agents/langchain-deepagents-code/start.sh index 31c56b571cf..bf3cfaebcb1 100755 --- a/agents/langchain-deepagents-code/start.sh +++ b/agents/langchain-deepagents-code/start.sh @@ -183,13 +183,48 @@ prepare_runtime_env() { } prepare_observability_marker() { - local target=/tmp/nemoclaw-observability-enabled + local marker_dir=/sandbox/.deepagents + local target="${marker_dir}/.nemoclaw-observability-enabled" local tmp - if [ "${NEMOCLAW_OBSERVABILITY:-}" != "1" ]; then + + # OpenShell policy replacement can reset the sandbox's ephemeral /tmp while + # preserving its /sandbox workspace. Keep this credential-free convenience + # bit with the managed DCode state so independent exec/login shells retain + # the host-selected observability setting across policy updates. Reject a + # symlinked state directory before creating a same-directory temporary file; + # the marker remains non-authoritative and the network policy controls OTLP. + if [ -L "$marker_dir" ] || { [ -e "$marker_dir" ] && [ ! -d "$marker_dir" ]; }; then + printf '%s\n' 'Unsafe managed Deep Agents Code state directory.' >&2 + return 1 + fi + if [ -d "$marker_dir" ] \ + && { [ -L "$target" ] || { [ -e "$target" ] && [ ! -f "$target" ]; }; }; then + printf '%s\n' 'Unsafe managed observability marker target.' >&2 + return 1 + fi + + # Policy replacement restarts the entrypoint without the sandbox-create + # environment. Absent therefore preserves the validated durable state; + # NemoClaw create/rebuild paths pass an explicit authoritative 1 or 0. + if [ -z "${NEMOCLAW_OBSERVABILITY+x}" ]; then + return 0 + fi + if [ "$NEMOCLAW_OBSERVABILITY" != "1" ]; then + [ -d "$marker_dir" ] || return 0 rm -f "$target" return 0 fi - tmp="$(mktemp /tmp/nemoclaw-observability-enabled.XXXXXX)" + mkdir -p "$marker_dir" + if [ -L "$marker_dir" ] || [ ! -d "$marker_dir" ]; then + printf '%s\n' 'Unsafe managed Deep Agents Code state directory.' >&2 + return 1 + fi + if [ -L "$target" ] || { [ -e "$target" ] && [ ! -f "$target" ]; }; then + printf '%s\n' 'Unsafe managed observability marker target.' >&2 + return 1 + fi + + tmp="$(mktemp "${target}.XXXXXX")" printf '%s\n' '1' >"$tmp" chmod 444 "$tmp" mv -f "$tmp" "$target" diff --git a/agents/langchain-deepagents-code/validate-nemotron-ultra-profile.py b/agents/langchain-deepagents-code/validate-nemotron-ultra-profile.py index 697f4556405..6cbbf0934ec 100644 --- a/agents/langchain-deepagents-code/validate-nemotron-ultra-profile.py +++ b/agents/langchain-deepagents-code/validate-nemotron-ultra-profile.py @@ -4,6 +4,7 @@ from __future__ import annotations +import asyncio import hashlib import importlib.metadata import importlib.util @@ -21,6 +22,7 @@ ) from deepagents.profiles.harness.harness_profiles import ( HarnessProfile, + _HARNESS_PROFILES, _harness_profile_for_model, ) from deepagents_code.agent import create_cli_agent @@ -45,7 +47,7 @@ ) EXPECTED_PLUGIN_LICENSE_EXPRESSION = "Apache-2.0" EXPECTED_PLUGIN_SOURCE_SHA256 = ( - "75ff7e7a5142cad4305126ccb1b8fc756306e82d4c559ddbc624012fb54ebfc4" + "1cee6afafcbe545f5d095c94cb0ad81ff2a1512f84ad9d128a69a9b3d72b3def" ) EXPECTED_NATIVE_PROFILE_SHA256 = ( "c8e8dd2b0182334b54be4f46ff0c7b45fbb95dc13bd9a92c249eb47a14fa13d7" @@ -57,7 +59,7 @@ "nvidia/nemotron-3-ultra-550b-a55b", "nvidia/nvidia/nemotron-3-ultra", ) -EXPECTED_MIDDLEWARE = ( +EXPECTED_NATIVE_MIDDLEWARE = ( "NemotronProgressBudgetMiddleware", "NemotronPolicyNudgeMiddleware", "NemotronToolCallShim", @@ -71,8 +73,11 @@ "EntityResolutionGuardMiddleware", "FinalAnswerGuardMiddleware", ) +MANAGED_GUARD = "NemoClawExecutePlaceholderGuardMiddleware" +EXPECTED_MANAGED_MIDDLEWARE = (*EXPECTED_NATIVE_MIDDLEWARE, MANAGED_GUARD) DISPATCH_COMMAND = "printf NEMOCLAW_DISPATCH_OK" DENIED_DISPATCH_COMMAND = "uname -a" +PLACEHOLDER_COMMAND = "\t[ CONTENT ]\n" def require(condition: bool, message: str) -> None: @@ -130,11 +135,7 @@ def validate_profile_entry_point() -> None: group, name, value = EXPECTED_PROFILE_ENTRY_POINT group_entries = tuple(importlib.metadata.entry_points().select(group=group)) require(group_entries, f"profile entry point group {group!r} was not found") - matches = [ - entry_point - for entry_point in group_entries - if entry_point.name == name - ] + matches = [entry_point for entry_point in group_entries if entry_point.name == name] require(len(matches) == 1, f"expected exactly one {name!r} profile entry point") entry_point = matches[0] require( @@ -226,12 +227,25 @@ def make_model(model_id: str) -> ChatOpenAI: ) -def middleware_names(profile: HarnessProfile) -> tuple[str, ...]: +def middleware_items(profile: HarnessProfile) -> tuple[AgentMiddleware, ...]: middleware = profile.extra_middleware if callable(middleware): factory = cast(Callable[[], Sequence[AgentMiddleware]], middleware) middleware = factory() - return tuple(type(item).__name__ for item in middleware) + return tuple(middleware) + + +def middleware_names(profile: HarnessProfile) -> tuple[str, ...]: + return tuple(type(item).__name__ for item in middleware_items(profile)) + + +def validate_canonical_profile() -> None: + canonical = _HARNESS_PROFILES.get("nvidia:nvidia/nemotron-3-ultra-550b-a55b") + require(canonical is not None, "canonical Ultra profile is missing") + require( + middleware_names(canonical) == EXPECTED_NATIVE_MIDDLEWARE, + "canonical Ultra middleware was changed by the managed plugin", + ) def validate_profile(model_id: str) -> ChatOpenAI: @@ -253,12 +267,124 @@ def validate_profile(model_id: str) -> ChatOpenAI: f"{model_id}: read_file override is missing {argument}", ) require( - middleware_names(profile) == EXPECTED_MIDDLEWARE, - f"{model_id}: native middleware stack does not match the reviewed profile", + middleware_names(profile) == EXPECTED_MANAGED_MIDDLEWARE, + f"{model_id}: managed middleware stack does not match the reviewed profile", ) + canonical = _HARNESS_PROFILES["nvidia:nvidia/nemotron-3-ultra-550b-a55b"] + require(profile is not canonical, f"{model_id}: managed profile aliases canonical") return model +class GuardRequest: + """Minimal request shape consumed by the managed tool-call guard.""" + + def __init__(self, name: str, command: str, call_id: str) -> None: + self.tool_call = { + "name": name, + "args": {"command": command}, + "id": call_id, + } + + +def validate_direct_guard_contract() -> None: + """Exercise exact sync/async rejection without a shell backend.""" + profile = _harness_profile_for_model(make_model(MANAGED_MODEL_IDS[0]), None) + guards = [ + item + for item in middleware_items(profile) + if type(item).__name__ == MANAGED_GUARD + ] + require(len(guards) == 1, "managed execute guard is not unique") + guard = guards[0] + + sync_calls: list[GuardRequest] = [] + sync_request = GuardRequest("execute", PLACEHOLDER_COMMAND, "sync-placeholder") + + def sync_handler(request: GuardRequest) -> str: + sync_calls.append(request) + return "sync-handler-result" + + sync_result = guard.wrap_tool_call(sync_request, sync_handler) + require( + isinstance(sync_result, ToolMessage), "sync guard did not return ToolMessage" + ) + require(sync_calls == [], "sync placeholder reached the execute handler") + require(sync_result.tool_call_id == "sync-placeholder", "sync guard lost call id") + require(sync_result.name == "execute", "sync guard lost tool name") + require( + sync_result.status == "error", "sync guard did not mark the result as error" + ) + require( + isinstance(sync_result.content, str), + "sync guard result content is not text", + ) + require( + "placeholder '[content]'" in sync_result.content + and "complete command" in sync_result.content, + "sync guard result is not actionable", + ) + + async_calls: list[GuardRequest] = [] + async_request = GuardRequest("execute", "[content]", "async-placeholder") + + async def async_handler(request: GuardRequest) -> str: + async_calls.append(request) + return "async-handler-result" + + async_result = asyncio.run(guard.awrap_tool_call(async_request, async_handler)) + require( + isinstance(async_result, ToolMessage), "async guard did not return ToolMessage" + ) + require(async_calls == [], "async placeholder reached the execute handler") + require( + async_result.tool_call_id == "async-placeholder", "async guard lost call id" + ) + require(async_result.name == "execute", "async guard lost tool name") + require( + async_result.status == "error", "async guard did not mark the result as error" + ) + require( + isinstance(async_result.content, str), + "async guard result content is not text", + ) + require( + "placeholder '[content]'" in async_result.content + and "complete command" in async_result.content, + "async guard result is not actionable", + ) + + concrete_calls: list[GuardRequest] = [] + concrete_request = GuardRequest("execute", DISPATCH_COMMAND, "concrete-command") + + def concrete_handler(request: GuardRequest) -> str: + concrete_calls.append(request) + return "concrete-handler-result" + + concrete_result = guard.wrap_tool_call(concrete_request, concrete_handler) + require( + concrete_result == "concrete-handler-result", + "concrete execute handler result changed", + ) + require( + concrete_calls == [concrete_request], + "concrete execute request did not pass through unchanged", + ) + + other_calls: list[GuardRequest] = [] + other_request = GuardRequest("write_file", "[content]", "other-tool") + + def other_handler(request: GuardRequest) -> str: + other_calls.append(request) + return "other-handler-result" + + other_result = guard.wrap_tool_call(other_request, other_handler) + require(other_result == "other-handler-result", "non-execute result changed") + require( + other_calls == [other_request], + "non-execute placeholder request did not pass through unchanged", + ) + + def validate_parser_tool_visibility() -> None: cases = ( ('{"tool": "bash", "cmd": "echo blocked"}', "execute"), @@ -292,8 +418,10 @@ def validate_parser_tool_visibility() -> None: def dispatch_execute_once( first_response: AIMessage, + *, + restrict_shell: bool = True, ) -> tuple[tuple[tuple[str, int | None], ...], tuple[str, str | None]]: - """Run one model-produced execute call through DCode's managed allow-list.""" + """Run one model-produced execute call through the managed DCode graph.""" with tempfile.TemporaryDirectory(prefix="nemoclaw-profile-dispatch-") as tmp: backend = RecordingManagedShell(Path(tmp)) model = ScriptedManagedModel( @@ -309,16 +437,16 @@ def dispatch_execute_once( sandbox_type="nemoclaw-validation", system_prompt="Use the execute tool once, then report the result.", interactive=False, - auto_approve=False, - interrupt_shell_only=True, - shell_allow_list=["printf"], + auto_approve=not restrict_shell, + interrupt_shell_only=restrict_shell, + shell_allow_list=["printf"] if restrict_shell else None, enable_ask_user=False, enable_memory=False, enable_skills=False, ) result = graph.invoke( {"messages": [HumanMessage(content="Run the validation command once.")]}, - context={"auto_approve": False}, + context={"auto_approve": not restrict_shell}, ) execute_results = [ @@ -337,9 +465,12 @@ def dispatch_execute_once( def validate_dispatch_case( command: str, + *, + restrict_shell: bool = True, ) -> tuple[tuple[tuple[str, int | None], ...], tuple[str, str | None]]: repaired = dispatch_execute_once( - AIMessage(content=json.dumps({"tool": "bash", "cmd": command})) + AIMessage(content=json.dumps({"tool": "bash", "cmd": command})), + restrict_shell=restrict_shell, ) native = dispatch_execute_once( AIMessage( @@ -352,7 +483,8 @@ def validate_dispatch_case( "type": "tool_call", } ], - ) + ), + restrict_shell=restrict_shell, ) require(repaired == native, "repaired and native execute dispatch results differ") return repaired @@ -375,6 +507,29 @@ def validate_parser_dispatch_parity() -> None: "denied execute command did not preserve the managed rejection result", ) + # invalidState: a literal placeholder reaches an unrestricted shell backend. + # sourceBoundary: this assertion mirrors the profile plugin's `_reject` + # method and intentionally bypasses DCode's separate headless allow-list so + # it isolates the installed profile middleware. + # regressionTest: direct sync/async checks above and this real graph dispatch + # must both reject the whitespace-normalized placeholder. + # removalCondition: remove this case with the guard under the reviewed + # dependency-review.md condition; neither may outlive the other. + placeholder = validate_dispatch_case( + PLACEHOLDER_COMMAND, + restrict_shell=False, + ) + require(placeholder[0] == (), "execute placeholder reached the shell backend") + require( + placeholder[1][1] == "error", + "execute placeholder did not return an error", + ) + require( + "placeholder '[content]'" in placeholder[1][0] + and "complete command" in placeholder[1][0], + "execute placeholder rejection was not actionable", + ) + def main() -> None: for distribution, expected in EXPECTED_VERSIONS.items(): @@ -387,7 +542,9 @@ def main() -> None: validate_profile_entry_point() validate_official_sources() managed_models = [validate_profile(model_id) for model_id in MANAGED_MODEL_IDS] + validate_canonical_profile() validate_parser_tool_visibility() + validate_direct_guard_contract() validate_parser_dispatch_parity() # One graph construction materializes the shared middleware schemas and diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index aa2d0a14994..f77c00ad72f 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -952,11 +952,11 @@ describe("runSandboxSnapshot", () => { }); it.each([ - { enabled: true, assignmentPresent: true }, - { enabled: false, assignmentPresent: false }, + { enabled: true, expectedValue: "1" }, + { enabled: false, expectedValue: "0" }, ])("starts a snapshot clone with the authoritative source observability state when enabled=$enabled", async ({ enabled, - assignmentPresent, + expectedValue, }) => { let registeredClone: SandboxRecord | null = null; registerSandboxMock.mockImplementation((entry) => (registeredClone = entry as SandboxRecord)); @@ -986,7 +986,7 @@ describe("runSandboxSnapshot", () => { await runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }); const [createCommandValue, createEnv] = streamSandboxCreateMock.mock.calls[0] ?? []; const createCommand = String(createCommandValue ?? ""); - expect(createCommand.includes("'NEMOCLAW_OBSERVABILITY=1'")).toBe(assignmentPresent); + expect(createCommand).toContain(`'NEMOCLAW_OBSERVABILITY=${expectedValue}'`); expect(createEnv?.NEMOCLAW_OBSERVABILITY).toBeUndefined(); expect(registerSandboxMock).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 6e79e4c38e6..ba525d55e31 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -232,9 +232,11 @@ async function autoCreateSandboxFromSource( const openshellBin = getOpenshellBinary(); const sourceObservabilityEnabled = (srcEntry as { observabilityEnabled?: boolean }).observabilityEnabled === true; - const startupCommand = sourceObservabilityEnabled - ? ["env", "NEMOCLAW_OBSERVABILITY=1", "nemoclaw-start"] - : ["nemoclaw-start"]; + const startupCommand = [ + "env", + `NEMOCLAW_OBSERVABILITY=${sourceObservabilityEnabled ? "1" : "0"}`, + "nemoclaw-start", + ]; const createEnv = { ...process.env }; delete createEnv.NEMOCLAW_OBSERVABILITY; diff --git a/src/lib/onboard/sandbox-create-launch-observability.test.ts b/src/lib/onboard/sandbox-create-launch-observability.test.ts index 6f237c5782b..ec0c51c9091 100644 --- a/src/lib/onboard/sandbox-create-launch-observability.test.ts +++ b/src/lib/onboard/sandbox-create-launch-observability.test.ts @@ -36,7 +36,7 @@ describe("prepareSandboxCreateLaunch observability", () => { expect(serialized).not.toContain("secret"); }); - it("does not forward observability for another agent or when disabled", () => { + it("forwards an explicit disabled state only to Deep Agents Code", () => { const render = (name: string, observabilityEnabled: boolean) => prepareSandboxCreateLaunch({ agent: { name } as any, @@ -52,10 +52,11 @@ describe("prepareSandboxCreateLaunch observability", () => { buildEnv: () => ({}), }); - expect(render("langchain-deepagents-code", false).envArgs).not.toContain( - "NEMOCLAW_OBSERVABILITY=1", + expect(render("langchain-deepagents-code", false).envArgs).toContain( + "NEMOCLAW_OBSERVABILITY=0", ); expect(render("hermes", true).envArgs).not.toContain("NEMOCLAW_OBSERVABILITY=1"); + expect(render("hermes", false).envArgs).not.toContain("NEMOCLAW_OBSERVABILITY=0"); }); it("never trusts ambient DCode auto-approval as sandbox runtime input (#6478)", () => { diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index f8c68952a5e..b61ec661046 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -169,8 +169,12 @@ describe("prepareSandboxCreateLaunch", () => { }); expect(result.effectiveDashboardPort).toBe("0"); - expect(result.envArgs).toEqual([]); - expect(result.sandboxStartupCommand).toEqual(["env", "nemoclaw-start"]); + expect(result.envArgs).toEqual(["NEMOCLAW_OBSERVABILITY=0"]); + expect(result.sandboxStartupCommand).toEqual([ + "env", + "NEMOCLAW_OBSERVABILITY=0", + "nemoclaw-start", + ]); }); it("drops credential-bearing proxy URLs from Deep Agents Code sandbox create env", () => { diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index f809c757349..7b904c24c97 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -118,9 +118,12 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San if (sandboxName) { envArgs.push(formatEnvAssignment("NEMOCLAW_SANDBOX_NAME", sandboxName)); } - if (input.observabilityEnabled === true) { - envArgs.push(formatEnvAssignment("NEMOCLAW_OBSERVABILITY", "1")); - } + envArgs.push( + formatEnvAssignment( + "NEMOCLAW_OBSERVABILITY", + input.observabilityEnabled === true ? "1" : "0", + ), + ); } appendExtraPlaceholderKeysEnvArg(envArgs, input.extraPlaceholderKeys, formatEnvAssignment); diff --git a/test/dcode-managed-exec.test.ts b/test/dcode-managed-exec.test.ts index a5e7cb34611..93cf99e20a1 100644 --- a/test/dcode-managed-exec.test.ts +++ b/test/dcode-managed-exec.test.ts @@ -37,7 +37,7 @@ function makeLauncherFixture(tempDir: string): { `readonly MANAGED_EXEC_LAUNCHER="${launcherPath}"`, ) .replace( - 'readonly MANAGED_OBSERVABILITY_MARKER="/tmp/nemoclaw-observability-enabled"', + 'readonly MANAGED_OBSERVABILITY_MARKER="/sandbox/.deepagents/.nemoclaw-observability-enabled"', `readonly MANAGED_OBSERVABILITY_MARKER="${markerPath}"`, ) .replace( diff --git a/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh b/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh index 7800cba9fdf..88668ed1953 100755 --- a/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh +++ b/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh @@ -33,6 +33,7 @@ encode_source() { profile_contract_source() { cat <<'PY' +import asyncio import hashlib import importlib.metadata from pathlib import Path @@ -40,7 +41,12 @@ import tomllib from deepagents.profiles import _builtin_profiles from deepagents.profiles.harness import _nvidia_nemotron_3_ultra -from deepagents.profiles.harness.harness_profiles import _harness_profile_for_model +from deepagents.profiles.harness.harness_profiles import ( + _HARNESS_PROFILES, + _harness_profile_for_model, +) +from deepagents_code.model_config import ModelConfig +from langchain_core.messages import ToolMessage from langchain_openai import ChatOpenAI CONFIG_PATH = Path("/sandbox/.deepagents/config.toml") @@ -53,13 +59,16 @@ MANAGED_MODEL_IDS = ( "nvidia/nemotron-3-ultra-550b-a55b", "nvidia/nvidia/nemotron-3-ultra", ) +EXPECTED_EXTRA_BODY = { + "chat_template_kwargs": {"force_nonempty_content": True}, +} EXPECTED_NATIVE_PROFILE_SHA256 = ( "c8e8dd2b0182334b54be4f46ff0c7b45fbb95dc13bd9a92c249eb47a14fa13d7" ) EXPECTED_BOOTSTRAP_SHA256 = ( "005a91e7fc4ca6b21220673dd9d02d6686bf63e1e4f1102d124b01f96886efcf" ) -EXPECTED_MIDDLEWARE = [ +EXPECTED_NATIVE_MIDDLEWARE = [ "NemotronProgressBudgetMiddleware", "NemotronPolicyNudgeMiddleware", "NemotronToolCallShim", @@ -73,6 +82,8 @@ EXPECTED_MIDDLEWARE = [ "EntityResolutionGuardMiddleware", "FinalAnswerGuardMiddleware", ] +MANAGED_GUARD = "NemoClawExecutePlaceholderGuardMiddleware" +EXPECTED_MIDDLEWARE = [*EXPECTED_NATIVE_MIDDLEWARE, MANAGED_GUARD] for distribution, expected in EXPECTED_VERSIONS.items(): actual = importlib.metadata.version(distribution) @@ -100,14 +111,26 @@ assert bootstrap_hash == EXPECTED_BOOTSTRAP_SHA256, bootstrap_hash config = tomllib.loads(CONFIG_PATH.read_text(encoding="utf-8")) default_model = config["models"]["default"] -assert default_model.removeprefix("openai:") in MANAGED_MODEL_IDS, default_model +configured_model_id = default_model.removeprefix("openai:") +assert configured_model_id in MANAGED_MODEL_IDS, default_model provider = config["models"]["providers"]["openai"] -assert provider["models"] == [default_model.removeprefix("openai:")] +assert provider["models"] == [configured_model_id] assert provider["api_key_env"] == "DEEPAGENTS_CODE_OPENAI_API_KEY" assert provider["base_url"] == "https://inference.local/v1" assert provider["enabled"] is True -assert provider["params"] == {"use_responses_api": False} +assert provider["params"] == { + "use_responses_api": False, + configured_model_id: {"extra_body": EXPECTED_EXTRA_BODY}, +} + +model_kwargs = ModelConfig.load(CONFIG_PATH).get_kwargs( + "openai", model_name=configured_model_id +) +assert model_kwargs == { + "use_responses_api": False, + "extra_body": EXPECTED_EXTRA_BODY, +} class ProfileOnlyChatOpenAI(ChatOpenAI): @@ -127,23 +150,31 @@ class ProfileOnlyChatOpenAI(ChatOpenAI): def make_model(model_id): - return ProfileOnlyChatOpenAI( + model = ProfileOnlyChatOpenAI( model=model_id, api_key="nemoclaw-managed-placeholder", base_url=provider["base_url"], - use_responses_api=provider["params"]["use_responses_api"], + **model_kwargs, ) + assert model.extra_body == EXPECTED_EXTRA_BODY + return model -def middleware_names(profile): +def middleware_items(profile): middleware_factory = profile.extra_middleware if callable(middleware_factory): - return [type(item).__name__ for item in middleware_factory()] - return [type(item).__name__ for item in middleware_factory] + return list(middleware_factory()) + return list(middleware_factory) + + +def middleware_names(profile): + return [type(item).__name__ for item in middleware_items(profile)] +managed_profiles = [] for model_id in MANAGED_MODEL_IDS: profile = _harness_profile_for_model(make_model(model_id), None) + managed_profiles.append(profile) suffix = profile.system_prompt_suffix assert suffix is not None for marker in ("", "", "", ""): @@ -156,6 +187,78 @@ for model_id in MANAGED_MODEL_IDS: assert argument in read_file_description assert middleware_names(profile) == EXPECTED_MIDDLEWARE, model_id +canonical_profile = _HARNESS_PROFILES[ + "nvidia:nvidia/nemotron-3-ultra-550b-a55b" +] +assert middleware_names(canonical_profile) == EXPECTED_NATIVE_MIDDLEWARE +assert all(profile is not canonical_profile for profile in managed_profiles) +assert managed_profiles[0] is managed_profiles[1] +guard = next( + item + for item in middleware_items(managed_profiles[0]) + if type(item).__name__ == MANAGED_GUARD +) + + +class GuardRequest: + def __init__(self, name, command, call_id): + self.tool_call = { + "name": name, + "args": {"command": command}, + "id": call_id, + } + + +sync_calls = [] +sync_request = GuardRequest("execute", "\t[ CONTENT ]\n", "e2e-sync") + + +def sync_handler(request): + sync_calls.append(request) + return "unexpected-sync-dispatch" + + +sync_result = guard.wrap_tool_call(sync_request, sync_handler) +assert isinstance(sync_result, ToolMessage) +assert sync_calls == [] +assert sync_result.tool_call_id == "e2e-sync" +assert sync_result.name == "execute" +assert sync_result.status == "error" +assert isinstance(sync_result.content, str) +assert "placeholder '[content]'" in sync_result.content +assert "complete command" in sync_result.content + +async_calls = [] +async_request = GuardRequest("execute", "[content]", "e2e-async") + + +async def async_handler(request): + async_calls.append(request) + return "unexpected-async-dispatch" + + +async_result = asyncio.run(guard.awrap_tool_call(async_request, async_handler)) +assert isinstance(async_result, ToolMessage) +assert async_calls == [] +assert async_result.tool_call_id == "e2e-async" +assert async_result.name == "execute" +assert async_result.status == "error" +assert isinstance(async_result.content, str) +assert "placeholder '[content]'" in async_result.content +assert "complete command" in async_result.content + +concrete_calls = [] +concrete_request = GuardRequest("execute", "printf concrete", "e2e-concrete") + + +def concrete_handler(request): + concrete_calls.append(request) + return "concrete-dispatch" + + +assert guard.wrap_tool_call(concrete_request, concrete_handler) == "concrete-dispatch" +assert concrete_calls == [concrete_request] + unrelated = _harness_profile_for_model(make_model("gpt-4.1-mini"), None) assert unrelated.system_prompt_suffix is None assert middleware_names(unrelated) == [] diff --git a/test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh b/test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh index 4f823fe607b..af2f6bc77e6 100755 --- a/test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh +++ b/test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh @@ -33,7 +33,7 @@ observability_marker_value() { # Expansion is intentionally deferred to the sandbox shell. # shellcheck disable=SC2016 openshell sandbox exec --name "$SANDBOX_NAME" -- \ - sh -c 'marker=/tmp/nemoclaw-observability-enabled; if test -f "$marker" && ! test -L "$marker"; then cat "$marker"; else printf "absent"; fi' \ + sh -c 'marker=/sandbox/.deepagents/.nemoclaw-observability-enabled; if test -f "$marker" && ! test -L "$marker"; then cat "$marker"; else printf "absent"; fi' \ 2>/dev/null } @@ -123,31 +123,34 @@ python_probe() { } restore_observability_state() { - local marker_after restore_output - [ "$OBSERVABILITY_MARKER_BEFORE" = "1" ] || return 0 - + local marker_after restore_output state_invalid=0 marker_after="$(observability_marker_value || true)" - if [ "$marker_after" != "1" ]; then - if ! restore_output="$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ + if [ "$OBSERVABILITY_MARKER_BEFORE" = "1" ] && [ "$marker_after" = "1" ]; then + pass "managed observability state remains enabled after policy-remove" + return 0 + fi + if [ "$OBSERVABILITY_MARKER_BEFORE" = "1" ]; then + fail_test "managed observability marker was lost after policy-remove" + fi + state_invalid=1 + + if [ "$marker_after" != "1" ] \ + && ! restore_output="$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ /usr/bin/env NEMOCLAW_OBSERVABILITY=1 \ /usr/local/bin/nemoclaw-start /usr/bin/true 2>&1)"; then - fail_test "could not restore managed observability after policy-remove: $restore_output" - return 1 - fi + fail_test "could not restore managed observability for ordered cleanup: $restore_output" + return 1 fi - - marker_after="$(observability_marker_value || true)" - if [ "$marker_after" = "1" ]; then - pass "managed observability state restores after policy-remove" - else + if [ "$(observability_marker_value || true)" != "1" ]; then fail_test "managed observability marker was not restored after policy-remove" return 1 fi + pass "managed observability state restored for ordered cleanup" + return "$state_invalid" } restore_tavily_denial() { local cleanup_status=0 remove_output post_remove_probe_output - OBSERVABILITY_MARKER_BEFORE="$(observability_marker_value || true)" if ! remove_output="$(nemoclaw_cli "$SANDBOX_NAME" policy-remove tavily --yes 2>&1)"; then fail_test "policy-remove tavily failed after the opt-in proof: $remove_output" cleanup_status=1 @@ -189,6 +192,7 @@ fi if [ "${NEMOCLAW_E2E_TAVILY_SELF_TEST:-}" = "restore-denial" ]; then OBSERVABILITY_MARKER_FIXTURE="$(mktemp)" + OBSERVABILITY_MARKER_BEFORE=1 printf '%s\n' "1" >"$OBSERVABILITY_MARKER_FIXTURE" trap 'rm -f "$OBSERVABILITY_MARKER_FIXTURE"' EXIT observability_marker_value() { @@ -197,16 +201,18 @@ if [ "${NEMOCLAW_E2E_TAVILY_SELF_TEST:-}" = "restore-denial" ]; then nemoclaw_cli() { [[ "$*" == "$SANDBOX_NAME policy-remove tavily --yes" ]] || return 1 [ "${NEMOCLAW_E2E_TAVILY_REMOVE_FIXTURE:-ok}" = "ok" ] || return 1 - printf '%s\n' "absent" >"$OBSERVABILITY_MARKER_FIXTURE" + if [ "${NEMOCLAW_E2E_TAVILY_MARKER_FIXTURE:-preserve}" = "lose" ]; then + printf '%s\n' "absent" >"$OBSERVABILITY_MARKER_FIXTURE" + fi } openshell() { [[ "$*" == "sandbox exec --name $SANDBOX_NAME -- /usr/bin/env NEMOCLAW_OBSERVABILITY=1 /usr/local/bin/nemoclaw-start /usr/bin/true" ]] || return 1 printf '%s\n' "1" >"$OBSERVABILITY_MARKER_FIXTURE" } - NEMOCLAW_E2E_POLICY_SETTLE_SECONDS=0 restore_tavily_denial + cleanup_status=0 + NEMOCLAW_E2E_POLICY_SETTLE_SECONDS=0 restore_tavily_denial || cleanup_status=$? [ "$(cat "$OBSERVABILITY_MARKER_FIXTURE")" = "1" ] - [ "$FAILED" -eq 0 ] - exit 0 + exit "$cleanup_status" fi if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1" >/dev/null; then @@ -240,6 +246,10 @@ APPLY_OUTPUT="$(nemoclaw_cli "$SANDBOX_NAME" policy-add tavily --yes 2>&1)" || { printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" exit 1 } +OBSERVABILITY_MARKER_BEFORE="$(observability_marker_value || true)" +if [ "$OBSERVABILITY_MARKER_BEFORE" != "1" ]; then + fail_test "managed observability marker is absent after policy-add" +fi trap restore_tavily_denial EXIT pass "tavily policy preset applies" diff --git a/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh b/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh index 4160a80f3db..a927f0634f9 100755 --- a/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh +++ b/test/e2e/e2e-cloud-experimental/checks/11-deepagents-code-observability.sh @@ -201,8 +201,10 @@ NODE )" [ "$registry_output" = "enabled" ] || fail "host registry does not record observability enabled" +# Expansion is intentionally deferred to the sandbox shell. +# shellcheck disable=SC2016 marker_output="$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ - sh -c 'test -f /tmp/nemoclaw-observability-enabled && cat /tmp/nemoclaw-observability-enabled' \ + sh -c 'marker=/sandbox/.deepagents/.nemoclaw-observability-enabled; test -f "$marker" && ! test -L "$marker" && cat "$marker"' \ 2>&1)" || fail "managed observability marker is absent" [ "$marker_output" = "1" ] || fail "managed observability marker has an unexpected value" pass "host registry, live policy, and sandbox marker agree on enabled observability" diff --git a/test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts b/test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts index e9cb8d57895..49652ba57b8 100644 --- a/test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts @@ -25,6 +25,15 @@ function readWorkflow(): Workflow { return YAML.parse(fs.readFileSync(WORKFLOW_PATH, "utf8")) as Workflow; } +function throwMissingStep(stepName: string): never { + throw new Error(`${stepName} step is missing`); +} + +function requireStepIndex(steps: WorkflowStep[], stepName: string): number { + const index = steps.findIndex((step) => step.name === stepName); + return index >= 0 ? index : throwMissingStep(stepName); +} + describe("inline E2E host dependency boundary", () => { it.each([ { @@ -39,6 +48,12 @@ describe("inline E2E host dependency boundary", () => { expected: "network-policy host dependency install must be exactly 'sudo apt-get install -y --no-install-recommends expect'", }, + { + jobName: "cloud-onboard", + stepName: "Install cloud-onboard DCode TUI host dependencies", + expected: + "cloud-onboard host dependency install must be exactly 'sudo apt-get install -y --no-install-recommends expect'", + }, { jobName: "issue-4434-tui-unreachable-inference", stepName: "Install issue #4434 host dependencies", @@ -59,4 +74,27 @@ describe("inline E2E host dependency boundary", () => { fs.rmSync(tmp, { recursive: true, force: true }); } }); + + it("keeps cloud-onboard host dependencies before workspace preparation", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-host-order-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow(); + const steps = workflow.jobs["cloud-onboard"].steps; + const installIndex = requireStepIndex( + steps, + "Install cloud-onboard DCode TUI host dependencies", + ); + const install = steps.splice(installIndex, 1)[0]!; + const prepareIndex = requireStepIndex(steps, "Prepare E2E workspace"); + steps.splice(prepareIndex + 1, 0, install); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eWorkflowBoundary(workflowPath)).toContain( + "cloud-onboard DCode TUI host dependencies must precede workspace prep", + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); }); diff --git a/test/e2e/support/platform-parity-cloud-experimental.test.ts b/test/e2e/support/platform-parity-cloud-experimental.test.ts index 6db622d0bcf..1e86941d1ad 100644 --- a/test/e2e/support/platform-parity-cloud-experimental.test.ts +++ b/test/e2e/support/platform-parity-cloud-experimental.test.ts @@ -310,13 +310,36 @@ describe("P0-E cloud-experimental parity guardrails", () => { }); it.each([ - ["BLOCKED:policy denied", "ok", 0, /returns to the default Tavily denial/], - ["REACHED:403", "ok", 1, /did not restore the default Tavily denial/], - ["BLOCKED:policy denied", "fail", 1, /policy-remove tavily failed/], - ])("restores Tavily denial after opt-in (%s/%s)", (fixture, removeFixture, status, expected) => { + [ + "BLOCKED:policy denied", + "ok", + "preserve", + 0, + /returns to the default Tavily denial/, + /remains enabled/, + ], + [ + "REACHED:403", + "ok", + "preserve", + 1, + /did not restore the default Tavily denial/, + /remains enabled/, + ], + [ + "BLOCKED:policy denied", + "fail", + "preserve", + 1, + /policy-remove tavily failed/, + /remains enabled/, + ], + ["BLOCKED:policy denied", "ok", "lose", 1, /marker was lost/, /restored for ordered cleanup/], + ])("restores Tavily denial after opt-in (%s/%s/%s)", (fixture, removeFixture, markerFixture, status, expected, markerExpected) => { const result = spawnSync("bash", [dcodeTavilyCheck], { encoding: "utf8", env: { + NEMOCLAW_E2E_TAVILY_MARKER_FIXTURE: markerFixture, NEMOCLAW_E2E_TAVILY_PROBE_FIXTURE: fixture, NEMOCLAW_E2E_TAVILY_REMOVE_FIXTURE: removeFixture, NEMOCLAW_E2E_TAVILY_SELF_TEST: "restore-denial", @@ -327,7 +350,7 @@ describe("P0-E cloud-experimental parity guardrails", () => { expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(status); expect(`${result.stdout}\n${result.stderr}`).toMatch(expected); - expect(result.stdout).toContain("managed observability state restores after policy-remove"); + expect(result.stdout).toMatch(markerExpected); }); it("keeps the managed DCode thread-auto-approval live check valid Bash (#6478)", () => { diff --git a/test/langchain-deepagents-code-config.test.ts b/test/langchain-deepagents-code-config.test.ts index bc393cac826..18c47b98354 100644 --- a/test/langchain-deepagents-code-config.test.ts +++ b/test/langchain-deepagents-code-config.test.ts @@ -70,6 +70,7 @@ describe("LangChain Deep Agents Code config generator", () => { "# NemoClaw provider route: inference; upstream provider: nvidia-prod; API: openai-completions.", ); expect(config).toContain("use_responses_api = false"); + expect(config).not.toContain("force_nonempty_content"); expect(config).toContain("check = false"); expect(config).toContain("auto_update = false"); expect(config).not.toMatch(/NVIDIA_API_KEY|OPENAI_API_KEY=|sk-/); @@ -82,6 +83,18 @@ describe("LangChain Deep Agents Code config generator", () => { expect(config).toContain('models = ["gpt-oss-120b"]'); }); + it.each([ + "nvidia/nemotron-3-ultra-550b-a55b", + "nvidia/nvidia/nemotron-3-ultra", + ])("adds the required coding-agent request options for %s", (model) => { + const config = runGenerator({ NEMOCLAW_MODEL: model }); + + expect(config).toContain(`[models.providers.openai.params."${model}"]`); + expect(config).toContain( + "extra_body = { chat_template_kwargs = { force_nonempty_content = true } }", + ); + }); + it("preserves colons that belong to the model ID", () => { const config = runGenerator({ NEMOCLAW_MODEL: "minimax/minimax-m2.5:free" }); diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index a984771b713..fa4d0ac7c92 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -765,7 +765,7 @@ describe("LangChain Deep Agents Code image contracts", () => { "uv tool run --python 3.13 pip-audit -r agents/langchain-deepagents-code/requirements.lock --progress-spinner off --disable-pip", ); expect(review).toContain("No known vulnerabilities found"); - expect(review).toContain("75ff7e7a5142cad4305126ccb1b8fc756306e82d4c559ddbc624012fb54ebfc4"); + expect(review).toContain("1cee6afafcbe545f5d095c94cb0ad81ff2a1512f84ad9d128a69a9b3d72b3def"); expect(review).toContain("7ba7b77bd6f889cc861eddbe3e38fc1f4433a85b7bc2a9b516e19a19a37a7686"); expect(review).toContain("Adapter dependency audit result: `No known vulnerabilities found`"); expect(review).toContain("Deep Agents Code `0.1.34` pins `deepagents==0.7.0a6`"); diff --git a/test/langchain-deepagents-code-nemotron-profile-plugin.test.ts b/test/langchain-deepagents-code-nemotron-profile-plugin.test.ts index 25a5cf36ccf..244b1cf31de 100644 --- a/test/langchain-deepagents-code-nemotron-profile-plugin.test.ts +++ b/test/langchain-deepagents-code-nemotron-profile-plugin.test.ts @@ -20,6 +20,14 @@ const pluginSourcePath = path.join( ); const pluginProjectPath = path.join(pluginProjectDir, "pyproject.toml"); const validatorPath = path.join(agentDir, "validate-nemotron-ultra-profile.py"); +const e2eProfileCheckPath = path.join( + repoRoot, + "test", + "e2e", + "e2e-cloud-experimental", + "checks", + "03-deepagents-code-nemotron-ultra-profile.sh", +); const pythonBin = execFileSync("python3", ["-c", "import sys; print(sys.executable)"], { encoding: "utf8", }).trim(); @@ -34,6 +42,7 @@ const MANAGED_MODEL_ALIASES = [ "openai:nvidia/nemotron-3-ultra-550b-a55b", "openai:nvidia/nvidia/nemotron-3-ultra", ] as const; +const MANAGED_MODEL_IDS = MANAGED_MODEL_ALIASES.map((alias) => alias.slice("openai:".length)); const NATIVE_PROFILE_SOURCE = `"""Focused native Nemotron profile fixture.""" @@ -55,15 +64,53 @@ type PluginFixture = { type ProbeResult = { aliases: boolean[]; + aliasesShareManagedProfile: boolean; + aliasMiddleware: string[]; + canonicalHasGuard: boolean; canonicalPresent: boolean; error: string | null; + guardProbe: { + async: { + content: string; + id: string; + legacyText: string; + name: string; + status: string; + text: string; + calls: number; + }; + concrete: { calls: number; command: string; result: string }; + internalWhitespace: { + calls: number; + content: string; + id: string; + status: string; + }; + nonExecute: { calls: number; result: string }; + sync: { + content: string; + id: string; + legacyText: string; + name: string; + status: string; + text: string; + calls: number; + }; + } | null; registryKeys: string[]; + unrelatedPresent: boolean; }; function sha256(value: string | Buffer): string { return createHash("sha256").update(value).digest("hex"); } +function managedUltraModelIdsIn(source: string): string[] { + return [ + ...new Set(source.match(/nvidia\/(?:nvidia\/)?nemotron-3-ultra(?:-550b-a55b)?/g) ?? []), + ].sort(); +} + function replaceHashDefinitions( source: string, replacements: readonly (readonly [name: string, currentHash: string, fixtureHash: string])[], @@ -113,10 +160,54 @@ function makePluginFixture( _HARNESS_PROFILES = {} +class HarnessProfile: + def __init__(self, *, extra_middleware=()): + self.extra_middleware = list(extra_middleware) + + def register_harness_profile(key, profile): - _HARNESS_PROFILES[key] = profile + existing = _HARNESS_PROFILES.get(key) + if existing is None: + _HARNESS_PROFILES[key] = profile + else: + _HARNESS_PROFILES[key] = HarnessProfile( + extra_middleware=[*existing.extra_middleware, *profile.extra_middleware] + ) if os.environ.get("NEMOCLAW_TEST_FAIL_KEY") == key: raise RuntimeError(f"injected registration failure for {key}") +`, + ); + writeFixtureFile( + root, + "langchain/agents/middleware/types.py", + "class AgentMiddleware:\n pass\n", + ); + writeFixtureFile( + root, + "langchain_core/messages.py", + `class TextAccessor(str): + def __call__(self): + return str(self) + + +class ToolMessage: + def __init__(self, *, content, name, tool_call_id, status): + self.content = content + self.name = name + self.tool_call_id = tool_call_id + self.status = status + + @property + def text(self): + if isinstance(self.content, str): + value = self.content + else: + value = "".join( + block if isinstance(block, str) else block.get("text", "") + for block in self.content + if isinstance(block, str) or block.get("type") == "text" + ) + return TextAccessor(value) `, ); const nativeProfilePath = writeFixtureFile( @@ -169,7 +260,7 @@ function makeValidatorDependencyStubRoot(): string { "deepagents/profiles/harness/_nvidia_nemotron_3_ultra.py": "class NemotronTextToolCallParser: pass\n", "deepagents/profiles/harness/harness_profiles.py": - "class HarnessProfile: pass\ndef _harness_profile_for_model(*args, **kwargs): return HarnessProfile()\n", + "_HARNESS_PROFILES = {}\nclass HarnessProfile: pass\ndef _harness_profile_for_model(*args, **kwargs): return HarnessProfile()\n", "deepagents_code/__init__.py": "", "deepagents_code/agent.py": "def create_cli_agent(*args, **kwargs): return None\n", "langchain/agents/middleware/types.py": "class AgentMiddleware: pass\n", @@ -305,19 +396,31 @@ function runPlugin( options: { additionalPythonRoots?: string[]; aliasState?: "complete" | "conflict" | "partial"; + concurrentRegisterCalls?: number; failKey?: string; + probeGuard?: boolean; registerCalls?: number; withCanonical?: boolean; + withUnrelated?: boolean; } = {}, ) { const pluginPath = prepareFixturePlugin(); const pluginRoot = path.dirname(path.dirname(pluginPath)); - const script = `import json -from deepagents.profiles.harness.harness_profiles import _HARNESS_PROFILES + const script = `import asyncio +import json +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier +from deepagents.profiles.harness.harness_profiles import HarnessProfile, _HARNESS_PROFILES -canonical = object() +class NativeMiddleware: + pass + +canonical = HarnessProfile(extra_middleware=[NativeMiddleware()]) if ${(options.withCanonical ?? true) ? "True" : "False"}: _HARNESS_PROFILES[${JSON.stringify(CANONICAL_MODEL_SPEC)}] = canonical +unrelated = object() +if ${options.withUnrelated ? "True" : "False"}: + _HARNESS_PROFILES["openai:gpt-4.1-mini"] = unrelated state = ${JSON.stringify(options.aliasState ?? "")} aliases = ${JSON.stringify(MANAGED_MODEL_ALIASES)} @@ -334,16 +437,146 @@ from nemoclaw_deepagents_profile import register error = None try: - for _ in range(${options.registerCalls ?? 1}): - register() + concurrent_calls = ${options.concurrentRegisterCalls ?? 0} + if concurrent_calls: + barrier = Barrier(concurrent_calls) + + def register_concurrently(): + barrier.wait() + register() + + with ThreadPoolExecutor(max_workers=concurrent_calls) as executor: + futures = [executor.submit(register_concurrently) for _ in range(concurrent_calls)] + for future in futures: + future.result() + else: + for _ in range(${options.registerCalls ?? 1}): + register() except Exception as exc: error = str(exc) +guard_probe = None +aliases_registered = [key in _HARNESS_PROFILES for key in aliases] +managed_profile = _HARNESS_PROFILES.get(aliases[0]) if all(aliases_registered) else None +alias_middleware = [ + type(item).__name__ + for item in getattr(managed_profile, "extra_middleware", ()) +] +if error is None and ${options.probeGuard ? "True" : "False"}: + guard = next( + item + for item in managed_profile.extra_middleware + if type(item).__name__ == "NemoClawExecutePlaceholderGuardMiddleware" + ) + + class Request: + def __init__(self, name, command, call_id): + self.tool_call = { + "name": name, + "args": {"command": command}, + "id": call_id, + } + + sync_calls = [] + sync_request = Request("execute", " [CONTENT] ", "sync-call") + + def sync_handler(request): + sync_calls.append(request) + return "sync-handler-result" + + sync_result = guard.wrap_tool_call(sync_request, sync_handler) + + async_calls = [] + async_request = Request("execute", "[content]", "async-call") + + async def async_handler(request): + async_calls.append(request) + return "async-handler-result" + + async_result = asyncio.run(guard.awrap_tool_call(async_request, async_handler)) + + concrete_calls = [] + concrete_request = Request("execute", "printf concrete", "concrete-call") + + def concrete_handler(request): + concrete_calls.append(request) + return "concrete-handler-result" + + concrete_result = guard.wrap_tool_call(concrete_request, concrete_handler) + + internal_whitespace_calls = [] + internal_whitespace_request = Request( + "execute", "\\t[ content ]\\n", "internal-whitespace-call" + ) + + def internal_whitespace_handler(request): + internal_whitespace_calls.append(request) + return "internal-whitespace-handler-result" + + internal_whitespace_result = guard.wrap_tool_call( + internal_whitespace_request, internal_whitespace_handler + ) + + non_execute_calls = [] + non_execute_request = Request("write_file", "[content]", "write-call") + + def non_execute_handler(request): + non_execute_calls.append(request) + return "non-execute-handler-result" + + non_execute_result = guard.wrap_tool_call(non_execute_request, non_execute_handler) + guard_probe = { + "sync": { + "content": sync_result.content, + "id": sync_result.tool_call_id, + "legacyText": sync_result.text(), + "name": sync_result.name, + "status": sync_result.status, + "text": str(sync_result.text), + "calls": len(sync_calls), + }, + "async": { + "content": async_result.content, + "id": async_result.tool_call_id, + "legacyText": async_result.text(), + "name": async_result.name, + "status": async_result.status, + "text": str(async_result.text), + "calls": len(async_calls), + }, + "concrete": { + "calls": len(concrete_calls), + "command": concrete_calls[0].tool_call["args"]["command"], + "result": concrete_result, + }, + "internalWhitespace": { + "calls": len(internal_whitespace_calls), + "content": internal_whitespace_result.content, + "id": internal_whitespace_result.tool_call_id, + "status": internal_whitespace_result.status, + }, + "nonExecute": { + "calls": len(non_execute_calls), + "result": non_execute_result, + }, + } + print(json.dumps({ - "aliases": [_HARNESS_PROFILES.get(key) is canonical for key in aliases], + "aliases": aliases_registered, + "aliasesShareManagedProfile": ( + all(aliases_registered) + and _HARNESS_PROFILES[aliases[0]] is _HARNESS_PROFILES[aliases[1]] + ), + "aliasMiddleware": alias_middleware, + "canonicalHasGuard": any( + type(item).__name__ == "NemoClawExecutePlaceholderGuardMiddleware" + for item in canonical.extra_middleware + ), "canonicalPresent": _HARNESS_PROFILES.get(${JSON.stringify(CANONICAL_MODEL_SPEC)}) is canonical, "error": error, + "guardProbe": guard_probe, "registryKeys": sorted(_HARNESS_PROFILES), + "unrelatedPresent": _HARNESS_PROFILES.get("openai:gpt-4.1-mini") is unrelated, })) raise SystemExit(1 if error else 0) `; @@ -418,6 +651,19 @@ describe("LangChain Deep Agents Code managed Nemotron profile plugin (#6424)", ( expect(project).toContain('"deepagents==0.7.0a6"'); }); + it("keeps language-local managed Ultra model ID allowlists in sync", () => { + const expected = [...MANAGED_MODEL_IDS].sort(); + for (const sourcePath of [ + path.join(agentDir, "generate-config.ts"), + validatorPath, + pluginSourcePath, + e2eProfileCheckPath, + ]) { + const source = fs.readFileSync(sourcePath, "utf8"); + expect(managedUltraModelIdsIn(source), path.relative(repoRoot, sourcePath)).toEqual(expected); + } + }); + it("accepts the exact plugin, then rejects source substitution", () => { const root = makeValidatorStubRoot("nemoclaw-managed-aliases"); expect(runEntryPointValidationWithRoots([root]).status).toBe(0); @@ -481,6 +727,31 @@ describe("LangChain Deep Agents Code managed Nemotron profile plugin (#6424)", ( expect(result.status, result.stderr).toBe(0); expect(result.probe.aliases).toEqual([true, true]); + expect(result.probe.aliasesShareManagedProfile).toBe(true); + expect(result.probe.aliasMiddleware).toEqual([ + "NativeMiddleware", + "NemoClawExecutePlaceholderGuardMiddleware", + ]); + expect(result.probe.canonicalHasGuard).toBe(false); + expect(result.probe.canonicalPresent).toBe(true); + expect(result.probe.registryKeys).toEqual( + [...MANAGED_MODEL_ALIASES, CANONICAL_MODEL_SPEC].sort(), + ); + expectOfficialSourcesUnchanged(fixture); + }); + + it("atomically registers one managed profile when plugin discovery races", () => { + const fixture = makePluginFixture(); + const result = runPlugin(fixture, { concurrentRegisterCalls: 8 }); + + expect(result.status, result.stderr).toBe(0); + expect(result.probe.aliases).toEqual([true, true]); + expect(result.probe.aliasesShareManagedProfile).toBe(true); + expect(result.probe.aliasMiddleware).toEqual([ + "NativeMiddleware", + "NemoClawExecutePlaceholderGuardMiddleware", + ]); + expect(result.probe.canonicalHasGuard).toBe(false); expect(result.probe.canonicalPresent).toBe(true); expect(result.probe.registryKeys).toEqual( [...MANAGED_MODEL_ALIASES, CANONICAL_MODEL_SPEC].sort(), @@ -488,6 +759,79 @@ describe("LangChain Deep Agents Code managed Nemotron profile plugin (#6424)", ( expectOfficialSourcesUnchanged(fixture); }); + it("rejects execute placeholder whitespace variants before sync and async dispatch", () => { + const fixture = makePluginFixture(); + const result = runPlugin(fixture, { probeGuard: true }); + + expect(result.status, result.stderr).toBe(0); + expect(result.probe.guardProbe).not.toBeNull(); + expect(result.probe.guardProbe?.sync).toMatchObject({ + id: "sync-call", + name: "execute", + status: "error", + calls: 0, + }); + expect(result.probe.guardProbe?.sync.content).toContain("placeholder '[content]'"); + expect(result.probe.guardProbe?.sync.content).toContain("complete command"); + expect(result.probe.guardProbe?.sync.text).toBe(result.probe.guardProbe?.sync.content); + expect(result.probe.guardProbe?.sync.legacyText).toBe(result.probe.guardProbe?.sync.content); + expect(result.probe.guardProbe?.async).toMatchObject({ + id: "async-call", + name: "execute", + status: "error", + calls: 0, + }); + expect(result.probe.guardProbe?.async.content).toContain("placeholder '[content]'"); + expect(result.probe.guardProbe?.async.content).toContain("complete command"); + expect(result.probe.guardProbe?.async.text).toBe(result.probe.guardProbe?.async.content); + expect(result.probe.guardProbe?.async.legacyText).toBe(result.probe.guardProbe?.async.content); + expect(result.probe.guardProbe?.concrete).toEqual({ + calls: 1, + command: "printf concrete", + result: "concrete-handler-result", + }); + expect(result.probe.guardProbe?.internalWhitespace).toMatchObject({ + calls: 0, + id: "internal-whitespace-call", + status: "error", + }); + expect(result.probe.guardProbe?.internalWhitespace.content).toContain( + "placeholder '[content]'", + ); + expect(result.probe.guardProbe?.nonExecute).toEqual({ + calls: 1, + result: "non-execute-handler-result", + }); + expectOfficialSourcesUnchanged(fixture); + }); + + it("pins guard validators to the ToolMessage string-content API", () => { + const requirements = fs.readFileSync(path.join(agentDir, "requirements.lock"), "utf8"); + const validator = fs.readFileSync(validatorPath, "utf8"); + const e2eCheck = fs.readFileSync(e2eProfileCheckPath, "utf8"); + + expect(requirements).toMatch(/^langchain-core==1\.4\.8 /m); + for (const source of [validator, e2eCheck]) { + expect(source).toContain("isinstance(sync_result.content, str)"); + expect(source).toContain("isinstance(async_result.content, str)"); + expect(source).toContain('"complete command" in sync_result.content'); + expect(source).toContain('"complete command" in async_result.content'); + expect(source).not.toContain("sync_result.text"); + expect(source).not.toContain("async_result.text"); + } + }); + + it("resolves a managed model before the E2E probe inspects lazy profile state", () => { + const e2eCheck = fs.readFileSync(e2eProfileCheckPath, "utf8"); + const managedResolution = e2eCheck.indexOf( + "_harness_profile_for_model(make_model(model_id), None)", + ); + const canonicalLookup = e2eCheck.indexOf("canonical_profile = _HARNESS_PROFILES["); + + expect(managedResolution).toBeGreaterThan(-1); + expect(canonicalLookup).toBeGreaterThan(managedResolution); + }); + it.each([ ["Deep Agents Code", { dcode: "0.1.35" }, "deepagents-code==0.1.34"], ["Deep Agents", { deepagents: "0.7.0a7" }, "deepagents==0.7.0a6"], @@ -582,12 +926,16 @@ describe("LangChain Deep Agents Code managed Nemotron profile plugin (#6424)", ( it("rolls back the first alias when the second registration fails", () => { const fixture = makePluginFixture(); - const result = runPlugin(fixture, { failKey: MANAGED_MODEL_ALIASES[1] }); + const result = runPlugin(fixture, { + failKey: MANAGED_MODEL_ALIASES[1], + withUnrelated: true, + }); expect(result.status).not.toBe(0); expect(result.probe.error).toContain("injected registration failure"); expect(result.probe.aliases).toEqual([false, false]); - expect(result.probe.registryKeys).toEqual([CANONICAL_MODEL_SPEC]); + expect(result.probe.unrelatedPresent).toBe(true); + expect(result.probe.registryKeys).toEqual([CANONICAL_MODEL_SPEC, "openai:gpt-4.1-mini"].sort()); expectOfficialSourcesUnchanged(fixture); }); }); diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index b38c806ad0d..e44db3a5186 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -22,8 +22,13 @@ const PROXY_URL_ENV_NAMES = ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_p const NO_PROXY_ENV_NAMES = ["NO_PROXY", "no_proxy"] as const; const CLEARED_PROXY_ENV_NAMES = ["ALL_PROXY", "all_proxy", "OPENAI_PROXY"] as const; const DEFAULT_MANAGED_PROXY = { host: "10.200.0.1", port: "3128" } as const; +const OBSERVABILITY_MARKER_NAME = ".nemoclaw-observability-enabled"; const TEST_OWNER_UID = process.getuid?.() ?? 0; +function observabilityMarkerPath(tempDir: string): string { + return path.join(tempDir, "persistent-dcode-state", OBSERVABILITY_MARKER_NAME); +} + function readAgentFile(name: string): string { return fs.readFileSync(path.join(agentDir, name), "utf8"); } @@ -78,8 +83,8 @@ function makeLauncherProxyProbeFixture( `readonly MANAGED_DCODE_WRAPPER="${probePath}"`, ) .replace( - 'readonly MANAGED_OBSERVABILITY_MARKER="/tmp/nemoclaw-observability-enabled"', - `readonly MANAGED_OBSERVABILITY_MARKER="${path.join(tempDir, "observability-enabled")}"`, + 'readonly MANAGED_OBSERVABILITY_MARKER="/sandbox/.deepagents/.nemoclaw-observability-enabled"', + `readonly MANAGED_OBSERVABILITY_MARKER="${observabilityMarkerPath(tempDir)}"`, ), tempDir, ); @@ -94,25 +99,24 @@ function makeLauncherProxyProbeFixture( function makeStartProxyProbeFixture( tempDir: string, managedProxy: { host: string; port: string } = DEFAULT_MANAGED_PROXY, -): { envFile: string; markerFile: string; scriptPath: string } { - const envFile = path.join(tempDir, "proxy-env.sh"); - const markerFile = path.join(tempDir, "observability-enabled"); +): { envFile: string; ephemeralDir: string; markerFile: string; scriptPath: string } { + const ephemeralDir = path.join(tempDir, "ephemeral-tmp"); + const envFile = path.join(ephemeralDir, "proxy-env.sh"); + const markerFile = observabilityMarkerPath(tempDir); + const markerDir = path.dirname(markerFile); const scriptPath = path.join(tempDir, "start.sh"); + fs.mkdirSync(ephemeralDir); const fixture = replaceManagedProxyFileConstants(readAgentFile("start.sh"), tempDir) .replace("local target=/tmp/nemoclaw-proxy-env.sh", `local target="${envFile}"`) .replace( 'tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"', - `tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`, + `tmp="$(mktemp "${ephemeralDir}/nemoclaw-proxy-env.XXXXXX")"`, ) - .replace("local target=/tmp/nemoclaw-observability-enabled", `local target="${markerFile}"`) - .replace( - 'tmp="$(mktemp /tmp/nemoclaw-observability-enabled.XXXXXX)"', - `tmp="$(mktemp "${tempDir}/nemoclaw-observability-enabled.XXXXXX")"`, - ); + .replace("local marker_dir=/sandbox/.deepagents", `local marker_dir="${markerDir}"`); fs.writeFileSync(scriptPath, fixture, "utf8"); writeManagedProxyFiles(tempDir, managedProxy); fs.chmodSync(scriptPath, 0o755); - return { envFile, markerFile, scriptPath }; + return { envFile, ephemeralDir, markerFile, scriptPath }; } function runLauncher( @@ -237,10 +241,10 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(output).not.toContain("all-password"); }); - it("recovers only the exact entrypoint observability bit for raw dcode exec", () => { + it("recovers the exact observability bit after ephemeral runtime state resets", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-observability-")); const launcherPath = makeLauncherProxyProbeFixture(tempDir); - const { markerFile, scriptPath } = makeStartProxyProbeFixture(tempDir); + const { ephemeralDir, markerFile, scriptPath } = makeStartProxyProbeFixture(tempDir); const noncanonicalStart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { env: { @@ -264,18 +268,43 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { }, encoding: "utf8", }); - const enabledLaunch = runLauncher(launcherPath, [], {}); expect(enabledStart.status, enabledStart.stderr).toBe(0); expect(fs.readFileSync(markerFile, "utf8")).toBe("1\n"); expect(fs.statSync(markerFile).mode & 0o777).toBe(0o444); + + const policyRestart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin" }, + encoding: "utf8", + }); + expect(policyRestart.status, policyRestart.stderr).toBe(0); + const restartedLaunch = runLauncher(launcherPath, [], {}); + expect(restartedLaunch.status, restartedLaunch.stderr).toBe(0); + expect(restartedLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=1"); + + fs.rmSync(ephemeralDir, { recursive: true, force: true }); + expect(fs.existsSync(markerFile)).toBe(true); + const enabledLaunch = runLauncher(launcherPath, [], {}); expect(enabledLaunch.status, enabledLaunch.stderr).toBe(0); expect(enabledLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=1"); + + fs.mkdirSync(ephemeralDir, { recursive: true }); + const disabledStart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + NEMOCLAW_OBSERVABILITY: "0", + }, + encoding: "utf8", + }); + expect(disabledStart.status, disabledStart.stderr).toBe(0); + expect(fs.existsSync(markerFile)).toBe(false); }); - it("ignores tampered and non-regular observability markers", () => { + it("ignores tampered, symlinked, and non-regular observability markers", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-observability-")); const launcherPath = makeLauncherProxyProbeFixture(tempDir); - const markerFile = path.join(tempDir, "observability-enabled"); + const { scriptPath } = makeStartProxyProbeFixture(tempDir); + const markerFile = observabilityMarkerPath(tempDir); + fs.mkdirSync(path.dirname(markerFile), { recursive: true }); fs.writeFileSync(markerFile, "true\n", { encoding: "utf8", mode: 0o644 }); const tamperedLaunch = runLauncher(launcherPath, [], { @@ -289,8 +318,31 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { const nonRegularLaunch = runLauncher(launcherPath, [], { NEMOCLAW_OBSERVABILITY: "1", }); + const nonRegularStart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + NEMOCLAW_OBSERVABILITY: "1", + }, + encoding: "utf8", + }); expect(nonRegularLaunch.status, nonRegularLaunch.stderr).toBe(0); expect(nonRegularLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=__unset__"); + expect(nonRegularStart.status).not.toBe(0); + expect(nonRegularStart.stderr).toContain("Unsafe managed observability marker target"); + + fs.rmSync(markerFile, { recursive: true }); + const symlinkTarget = path.join(tempDir, "observability-symlink-target"); + fs.writeFileSync(symlinkTarget, "1\n", "utf8"); + fs.symlinkSync(symlinkTarget, markerFile); + const symlinkedLaunch = runLauncher(launcherPath, [], {}); + const symlinkedStart = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin" }, + encoding: "utf8", + }); + expect(symlinkedLaunch.status, symlinkedLaunch.stderr).toBe(0); + expect(symlinkedLaunch.stdout).toContain("LAUNCHER_NEMOCLAW_OBSERVABILITY=__unset__"); + expect(symlinkedStart.status).not.toBe(0); + expect(symlinkedStart.stderr).toContain("Unsafe managed observability marker target"); }); it("pins validated proxy overrides into direct dcode execution paths (#6191)", () => { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 909c27938da..3f9ff3128ed 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -3791,6 +3791,33 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ } } + const cloudOnboardSteps = asSteps(asRecord(jobs["cloud-onboard"]).steps); + validateInlineHostDependencyInstall( + errors, + "cloud-onboard", + cloudOnboardSteps, + "Install cloud-onboard DCode TUI host dependencies", + ["expect"], + ); + const cloudOnboardHostDependencies = requireStep( + errors, + cloudOnboardSteps, + "Install cloud-onboard DCode TUI host dependencies", + ); + const cloudOnboardPrepareWorkspace = requireStep( + errors, + cloudOnboardSteps, + "Prepare E2E workspace", + ); + if ( + cloudOnboardHostDependencies && + cloudOnboardPrepareWorkspace && + cloudOnboardSteps.indexOf(cloudOnboardHostDependencies) >= + cloudOnboardSteps.indexOf(cloudOnboardPrepareWorkspace) + ) { + errors.push("cloud-onboard DCode TUI host dependencies must precede workspace prep"); + } + validateOpenShellVersionPinJob(errors, jobs); validateOnboardNegativePathsJob(errors, jobs); validateSkillAgentJob(errors, jobs);