From 82c16b60c3f080041d56f8734c76e64ed555ad93 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 09:11:07 -0700 Subject: [PATCH 01/26] feat: migrate sandbox operations to OpenShell gRPC Signed-off-by: Aaron Erickson --- package-lock.json | 65 +- package.json | 4 +- scripts/copy-openshell-protos.mjs | 15 + src/lib/actions/sandbox/connect.ts | 43 +- src/lib/actions/sandbox/policy-channel.ts | 4 +- src/lib/actions/sandbox/process-recovery.ts | 170 ++--- src/lib/actions/sandbox/skill-install.ts | 85 +-- .../openshell/forward-bridge-runner.ts | 53 ++ .../openshell/forward-bridge-state.ts | 206 ++++++ .../openshell/gateway-metadata.test.ts | 112 +++ .../adapters/openshell/gateway-metadata.ts | 196 +++++ .../openshell/grpc-migration-guard.test.ts | 67 ++ src/lib/adapters/openshell/grpc.ts | 667 ++++++++++++++++++ .../adapters/openshell/proto/datamodel.proto | 15 + .../adapters/openshell/proto/openshell.proto | 214 ++++++ src/lib/adapters/openshell/sync-runner.ts | 145 ++++ src/lib/adapters/openshell/timeouts.ts | 8 +- src/lib/onboard/dashboard.ts | 104 +-- src/lib/onboard/sandbox-verification-exec.ts | 19 +- src/lib/sandbox/version.test.ts | 63 +- src/lib/sandbox/version.ts | 56 +- src/lib/share-command-deps.ts | 40 +- src/lib/share-command.test.ts | 78 +- src/lib/share-command.ts | 138 +--- src/lib/skill-install.ts | 59 +- src/lib/state/sandbox.ts | 324 ++++----- src/lib/status-command-deps.ts | 34 +- src/lib/tunnel/services.ts | 36 +- src/lib/verify-deployment.ts | 2 +- test/rebuild-credential-preflight.test.ts | 3 + test/rebuild-shields-auto-unlock.test.ts | 3 + test/share-command-deps-probe-argv.test.ts | 60 +- test/share-command-remote-path.test.ts | 1 - test/snapshot.test.ts | 21 + 34 files changed, 2229 insertions(+), 881 deletions(-) create mode 100644 scripts/copy-openshell-protos.mjs create mode 100644 src/lib/adapters/openshell/forward-bridge-runner.ts create mode 100644 src/lib/adapters/openshell/forward-bridge-state.ts create mode 100644 src/lib/adapters/openshell/gateway-metadata.test.ts create mode 100644 src/lib/adapters/openshell/gateway-metadata.ts create mode 100644 src/lib/adapters/openshell/grpc-migration-guard.test.ts create mode 100644 src/lib/adapters/openshell/grpc.ts create mode 100644 src/lib/adapters/openshell/proto/datamodel.proto create mode 100644 src/lib/adapters/openshell/proto/openshell.proto create mode 100644 src/lib/adapters/openshell/sync-runner.ts diff --git a/package-lock.json b/package-lock.json index 62d8154c30..bb258c5b61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,8 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", "@oclif/core": "^4.10.5", "js-yaml": "^4.1.1", "p-retry": "^4.6.2", @@ -1766,6 +1768,37 @@ } } }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@j178/prek": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/@j178/prek/-/prek-0.3.6.tgz", @@ -1836,6 +1869,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@mariozechner/clipboard": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.5.tgz", @@ -2127,35 +2170,30 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/codegen": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dev": true, "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.1", @@ -2166,35 +2204,30 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/inquire": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/utf8": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@rolldown/binding-android-arm64": { @@ -2703,7 +2736,6 @@ "version": "25.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.2.tgz", "integrity": "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -3293,7 +3325,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3752,7 +3783,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4174,7 +4204,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -5125,7 +5154,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "dev": true, "license": "MIT" }, "node_modules/lodash.kebabcase": { @@ -5167,7 +5195,6 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "dev": true, "license": "Apache-2.0" }, "node_modules/lru-cache": { @@ -5677,7 +5704,6 @@ "version": "7.5.8", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.8.tgz", "integrity": "sha512-dvpCIeLPbXZS/Ete7yLaO7RenOdken2NHKykBXbsaGxZT0UTltcarBciw+A78SRQs9iMAAVpsYA+l8b1hTePIA==", - "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -5768,7 +5794,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6220,7 +6245,6 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -6525,7 +6549,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -6550,7 +6573,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -6569,7 +6591,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" diff --git a/package.json b/package.json index 676878c893..dd69804382 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "format:ts": "cd nemoclaw && npm run lint:fix && npm run format", "check:installer-hash": "bash scripts/check-installer-hash.sh", "typecheck": "tsc -p jsconfig.json", - "build:cli": "tsc -p tsconfig.src.json && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", + "build:cli": "tsc -p tsconfig.src.json && node scripts/copy-openshell-protos.mjs && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", "typecheck:cli": "tsc -p tsconfig.cli.json", "validate:configs": "tsx scripts/validate-configs.ts", "type-safety:hotspots": "tsx scripts/type-safety-hotspots.ts", @@ -47,6 +47,8 @@ }, "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", "@oclif/core": "^4.10.5", "js-yaml": "^4.1.1", "p-retry": "^4.6.2", diff --git a/scripts/copy-openshell-protos.mjs b/scripts/copy-openshell-protos.mjs new file mode 100644 index 0000000000..3c9c544723 --- /dev/null +++ b/scripts/copy-openshell-protos.mjs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const source = path.join(root, "src", "lib", "adapters", "openshell", "proto"); +const dest = path.join(root, "dist", "lib", "adapters", "openshell", "proto"); + +fs.mkdirSync(dest, { recursive: true }); +for (const entry of fs.readdirSync(source, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(".proto")) continue; + fs.copyFileSync(path.join(source, entry.name), path.join(dest, entry.name)); +} diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 47dd217609..3bca6dd379 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import os from "node:os"; +import { execTextSync } from "../../adapters/openshell/grpc"; import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { captureOpenshell, @@ -94,7 +95,7 @@ export function printSandboxConnectHelp(sandboxName = ""): void { console.log(""); console.log(" Options:"); console.log( - " --probe-only Run recovery checks and exit without opening SSH", + " --probe-only Run recovery checks and exit without opening a shell", ); console.log(" -h, --help Show this help"); console.log(""); @@ -225,31 +226,25 @@ function probeSandboxInferenceRoute( const boundedAttempts = Math.max(1, attempts); for (let attempt = 1; attempt <= boundedAttempts; attempt += 1) { - // Keep the shell string inside the sandbox: curl write-out, body capture, - // and status classification must run as one bounded probe. sandboxName - // remains an argv value, so no user input is interpolated into the script. - const probe = captureOpenshell( - [ - "sandbox", - "exec", - "--name", - sandboxName, - "--", - "sh", - "-c", - [ - "OUT=/tmp/nemoclaw-inference-route-probe.out", - "HTTP_CODE=$(curl -sk -o \"$OUT\" -w '%{http_code}' --connect-timeout 3 --max-time 8 https://inference.local/v1/models 2>/dev/null) || HTTP_CODE=000", - "case \"$HTTP_CODE\" in 000|5*) printf 'BROKEN %s ' \"$HTTP_CODE\"; head -c 160 \"$OUT\" 2>/dev/null || true ;; *) printf 'OK %s' \"$HTTP_CODE\" ;; esac", - ].join("; "), - ], - { ignoreError: true, timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS }, - ); - const detail = probe.output.trim(); + const script = [ + "OUT=/tmp/nemoclaw-inference-route-probe.out", + "HTTP_CODE=$(curl -sk -o \"$OUT\" -w '%{http_code}' --connect-timeout 3 --max-time 8 https://inference.local/v1/models 2>/dev/null) || HTTP_CODE=000", + "case \"$HTTP_CODE\" in 000|5*) printf 'BROKEN %s ' \"$HTTP_CODE\"; head -c 160 \"$OUT\" 2>/dev/null || true ;; *) printf 'OK %s' \"$HTTP_CODE\" ;; esac", + ].join("; "); + let probe: ReturnType; + try { + probe = execTextSync(sandboxName, ["sh", "-c", script], { + timeoutMs: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + probe = { status: 1, stdout: "", stderr: detail }; + } + const detail = (probe.stdout || probe.stderr || "").trim(); lastProbe = { healthy: probe.status === 0 && /^OK\s+[0-9]{3}\b/.test(detail), broken: /^BROKEN\s+[0-9]{3}\b/.test(detail), - detail: detail || `openshell sandbox exec exited with status ${String(probe.status)}`, + detail: detail || `OpenShell gRPC exec exited with status ${String(probe.status)}`, }; if (lastProbe.healthy || attempt === boundedAttempts) return lastProbe; sleepSync(delayMs); @@ -891,7 +886,7 @@ export async function connectSandbox( // ── Inference route swap (#1248, #3390) ─────────────────────────── // When the user has multiple sandboxes with different providers, the // cluster-wide inference.local route may still point at the other provider. - // After the sandbox is Ready, verify and recover the route before SSH. + // After the sandbox is Ready, verify and recover the route before opening a shell. sb = ensureSandboxInferenceRouteOrExit(sandboxName); maybeEnsureHermesToolGatewayBroker(sb); diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index 669a1cbc6a..aa9aaf6887 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -767,8 +767,8 @@ const CHANNEL_CLEAR_SENTINEL = "NEMOCLAW_CHANNEL_CLEAR_OK"; // the state_dirs backup does not restore an auth blob the operator just // asked NemoClaw to forget. Returns true when no cleanup was needed OR // when the in-sandbox rm produced our success sentinel; false otherwise. -// Tries `openshell sandbox exec` first and falls back to SSH for transient -// wrapper hiccups (mirrors the pattern in process-recovery.ts:286-296). +// Tries the fast gRPC exec helper first and falls back to the same gRPC shell +// helper for transient wrapper hiccups (mirrors process-recovery). // Fixes #3998. function clearSandboxChannelDurableState(sandboxName: string, channelName: string): boolean { const agent = resolveAgentForSandbox(sandboxName); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 6086b799d0..82e76bfc1b 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -3,23 +3,18 @@ import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; +import { createSandboxGrpcClient, execTextSync } from "../../adapters/openshell/grpc"; import { - captureOpenshell, - captureOpenshellForStatus, - captureSandboxSshConfig, - getOpenshellBinary, - isCommandTimeout, - runOpenshell, -} from "../../adapters/openshell/runtime"; + forwardStatesAsListOutput, + startForwardBridgeDetached, + stopForwardBridge, +} from "../../adapters/openshell/forward-bridge-state"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; import { G, R } from "../../cli/terminal-style"; import { DASHBOARD_PORT } from "../../core/ports"; import { sleepSeconds } from "../../core/wait"; -import { ROOT, shellQuote } from "../../runner"; +import { shellQuote } from "../../runner"; import * as registry from "../../state/registry"; import { parseForwardList } from "../../state/sandbox-session"; @@ -76,55 +71,20 @@ function getSandboxHealthProbeUrl(sandboxName: string): string { return `http://127.0.0.1:${resolveSandboxDashboardPort(sandboxName)}/health`; } -/** - * Run a command inside the sandbox via SSH and return { status, stdout, stderr }. - * Returns null if SSH config cannot be obtained. - */ +/** Run a shell command inside the sandbox over OpenShell gRPC. */ export function executeSandboxCommand( sandboxName: string, command: string, ): SandboxCommandResult | null { - const sshConfigResult = captureSandboxSshConfig(sandboxName, { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - if (sshConfigResult.status !== 0) return null; - if (!sshConfigResult.output.trim()) return null; - - const tmpFile = path.join(os.tmpdir(), `nemoclaw-ssh-${process.pid}-${Date.now()}.conf`); - fs.writeFileSync(tmpFile, sshConfigResult.output, { mode: 0o600 }); try { - const result = spawnSync( - "ssh", - [ - "-F", - tmpFile, - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - "ConnectTimeout=5", - "-o", - "LogLevel=ERROR", - `openshell-${sandboxName}`, - command, - ], - { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - ); + const result = execTextSync(sandboxName, ["sh", "-c", command], { timeoutMs: 15_000 }); return { - status: result.status ?? 1, - stdout: (result.stdout || "").trim(), - stderr: (result.stderr || "").trim(), + status: result.status, + stdout: result.stdout.trim(), + stderr: result.stderr.trim(), }; } catch { return null; - } finally { - try { - fs.unlinkSync(tmpFile); - } catch { - /* ignore */ - } } } @@ -138,27 +98,18 @@ export function executeSandboxExecCommand( const effectiveTimeout = Number.isFinite(timeoutOverride) && timeoutOverride > 0 ? timeoutOverride : timeout; try { - const result = spawnSync( - getOpenshellBinary(), - ["sandbox", "exec", "--name", sandboxName, "--", "sh", "-c", markedCommand], - { - cwd: ROOT, - encoding: "utf-8", - env: process.env, - stdio: ["ignore", "pipe", "pipe"], - timeout: effectiveTimeout, - }, - ); - if (result.error) return null; - const stdout = (result.stdout || "").trim(); + const result = execTextSync(sandboxName, ["sh", "-c", markedCommand], { + timeoutMs: effectiveTimeout, + }); + const stdout = result.stdout.trim(); const stdoutLines = stdout.split(/\r?\n/); const markerIndex = stdoutLines.indexOf(SANDBOX_EXEC_STARTED_MARKER); if (markerIndex === -1) return null; const commandStdoutLines = stdoutLines.slice(markerIndex + 1); return { - status: result.status ?? 1, + status: result.status, stdout: commandStdoutLines.join("\n").trim(), - stderr: (result.stderr || "").trim(), + stderr: result.stderr.trim(), }; } catch { return null; @@ -170,21 +121,26 @@ async function executeSandboxExecCommandForStatus( command: string, ): Promise { const markedCommand = `printf '%s\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; - const result = await captureOpenshellForStatus( - ["sandbox", "exec", "--name", sandboxName, "--", "sh", "-c", markedCommand], - { ignoreError: true }, - ); - if (isCommandTimeout(result) || result.error) return null; - const stdout = (result.output || "").trim(); - const stdoutLines = stdout.split(/\r?\n/); - const markerIndex = stdoutLines.indexOf(SANDBOX_EXEC_STARTED_MARKER); - if (markerIndex === -1) return null; - const commandStdoutLines = stdoutLines.slice(markerIndex + 1); - return { - status: result.status ?? 1, - stdout: commandStdoutLines.join("\n").trim(), - stderr: "", - }; + const client = createSandboxGrpcClient(); + try { + const result = await client.execText(sandboxName, ["sh", "-c", markedCommand], { + timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const stdout = result.stdout.trim(); + const stdoutLines = stdout.split(/\r?\n/); + const markerIndex = stdoutLines.indexOf(SANDBOX_EXEC_STARTED_MARKER); + if (markerIndex === -1) return null; + const commandStdoutLines = stdoutLines.slice(markerIndex + 1); + return { + status: result.status, + stdout: commandStdoutLines.join("\n").trim(), + stderr: result.stderr.trim(), + }; + } catch { + return null; + } finally { + client.close(); + } } function parseSandboxGatewayProbe(result: SandboxCommandResult | null): boolean | null { @@ -223,8 +179,8 @@ export async function isSandboxGatewayRunningForStatus( /** * Probe the full inference chain by curling `https://inference.local/v1/models` - * from inside the sandbox via `openshell sandbox exec`. This is the path agent - * traffic actually takes (openclaw gateway → auth proxy → backend). Any HTTP + * from inside the sandbox via OpenShell gRPC exec. This is the path agent + * traffic actually takes (openclaw gateway -> auth proxy -> backend). Any HTTP * response (including 401) means routing works; 000 / no response means DNS, * proxy, or gateway is broken. The optional 3rd line in #3265. * @@ -280,21 +236,18 @@ function recoverSandboxProcesses(sandboxName: string): boolean { result && (result.stdout.includes("GATEWAY_PID=") || result.stdout.includes("ALREADY_RUNNING")) ); - const recoveredSsh = (result: SandboxCommandResult | null) => + const recoveredCommand = (result: SandboxCommandResult | null) => !!(result && result.status === 0 && hasRecoveryMarker(result)); if (agentScript) { - // Non-OpenClaw manifests do not yet declare a runtime user for root - // sandbox exec. Recover them over SSH so the launch inherits the sandbox - // login user instead of creating root-owned agent state under /sandbox. - return recoveredSsh(executeSandboxCommand(sandboxName, agentScript)); + return recoveredCommand(executeSandboxCommand(sandboxName, agentScript)); } const script = agentRuntime.buildOpenClawRecoveryScript(dashboardPort); const execResult = executeSandboxExecCommand(sandboxName, script, 30000); if (hasRecoveryMarker(execResult)) return true; if (execResult !== null) return false; - return recoveredSsh(executeSandboxCommand(sandboxName, script)); + return false; } function readNonNegativeNumberEnv(name: string, fallback: number): number { @@ -338,42 +291,37 @@ function ensureSandboxPortForward(sandboxName: string): boolean { if (forwardHealth === true) return true; if (forwardHealth === "occupied") return false; - const port = String(resolveSandboxDashboardPort(sandboxName)); - runOpenshell(["forward", "stop", port], { ignoreError: true, stdio: "ignore" }); - const startResult = runOpenshell(["forward", "start", "--background", port, sandboxName], { - ignoreError: true, + const port = resolveSandboxDashboardPort(sandboxName); + stopForwardBridge(sandboxName, port); + const startResult = startForwardBridgeDetached(sandboxName, { + bind: "127.0.0.1", + port, + targetHost: "127.0.0.1", + targetPort: port, + timeoutMs: 30_000, }); - if (startResult.status !== 0) return false; + if (!startResult.ok) return false; return isSandboxForwardHealthy(sandboxName) === true; } /** - * Probe `openshell forward list` for the sandbox's dashboard forward. + * Probe NemoClaw's gRPC forward state for the sandbox's dashboard forward. * Returns true when an entry exists for the expected sandbox+port pair * with STATUS=running, false when the entry is missing or non-running, * "occupied" when another sandbox already owns the expected port, and - * null when openshell is unreachable. + * null when local state cannot prove ownership. * * The in-sandbox gateway and the host-side forward are independent - * dimensions: the forward can die (host SSH session dropped, list shows + * dimensions: the bridge can die (host process exited, state shows * STATUS=dead) while the gateway keeps listening on 127.0.0.1:. * * Also falls back to a local TCP/HTTP probe of 127.0.0.1: when - * `forward list` would classify the entry as not-running. openshell's - * STATUS column lags real state — it can show "dead" for an entry that - * is still serving traffic, or hide an entry whose SSH session was just - * recycled (#3334). Trusting the column verbatim made every `connect` - * print a "missing or dead" preamble followed by a "Failed to - * re-establish" line even though the forward worked. + * local state would classify the entry as not-running, because a listener + * can still be serving traffic while process-state cleanup catches up. */ function isSandboxForwardHealthy(sandboxName: string): SandboxForwardHealth { const port = resolveSandboxDashboardPort(sandboxName); - const result = captureOpenshell(["forward", "list"], { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - if (!result || isCommandTimeout(result) || result.status !== 0) return null; - const entries = parseForwardList(result.output) as SandboxForwardListEntry[]; + const entries = parseForwardList(forwardStatesAsListOutput()) as SandboxForwardListEntry[]; return classifyForwardHealthWithReachability(entries, sandboxName, String(port), () => isLocalForwardReachable(port), ); @@ -469,7 +417,7 @@ export function checkAndRecoverSandboxProcesses( } else { console.error(" Failed to re-establish the dashboard port forward."); console.error( - ` Run \`openshell forward start --background ${sandboxName}\` manually.`, + ` Retry: nemoclaw ${sandboxName} connect --probe-only`, ); } } @@ -520,7 +468,7 @@ export function checkAndRecoverSandboxProcesses( } else { console.error(" Failed to re-establish the dashboard port forward."); console.error( - ` Run \`openshell forward start --background ${sandboxName}\` manually.`, + ` Retry: nemoclaw ${sandboxName} connect --probe-only`, ); } } diff --git a/src/lib/actions/sandbox/skill-install.ts b/src/lib/actions/sandbox/skill-install.ts index 956fc9441e..ef54c4d326 100644 --- a/src/lib/actions/sandbox/skill-install.ts +++ b/src/lib/actions/sandbox/skill-install.ts @@ -3,10 +3,7 @@ import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; -import { captureSandboxSshConfig } from "../../adapters/openshell/runtime"; -import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; import { CLI_NAME } from "../../cli/branding"; import { D, G, R, YW } from "../../cli/terminal-style"; @@ -168,62 +165,38 @@ export async function installSandboxSkill( const agent = agentRuntime.getSessionAgent(sandboxName); const paths = skillInstall.resolveSkillPaths(agent, frontmatter.name); - // 4. Get SSH config - const sshConfigResult = captureSandboxSshConfig(sandboxName, { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - if (sshConfigResult.status !== 0) { - console.error(" Failed to obtain SSH configuration for the sandbox."); - process.exit(1); - } - - const tmpSshConfig = path.join( - os.tmpdir(), - `nemoclaw-ssh-skill-${process.pid}-${Date.now()}.conf`, - ); - fs.writeFileSync(tmpSshConfig, sshConfigResult.output, { mode: 0o600 }); + const ctx = { sandboxName }; - try { - const ctx = { configFile: tmpSshConfig, sandboxName }; - - // 5. Check if skill already exists (update vs fresh install) - const isUpdate = skillInstall.checkExisting(ctx, paths); - - // 6. Upload skill directory - const { uploaded, failed } = skillInstall.uploadDirectory(ctx, skillDir, paths.uploadDir); - if (failed.length > 0) { - console.error(` Failed to upload ${failed.length} file(s): ${failed.join(", ")}`); - process.exit(1); - } - console.log(` ${G}✓${R} Uploaded ${uploaded} file(s) to sandbox`); - - // 7. Post-install (OpenClaw mirror + refresh, or restart hint). - // OpenClaw caches skill content per session, so always refresh the - // session index after an install/update to avoid stale SKILL.md data. - const post = skillInstall.postInstall(ctx, paths, skillDir); - for (const msg of post.messages) { - if (msg.startsWith("Warning:")) { - console.error(` ${YW}${msg}${R}`); - } else { - console.log(` ${D}${msg}${R}`); - } - } + // 4. Check if skill already exists (update vs fresh install) + const isUpdate = skillInstall.checkExisting(ctx, paths); - // 8. Verify - const verified = skillInstall.verifyInstall(ctx, paths); - if (verified) { - const verb = isUpdate ? "updated" : "installed"; - console.log(` ${G}✓${R} Skill '${frontmatter.name}' ${verb}`); + // 5. Upload skill directory + const { uploaded, failed } = skillInstall.uploadDirectory(ctx, skillDir, paths.uploadDir); + if (failed.length > 0) { + console.error(` Failed to upload ${failed.length} file(s): ${failed.join(", ")}`); + process.exit(1); + } + console.log(` ${G}✓${R} Uploaded ${uploaded} file(s) to sandbox`); + + // 6. Post-install (OpenClaw mirror + refresh, or restart hint). + // OpenClaw caches skill content per session, so always refresh the + // session index after an install/update to avoid stale SKILL.md data. + const post = skillInstall.postInstall(ctx, paths, skillDir); + for (const msg of post.messages) { + if (msg.startsWith("Warning:")) { + console.error(` ${YW}${msg}${R}`); } else { - console.error(` Skill uploaded but verification failed at ${paths.uploadDir}/SKILL.md`); - process.exit(1); - } - } finally { - try { - fs.unlinkSync(tmpSshConfig); - } catch { - /* ignore */ + console.log(` ${D}${msg}${R}`); } } + + // 7. Verify + const verified = skillInstall.verifyInstall(ctx, paths); + if (verified) { + const verb = isUpdate ? "updated" : "installed"; + console.log(` ${G}✓${R} Skill '${frontmatter.name}' ${verb}`); + } else { + console.error(` Skill uploaded but verification failed at ${paths.uploadDir}/SKILL.md`); + process.exit(1); + } } diff --git a/src/lib/adapters/openshell/forward-bridge-runner.ts b/src/lib/adapters/openshell/forward-bridge-runner.ts new file mode 100644 index 0000000000..6b248d3cde --- /dev/null +++ b/src/lib/adapters/openshell/forward-bridge-runner.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createSandboxGrpcClient } from "./grpc"; +import { writeForwardState } from "./forward-bridge-state"; + +type RunnerArgs = { + sandboxName: string; + bind: string; + port: number; + targetHost: string; + targetPort: number; +}; + +async function main(): Promise { + const raw = process.argv[2]; + if (!raw) throw new Error("missing forward bridge args"); + const args = JSON.parse(raw) as RunnerArgs; + const client = createSandboxGrpcClient(); + const handle = await client.startForward(args.sandboxName, { + localHost: args.bind, + localPort: args.port, + targetHost: args.targetHost, + targetPort: args.targetPort, + serviceId: `nemoclaw-dashboard-${args.port}`, + }); + writeForwardState({ + sandboxName: args.sandboxName, + bind: args.bind, + port: args.port, + targetHost: args.targetHost, + targetPort: args.targetPort, + pid: process.pid, + startedAt: new Date().toISOString(), + }); + + const close = async () => { + try { + await handle.close(); + } catch { + /* ignore */ + } + client.close(); + process.exit(0); + }; + process.on("SIGTERM", () => void close()); + process.on("SIGINT", () => void close()); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts new file mode 100644 index 0000000000..06625595b8 --- /dev/null +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { sleepMs } from "../../core/wait"; + +export interface SandboxForwardState { + sandboxName: string; + bind: string; + port: number; + targetHost: string; + targetPort: number; + pid: number; + startedAt: string; +} + +export interface ForwardBridgeStartOptions { + bind?: string; + port: number; + targetHost?: string; + targetPort: number; + timeoutMs?: number; +} + +export interface ForwardBridgeStartResult { + ok: boolean; + state?: SandboxForwardState; + diagnostic: string; +} + +function stateDir(): string { + return path.join(process.env.HOME || os.homedir(), ".nemoclaw", "forwards"); +} + +function statePath(sandboxName: string, port: number | string): string { + const safeSandbox = sandboxName.replace(/[^A-Za-z0-9._-]/g, "_"); + return path.join(stateDir(), `${safeSandbox}-${String(port)}.json`); +} + +function ensureStateDir(): void { + fs.mkdirSync(stateDir(), { recursive: true, mode: 0o700 }); +} + +export function writeForwardState(state: SandboxForwardState): void { + ensureStateDir(); + fs.writeFileSync(statePath(state.sandboxName, state.port), JSON.stringify(state, null, 2), { + mode: 0o600, + }); +} + +export function removeForwardState(sandboxName: string, port: number | string): void { + try { + fs.unlinkSync(statePath(sandboxName, port)); + } catch { + /* ignore */ + } +} + +function readStateFile(filePath: string): SandboxForwardState | null { + try { + const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8")) as SandboxForwardState; + if ( + typeof parsed.sandboxName === "string" && + typeof parsed.bind === "string" && + Number.isInteger(parsed.port) && + typeof parsed.targetHost === "string" && + Number.isInteger(parsed.targetPort) && + Number.isInteger(parsed.pid) + ) { + return parsed; + } + } catch { + /* ignore */ + } + return null; +} + +export function listForwardStates(): SandboxForwardState[] { + try { + return fs + .readdirSync(stateDir(), { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => readStateFile(path.join(stateDir(), entry.name))) + .filter((entry): entry is SandboxForwardState => entry !== null) + .filter((entry) => isPidAlive(entry.pid)); + } catch { + return []; + } +} + +export function getForwardState(sandboxName: string, port: number | string): SandboxForwardState | null { + const state = readStateFile(statePath(sandboxName, port)); + return state && isPidAlive(state.pid) ? state : null; +} + +export function isPidAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +export function stopForwardBridge(sandboxName: string, port: number | string): boolean { + const state = readStateFile(statePath(sandboxName, port)); + if (state && isPidAlive(state.pid)) { + try { + process.kill(state.pid, "SIGTERM"); + } catch { + /* ignore */ + } + } + removeForwardState(sandboxName, port); + return Boolean(state); +} + +export function stopAllForwardBridges(): void { + for (const state of listForwardStates()) { + stopForwardBridge(state.sandboxName, state.port); + } +} + +export function forwardStatesAsListOutput(states = listForwardStates()): string { + const lines = ["SANDBOX BIND PORT PID STATUS"]; + for (const state of states) { + lines.push(`${state.sandboxName} ${state.bind} ${state.port} ${state.pid} running`); + } + return `${lines.join("\n")}\n`; +} + +function runnerCommand(): { command: string; args: string[] } { + const built = path.join(__dirname, "forward-bridge-runner.js"); + if (fs.existsSync(built)) return { command: process.execPath, args: [built] }; + + const source = path.join(__dirname, "forward-bridge-runner.ts"); + const root = path.resolve(__dirname, "..", "..", "..", ".."); + const tsxBin = path.join(root, "node_modules", ".bin", process.platform === "win32" ? "tsx.cmd" : "tsx"); + if (fs.existsSync(source) && fs.existsSync(tsxBin)) return { command: tsxBin, args: [source] }; + + throw new Error("OpenShell gRPC forward bridge runner is not available. Run `npm run build:cli` first."); +} + +export function startForwardBridgeDetached( + sandboxName: string, + options: ForwardBridgeStartOptions, +): ForwardBridgeStartResult { + const bind = options.bind || "127.0.0.1"; + const targetHost = options.targetHost || "127.0.0.1"; + const timeoutMs = options.timeoutMs ?? 30_000; + stopForwardBridge(sandboxName, options.port); + const { command, args } = runnerCommand(); + ensureStateDir(); + const diagnosticPath = path.join( + stateDir(), + `${sandboxName.replace(/[^A-Za-z0-9._-]/g, "_")}-${options.port}.log`, + ); + const out = fs.openSync(diagnosticPath, "w", 0o600); + const child = spawn( + command, + [ + ...args, + JSON.stringify({ + sandboxName, + bind, + port: options.port, + targetHost, + targetPort: options.targetPort, + }), + ], + { detached: true, stdio: ["ignore", out, out], env: process.env }, + ); + child.unref(); + fs.closeSync(out); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const state = getForwardState(sandboxName, options.port); + if (state?.pid === child.pid) { + return { ok: true, state: state ?? undefined, diagnostic: "" }; + } + sleepMs(250); + } + + let diagnostic = ""; + try { + diagnostic = fs.readFileSync(diagnosticPath, "utf-8").trim(); + } catch { + /* ignore */ + } + try { + if (child.pid) process.kill(child.pid, "SIGTERM"); + } catch { + /* ignore */ + } + return { + ok: false, + diagnostic: + diagnostic || `forward bridge did not become ready within ${String(timeoutMs)}ms`, + }; +} diff --git a/src/lib/adapters/openshell/gateway-metadata.test.ts b/src/lib/adapters/openshell/gateway-metadata.test.ts new file mode 100644 index 0000000000..10d88f60f1 --- /dev/null +++ b/src/lib/adapters/openshell/gateway-metadata.test.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { resolveGatewayMetadata } from "./gateway-metadata"; + +function writeGateway(home: string, name: string, metadata: Record): void { + const dir = path.join(home, ".config", "openshell", "gateways", name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "metadata.json"), JSON.stringify(metadata)); + fs.writeFileSync(path.join(home, ".config", "openshell", "active_gateway"), name); +} + +describe("resolveGatewayMetadata", () => { + it("resolves the active plaintext local gateway", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); + try { + writeGateway(home, "nemoclaw", { + name: "nemoclaw", + gateway_endpoint: "http://127.0.0.1:8080", + auth_mode: "plaintext", + }); + + const gateway = resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv }); + + expect(gateway.name).toBe("nemoclaw"); + expect(gateway.target).toBe("127.0.0.1:8080"); + expect(gateway.authMode).toBe("plaintext"); + expect(gateway.mtlsDir).toContain(path.join("gateways", "nemoclaw", "mtls")); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects bearer-auth remote gateways with an actionable error", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); + try { + writeGateway(home, "remote", { + name: "remote", + gateway_endpoint: "https://gateway.example.test", + auth_mode: "oidc", + is_remote: true, + }); + + expect(() => + resolveGatewayMetadata({ env: { HOME: home, OPENSHELL_GATEWAY: "remote" } as NodeJS.ProcessEnv }), + ).toThrow(/supports local plaintext and mTLS gateways only/); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("resolves mTLS gateways to the local certificate bundle", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); + try { + writeGateway(home, "secure", { + name: "secure", + gateway_endpoint: "https://127.0.0.1:17670", + auth_mode: "mtls", + }); + + const gateway = resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv }); + + expect(gateway.target).toBe("127.0.0.1:17670"); + expect(gateway.authMode).toBe("mtls"); + expect(gateway.mtlsDir).toBe( + path.join(home, ".config", "openshell", "gateways", "secure", "mtls"), + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects mTLS metadata with a plaintext endpoint", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); + try { + writeGateway(home, "bad", { + name: "bad", + gateway_endpoint: "http://127.0.0.1:8080", + auth_mode: "mtls", + }); + + expect(() => resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv })).toThrow( + /requires an https:\/\/ endpoint/, + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects plaintext metadata with a TLS endpoint", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); + try { + writeGateway(home, "bad", { + name: "bad", + gateway_endpoint: "https://127.0.0.1:17670", + auth_mode: "plaintext", + }); + + expect(() => resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv })).toThrow( + /requires an http:\/\/ endpoint/, + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/adapters/openshell/gateway-metadata.ts b/src/lib/adapters/openshell/gateway-metadata.ts new file mode 100644 index 0000000000..09838c1364 --- /dev/null +++ b/src/lib/adapters/openshell/gateway-metadata.ts @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +export interface GatewayMetadata { + name: string; + gateway_endpoint: string; + is_remote?: boolean; + gateway_port?: number; + remote_host?: string; + resolved_host?: string; + auth_mode?: string | null; + edge_team_domain?: string; + edge_auth_url?: string; + oidc_issuer?: string; + oidc_client_id?: string; + oidc_audience?: string; + oidc_scopes?: string; +} + +export interface ResolvedGatewayMetadata { + name: string; + endpoint: URL; + target: string; + authMode: "plaintext" | "mtls"; + metadataPath: string | null; + gatewayDir: string | null; + mtlsDir: string | null; + insecureTls: boolean; +} + +export interface GatewayMetadataOptions { + env?: NodeJS.ProcessEnv; + gatewayName?: string; + gatewayEndpoint?: string; + gatewayInsecure?: boolean; +} + +function configRoot(env: NodeJS.ProcessEnv): string { + return path.join(env.XDG_CONFIG_HOME || path.join(env.HOME || os.homedir(), ".config"), "openshell"); +} + +function activeGatewayPath(env: NodeJS.ProcessEnv): string { + return path.join(configRoot(env), "active_gateway"); +} + +function gatewaysDir(env: NodeJS.ProcessEnv): string { + return path.join(configRoot(env), "gateways"); +} + +function sanitizeGatewayName(name: string): string { + return name + .split("") + .map((ch) => (/[A-Za-z0-9._-]/.test(ch) ? ch : "_")) + .join(""); +} + +function readActiveGateway(env: NodeJS.ProcessEnv): string | null { + try { + const name = fs.readFileSync(activeGatewayPath(env), "utf-8").trim(); + return name || null; + } catch { + return null; + } +} + +function readGatewayMetadata(name: string, env: NodeJS.ProcessEnv): { + metadata: GatewayMetadata; + metadataPath: string; + gatewayDir: string; +} { + const safeName = sanitizeGatewayName(name); + const gatewayDir = path.join(gatewaysDir(env), safeName); + const metadataPath = path.join(gatewayDir, "metadata.json"); + const raw = fs.readFileSync(metadataPath, "utf-8"); + const parsed = JSON.parse(raw) as GatewayMetadata; + return { metadata: parsed, metadataPath, gatewayDir }; +} + +function parseEndpoint(endpoint: string): URL { + let url: URL; + try { + url = new URL(endpoint); + } catch (error) { + throw new Error( + `OpenShell gateway endpoint '${endpoint}' is not a valid URL: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error( + `OpenShell gateway endpoint '${endpoint}' must use http:// or https:// for gRPC.`, + ); + } + return url; +} + +function endpointTarget(url: URL): string { + if (url.port) return url.host; + return `${url.hostname}:${url.protocol === "https:" ? 443 : 80}`; +} + +function normalizeAuthMode(mode: string | null | undefined, url: URL): "plaintext" | "mtls" { + const normalized = (mode || "").trim().toLowerCase(); + if (!normalized) return url.protocol === "http:" ? "plaintext" : "mtls"; + if (normalized === "mtls") { + if (url.protocol !== "https:") { + throw new Error( + `OpenShell gateway auth mode 'mtls' requires an https:// endpoint, got '${url.toString()}'.`, + ); + } + return "mtls"; + } + if (normalized === "plaintext") { + if (url.protocol !== "http:") { + throw new Error( + `OpenShell gateway auth mode 'plaintext' requires an http:// endpoint, got '${url.toString()}'.`, + ); + } + return "plaintext"; + } + if (normalized === "cloudflare_jwt" || normalized === "oidc") { + throw new Error( + `OpenShell gateway auth mode '${normalized}' is bearer-token based. ` + + "NemoClaw's direct gRPC transport currently supports local plaintext and mTLS gateways only. " + + "Select a local NemoClaw gateway or configure mTLS for this gateway.", + ); + } + throw new Error( + `OpenShell gateway auth mode '${normalized}' is not supported by NemoClaw's direct gRPC transport. ` + + "Supported modes: plaintext, mtls.", + ); +} + +export function resolveGatewayMetadata( + options: GatewayMetadataOptions = {}, +): ResolvedGatewayMetadata { + const env = options.env ?? process.env; + const endpointOverride = + options.gatewayEndpoint || env.OPENSHELL_GATEWAY_ENDPOINT || env.OPENSHELL_GATEWAY_URL; + const gatewayName = options.gatewayName || env.OPENSHELL_GATEWAY || readActiveGateway(env); + const insecureTls = + options.gatewayInsecure === true || + env.OPENSHELL_GATEWAY_INSECURE === "1" || + env.OPENSHELL_GATEWAY_INSECURE === "true"; + + if (endpointOverride) { + const endpoint = parseEndpoint(endpointOverride); + return { + name: gatewayName || "endpoint", + endpoint, + target: endpointTarget(endpoint), + authMode: normalizeAuthMode(endpoint.protocol === "http:" ? "plaintext" : "mtls", endpoint), + metadataPath: null, + gatewayDir: gatewayName ? path.join(gatewaysDir(env), sanitizeGatewayName(gatewayName)) : null, + mtlsDir: gatewayName + ? path.join(gatewaysDir(env), sanitizeGatewayName(gatewayName), "mtls") + : null, + insecureTls, + }; + } + + if (!gatewayName) { + throw new Error( + "No active OpenShell gateway is configured. Run `openshell gateway select ` or set OPENSHELL_GATEWAY.", + ); + } + + let loaded: ReturnType; + try { + loaded = readGatewayMetadata(gatewayName, env); + } catch (error) { + throw new Error( + `Failed to load OpenShell gateway metadata for '${gatewayName}': ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + const endpoint = parseEndpoint(loaded.metadata.gateway_endpoint); + const authMode = normalizeAuthMode(loaded.metadata.auth_mode, endpoint); + return { + name: loaded.metadata.name || gatewayName, + endpoint, + target: endpointTarget(endpoint), + authMode, + metadataPath: loaded.metadataPath, + gatewayDir: loaded.gatewayDir, + mtlsDir: path.join(loaded.gatewayDir, "mtls"), + insecureTls, + }; +} diff --git a/src/lib/adapters/openshell/grpc-migration-guard.test.ts b/src/lib/adapters/openshell/grpc-migration-guard.test.ts new file mode 100644 index 0000000000..9a1abbeda4 --- /dev/null +++ b/src/lib/adapters/openshell/grpc-migration-guard.test.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(__dirname, "../../../.."); + +const PRODUCTION_TARGETS = [ + "src/lib/actions/sandbox", + "src/lib/state/sandbox.ts", + "src/lib/skill-install.ts", + "src/lib/onboard/dashboard.ts", + "src/lib/onboard/sandbox-verification-exec.ts", + "src/lib/sandbox/version.ts", + "src/lib/share-command.ts", + "src/lib/share-command-deps.ts", + "src/lib/status-command-deps.ts", + "src/lib/tunnel/services.ts", + "src/lib/verify-deployment.ts", +]; + +const BANNED = [ + /captureSandboxSshConfig/, + /sandbox ssh-config/, + /spawnSync\(\s*["']ssh["']/, + /\bsshfs\b/, + /sandbox upload/, + /sandbox download/, + /forward start --background/, + /openshell forward start/, +]; + +function filesUnder(target: string): string[] { + const absolute = path.join(ROOT, target); + if (fs.statSync(absolute).isFile()) return [absolute]; + const out: string[] = []; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + if (entry.isFile() && entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) { + out.push(full); + } + } + }; + walk(absolute); + return out; +} + +describe("OpenShell gRPC migration guard", () => { + it("keeps sandbox lifecycle production code off SSH and SSH-backed forwards", () => { + const violations: string[] = []; + for (const target of PRODUCTION_TARGETS) { + for (const file of filesUnder(target)) { + const rel = path.relative(ROOT, file); + const text = fs.readFileSync(file, "utf-8"); + for (const pattern of BANNED) { + if (pattern.test(text)) violations.push(`${rel}: ${pattern}`); + } + } + } + expect(violations).toEqual([]); + }); +}); diff --git a/src/lib/adapters/openshell/grpc.ts b/src/lib/adapters/openshell/grpc.ts new file mode 100644 index 0000000000..1df5869a56 --- /dev/null +++ b/src/lib/adapters/openshell/grpc.ts @@ -0,0 +1,667 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import * as grpc from "@grpc/grpc-js"; +import * as protoLoader from "@grpc/proto-loader"; + +import { + type GatewayMetadataOptions, + type ResolvedGatewayMetadata, + resolveGatewayMetadata, +} from "./gateway-metadata"; + +export interface SandboxExecOptions { + workdir?: string; + env?: Record; + timeoutMs?: number; + timeoutSeconds?: number; + stdin?: Buffer | string; + tty?: boolean; + cols?: number; + rows?: number; +} + +export interface SandboxExecResult { + status: number; + stdout: string; + stderr: string; +} + +export interface SandboxStreamResult { + status: number; + stdout: Buffer; + stderr: Buffer; +} + +export interface SandboxLogLine { + sandbox_id?: string; + timestamp_ms?: number | string; + level?: string; + target?: string; + message?: string; + source?: string; + fields?: Record; +} + +export interface SandboxLogsResult { + logs: SandboxLogLine[]; + bufferTotal: number; +} + +export type SandboxWatchEvent = + | { type: "sandbox"; sandbox: unknown } + | { type: "log"; log: SandboxLogLine } + | { type: "event"; event: unknown } + | { type: "warning"; message: string }; + +export interface SandboxForwardHandle { + sandboxName: string; + sandboxId: string; + localPort: number; + localHost: string; + targetHost: string; + targetPort: number; + close(): Promise; +} + +export interface SandboxGrpcClientOptions extends GatewayMetadataOptions { + gateway?: ResolvedGatewayMetadata; +} + +type GrpcClient = grpc.Client & Record any>; + +const DEFAULT_GRPC_TIMEOUT_MS = 30_000; +const STREAM_CHUNK_SIZE = 64 * 1024; + +let packageDefinition: grpc.GrpcObject | null = null; + +function protoRoot(): string { + return path.join(__dirname, "proto"); +} + +function loadOpenShellPackage(): grpc.GrpcObject { + if (packageDefinition) return packageDefinition; + const definition = protoLoader.loadSync(path.join(protoRoot(), "openshell.proto"), { + defaults: false, + enums: String, + includeDirs: [protoRoot()], + keepCase: true, + longs: String, + oneofs: true, + bytes: Buffer, + }); + packageDefinition = grpc.loadPackageDefinition(definition) as grpc.GrpcObject; + return packageDefinition; +} + +function getOpenShellConstructor(): typeof grpc.Client { + const loaded = loadOpenShellPackage() as any; + const ctor = loaded.openshell?.v1?.OpenShell; + if (!ctor) { + throw new Error("Failed to load openshell.v1.OpenShell from vendored proto files."); + } + return ctor; +} + +function readRequiredFile(filePath: string, label: string): Buffer { + try { + return fs.readFileSync(filePath); + } catch (error) { + throw new Error( + `Failed to read OpenShell ${label} at ${filePath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function createCredentials(gateway: ResolvedGatewayMetadata): grpc.ChannelCredentials { + if (gateway.authMode === "plaintext" || gateway.endpoint.protocol === "http:") { + return grpc.credentials.createInsecure(); + } + if (!gateway.mtlsDir) { + throw new Error( + "OpenShell mTLS gateway metadata did not include a certificate directory. " + + "Run `openshell gateway info` and verify the gateway is registered locally.", + ); + } + const ca = readRequiredFile(path.join(gateway.mtlsDir, "ca.crt"), "mTLS CA"); + const cert = readRequiredFile(path.join(gateway.mtlsDir, "tls.crt"), "mTLS client certificate"); + const key = readRequiredFile(path.join(gateway.mtlsDir, "tls.key"), "mTLS client key"); + return grpc.credentials.createSsl(ca, key, cert); +} + +function method any>(client: GrpcClient, name: string): T { + const lower = name.charAt(0).toLowerCase() + name.slice(1); + const fn = client[lower] || client[name]; + if (typeof fn !== "function") { + throw new Error(`OpenShell gRPC method ${name} is unavailable.`); + } + return fn.bind(client) as T; +} + +function callOptions(timeoutMs?: number): grpc.CallOptions { + const deadlineMs = timeoutMs ?? DEFAULT_GRPC_TIMEOUT_MS; + return deadlineMs > 0 ? { deadline: new Date(Date.now() + deadlineMs) } : {}; +} + +function grpcStatusName(code: number | undefined): string { + if (typeof code !== "number") return "UNKNOWN"; + return grpc.status[code] || String(code); +} + +export function formatGrpcError(error: unknown): string { + if (!error || typeof error !== "object") return String(error); + const maybe = error as Partial; + const status = grpcStatusName(maybe.code); + const details = maybe.details || maybe.message || String(error); + return `${status}: ${details}`; +} + +function makeTimeoutError(label: string, timeoutMs: number): Error { + const error = new Error(`${label} timed out after ${timeoutMs} ms`) as NodeJS.ErrnoException; + error.code = "ETIMEDOUT"; + return error; +} + +function bufferFromData(data: unknown): Buffer { + if (Buffer.isBuffer(data)) return data; + if (data instanceof Uint8Array) return Buffer.from(data); + if (typeof data === "string") return Buffer.from(data, "base64"); + return Buffer.alloc(0); +} + +function stdinBuffer(input: Buffer | string | undefined): Buffer | undefined { + if (input === undefined) return undefined; + return Buffer.isBuffer(input) ? input : Buffer.from(input); +} + +function sandboxIdFromResponse(response: any, sandboxName: string): string { + const sandbox = response?.sandbox; + const id = sandbox?.metadata?.id || sandbox?.id; + if (typeof id === "string" && id.trim()) return id; + throw new Error(`OpenShell gateway returned no sandbox id for '${sandboxName}'.`); +} + +function execRequest(sandboxId: string, argv: string[], opts: SandboxExecOptions = {}) { + return { + sandbox_id: sandboxId, + command: argv, + workdir: opts.workdir || "", + environment: opts.env || {}, + timeout_seconds: opts.timeoutSeconds ?? Math.max(0, Math.ceil((opts.timeoutMs ?? 0) / 1000)), + stdin: stdinBuffer(opts.stdin), + tty: opts.tty === true, + cols: opts.cols ?? 0, + rows: opts.rows ?? 0, + }; +} + +function collectExecStream( + stream: grpc.ClientReadableStream, + label: string, + timeoutMs?: number, +): Promise { + return new Promise((resolve, reject) => { + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let exitCode: number | null = null; + let settled = false; + let timer: NodeJS.Timeout | undefined; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + fn(); + }; + + stream.on("data", (event: any) => { + if (event?.stdout?.data !== undefined) stdout.push(bufferFromData(event.stdout.data)); + if (event?.stderr?.data !== undefined) stderr.push(bufferFromData(event.stderr.data)); + if (event?.exit?.exit_code !== undefined) { + const parsed = Number(event.exit.exit_code); + exitCode = Number.isFinite(parsed) ? parsed : 1; + } + }); + stream.on("error", (error) => { + settle(() => reject(new Error(`${label} failed: ${formatGrpcError(error)}`))); + }); + stream.on("end", () => { + settle(() => + resolve({ + status: exitCode ?? 1, + stdout: Buffer.concat(stdout), + stderr: Buffer.concat(stderr), + }), + ); + }); + + if (timeoutMs && timeoutMs > 0) { + timer = setTimeout(() => { + try { + stream.cancel(); + } catch { + /* ignore */ + } + settle(() => reject(makeTimeoutError(label, timeoutMs))); + }, timeoutMs); + } + }); +} + +function collectDuplexExecStream( + stream: grpc.ClientDuplexStream, + label: string, + timeoutMs?: number, +): Promise { + return collectExecStream(stream as unknown as grpc.ClientReadableStream, label, timeoutMs); +} + +function writeInputChunks(stream: grpc.ClientDuplexStream, input: Buffer): void { + for (let offset = 0; offset < input.length; offset += STREAM_CHUNK_SIZE) { + stream.write({ stdin: input.subarray(offset, offset + STREAM_CHUNK_SIZE) }); + } +} + +export class SandboxGrpcClient { + readonly gateway: ResolvedGatewayMetadata; + private readonly client: GrpcClient; + + constructor(options: SandboxGrpcClientOptions = {}) { + this.gateway = options.gateway ?? resolveGatewayMetadata(options); + const OpenShell = getOpenShellConstructor(); + this.client = new OpenShell(this.gateway.target, createCredentials(this.gateway), { + "grpc.max_receive_message_length": 256 * 1024 * 1024, + "grpc.max_send_message_length": 256 * 1024 * 1024, + }) as GrpcClient; + } + + close(): void { + this.client.close(); + } + + async getSandboxId(sandboxName: string, timeoutMs?: number): Promise { + const getSandbox = method< + (request: { name: string }, options: grpc.CallOptions, callback: grpc.requestCallback) => void + >(this.client, "GetSandbox"); + const response = await new Promise((resolve, reject) => { + getSandbox({ name: sandboxName }, callOptions(timeoutMs), (error, value) => { + if (error) { + reject(new Error(`GetSandbox '${sandboxName}' failed: ${formatGrpcError(error)}`)); + } else { + resolve(value); + } + }); + }); + return sandboxIdFromResponse(response, sandboxName); + } + + async execBinaryStream( + sandboxName: string, + argv: string[], + opts: SandboxExecOptions = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const execSandbox = method< + (request: unknown, options?: grpc.CallOptions) => grpc.ClientReadableStream + >(this.client, "ExecSandbox"); + const request = execRequest(sandboxId, argv, opts); + return collectExecStream( + execSandbox(request, callOptions(opts.timeoutMs)), + `ExecSandbox '${sandboxName}'`, + opts.timeoutMs, + ); + } + + async execText( + sandboxName: string, + argv: string[], + opts: SandboxExecOptions = {}, + ): Promise { + const result = await this.execBinaryStream(sandboxName, argv, opts); + return { + status: result.status, + stdout: result.stdout.toString("utf-8"), + stderr: result.stderr.toString("utf-8"), + }; + } + + async execInputStream( + sandboxName: string, + argv: string[], + input: Buffer | string, + opts: SandboxExecOptions = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const execInteractive = method<() => grpc.ClientDuplexStream>( + this.client, + "ExecSandboxInteractive", + ); + const stream = execInteractive(); + const result = collectDuplexExecStream( + stream, + `ExecSandboxInteractive '${sandboxName}'`, + opts.timeoutMs, + ); + stream.write({ start: execRequest(sandboxId, argv, opts) }); + writeInputChunks(stream, stdinBuffer(input) ?? Buffer.alloc(0)); + stream.end(); + return result; + } + + async getSandboxLogs( + sandboxName: string, + opts: { + lines?: number; + sinceMs?: number; + sources?: string[]; + minLevel?: string; + timeoutMs?: number; + } = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const getLogs = method< + (request: unknown, options: grpc.CallOptions, callback: grpc.requestCallback) => void + >(this.client, "GetSandboxLogs"); + const response = await new Promise((resolve, reject) => { + getLogs( + { + sandbox_id: sandboxId, + lines: opts.lines ?? 0, + since_ms: opts.sinceMs ?? 0, + sources: opts.sources ?? [], + min_level: opts.minLevel ?? "", + }, + callOptions(opts.timeoutMs), + (error, value) => { + if (error) { + reject(new Error(`GetSandboxLogs '${sandboxName}' failed: ${formatGrpcError(error)}`)); + } else { + resolve(value); + } + }, + ); + }); + return { + logs: Array.isArray(response?.logs) ? response.logs : [], + bufferTotal: Number(response?.buffer_total ?? 0), + }; + } + + async watchSandbox( + sandboxName: string, + opts: { + followStatus?: boolean; + followLogs?: boolean; + followEvents?: boolean; + logTailLines?: number; + eventTail?: number; + stopOnTerminal?: boolean; + logSinceMs?: number; + logSources?: string[]; + logMinLevel?: string; + timeoutMs?: number; + onEvent?: (event: SandboxWatchEvent) => void; + } = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const watchSandbox = method< + (request: unknown, options?: grpc.CallOptions) => grpc.ClientReadableStream + >(this.client, "WatchSandbox"); + const stream = watchSandbox( + { + id: sandboxId, + follow_status: opts.followStatus === true, + follow_logs: opts.followLogs === true, + follow_events: opts.followEvents === true, + log_tail_lines: opts.logTailLines ?? 0, + event_tail: opts.eventTail ?? 0, + stop_on_terminal: opts.stopOnTerminal === true, + log_since_ms: opts.logSinceMs ?? 0, + log_sources: opts.logSources ?? [], + log_min_level: opts.logMinLevel ?? "", + }, + callOptions(opts.timeoutMs), + ); + await new Promise((resolve, reject) => { + stream.on("data", (event: any) => { + if (!opts.onEvent) return; + if (event?.sandbox !== undefined) opts.onEvent({ type: "sandbox", sandbox: event.sandbox }); + if (event?.log !== undefined) opts.onEvent({ type: "log", log: event.log }); + if (event?.event !== undefined) opts.onEvent({ type: "event", event: event.event }); + if (event?.warning !== undefined) { + opts.onEvent({ + type: "warning", + message: String(event.warning.message || ""), + }); + } + }); + stream.on("error", (error) => { + reject(new Error(`WatchSandbox '${sandboxName}' failed: ${formatGrpcError(error)}`)); + }); + stream.on("end", () => resolve()); + }); + } + + async createSshSession( + sandboxName: string, + timeoutMs = DEFAULT_GRPC_TIMEOUT_MS, + ): Promise<{ sandboxId: string; token: string }> { + const sandboxId = await this.getSandboxId(sandboxName, timeoutMs); + const createSession = method< + ( + request: { sandbox_id: string }, + options: grpc.CallOptions, + callback: grpc.requestCallback, + ) => void + >(this.client, "CreateSshSession"); + const response = await new Promise((resolve, reject) => { + createSession({ sandbox_id: sandboxId }, callOptions(timeoutMs), (error, value) => { + if (error) { + reject(new Error(`CreateSshSession '${sandboxName}' failed: ${formatGrpcError(error)}`)); + } else { + resolve(value); + } + }); + }); + const token = response?.token; + if (typeof token !== "string" || token.length === 0) { + throw new Error(`OpenShell gateway returned no relay token for '${sandboxName}'.`); + } + return { sandboxId, token }; + } + + async forwardTcpConnection( + socket: net.Socket, + params: { + sandboxName: string; + targetHost: string; + targetPort: number; + serviceId?: string; + timeoutMs?: number; + }, + ): Promise { + const { sandboxId, token } = await this.createSshSession(params.sandboxName, params.timeoutMs); + const forwardTcp = method<() => grpc.ClientDuplexStream>(this.client, "ForwardTcp"); + const stream = forwardTcp(); + stream.write({ + init: { + sandbox_id: sandboxId, + service_id: params.serviceId || "", + tcp: { host: params.targetHost, port: params.targetPort }, + authorization_token: token, + }, + }); + socket.on("data", (chunk) => stream.write({ data: chunk })); + socket.on("end", () => stream.end()); + socket.on("error", () => { + try { + stream.cancel(); + } catch { + /* ignore */ + } + }); + stream.on("data", (frame: any) => { + const data = frame?.data; + if (data !== undefined) socket.write(bufferFromData(data)); + }); + stream.on("end", () => socket.end()); + stream.on("error", () => socket.destroy()); + } + + async startForward( + sandboxName: string, + options: { + localPort: number; + localHost?: string; + targetHost?: string; + targetPort: number; + serviceId?: string; + }, + ): Promise { + const localHost = options.localHost || "127.0.0.1"; + const targetHost = options.targetHost || "127.0.0.1"; + const sandboxId = await this.getSandboxId(sandboxName); + const server = net.createServer((socket) => { + this.forwardTcpConnection(socket, { + sandboxName, + targetHost, + targetPort: options.targetPort, + serviceId: options.serviceId, + }).catch(() => socket.destroy()); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(options.localPort, localHost, () => { + server.off("error", reject); + resolve(); + }); + }); + return { + sandboxName, + sandboxId, + localPort: options.localPort, + localHost, + targetHost, + targetPort: options.targetPort, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + }; + } +} + +export function createSandboxGrpcClient(options: SandboxGrpcClientOptions = {}): SandboxGrpcClient { + return new SandboxGrpcClient(options); +} + +interface SyncRunnerResponse { + ok: boolean; + result?: { + status: number; + stdout?: string; + stderr?: string; + stdoutBase64?: string; + stderrBase64?: string; + }; + error?: string; +} + +function syncRunnerCommand(): { command: string; args: string[] } { + const built = path.join(__dirname, "sync-runner.js"); + if (fs.existsSync(built)) return { command: process.execPath, args: [built] }; + + const source = path.join(__dirname, "sync-runner.ts"); + const root = path.resolve(__dirname, "..", "..", "..", ".."); + const tsxBin = path.join(root, "node_modules", ".bin", process.platform === "win32" ? "tsx.cmd" : "tsx"); + if (fs.existsSync(source) && fs.existsSync(tsxBin)) return { command: tsxBin, args: [source] }; + + throw new Error("OpenShell gRPC sync runner is not available. Run `npm run build:cli` first."); +} + +function runSyncRunner( + payload: Record, + timeoutMs?: number, +): SyncRunnerResponse { + const { command, args } = syncRunnerCommand(); + const result = spawnSync(command, args, { + input: JSON.stringify(payload), + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + timeout: timeoutMs && timeoutMs > 0 ? timeoutMs + 5_000 : undefined, + env: process.env, + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error((result.stderr || result.stdout || `sync runner exited ${result.status}`).trim()); + } + try { + return JSON.parse(result.stdout) as SyncRunnerResponse; + } catch (error) { + throw new Error( + `OpenShell gRPC sync runner returned invalid JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +export function execTextSync( + sandboxName: string, + argv: string[], + opts: SandboxExecOptions = {}, +): SandboxExecResult { + const response = runSyncRunner({ op: "execText", sandboxName, argv, opts }, opts.timeoutMs); + if (!response.ok || !response.result) throw new Error(response.error || "OpenShell gRPC exec failed"); + return { + status: response.result.status, + stdout: response.result.stdout || "", + stderr: response.result.stderr || "", + }; +} + +export function execBinaryStreamSync( + sandboxName: string, + argv: string[], + opts: SandboxExecOptions = {}, +): SandboxStreamResult { + const response = runSyncRunner({ op: "execBinary", sandboxName, argv, opts }, opts.timeoutMs); + if (!response.ok || !response.result) throw new Error(response.error || "OpenShell gRPC exec failed"); + return { + status: response.result.status, + stdout: Buffer.from(response.result.stdoutBase64 || "", "base64"), + stderr: Buffer.from(response.result.stderrBase64 || "", "base64"), + }; +} + +export function execInputStreamSync( + sandboxName: string, + argv: string[], + input: Buffer | string, + opts: SandboxExecOptions = {}, +): SandboxStreamResult { + const inputBase64 = (Buffer.isBuffer(input) ? input : Buffer.from(input)).toString("base64"); + const response = runSyncRunner( + { op: "execInput", sandboxName, argv, inputBase64, opts }, + opts.timeoutMs, + ); + if (!response.ok || !response.result) throw new Error(response.error || "OpenShell gRPC exec failed"); + return { + status: response.result.status, + stdout: Buffer.from(response.result.stdoutBase64 || "", "base64"), + stderr: Buffer.from(response.result.stderrBase64 || "", "base64"), + }; +} + +// Keep CommonJS transpilation happy if this file is ever executed through ESM-aware tooling. +void fileURLToPath; diff --git a/src/lib/adapters/openshell/proto/datamodel.proto b/src/lib/adapters/openshell/proto/datamodel.proto new file mode 100644 index 0000000000..ee4434d4fc --- /dev/null +++ b/src/lib/adapters/openshell/proto/datamodel.proto @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.datamodel.v1; + +// Minimal public metadata subset used by NemoClaw's OpenShell gRPC adapter. +message ObjectMeta { + string id = 1; + string name = 2; + int64 created_at_ms = 3; + map labels = 4; + uint64 resource_version = 5; +} diff --git a/src/lib/adapters/openshell/proto/openshell.proto b/src/lib/adapters/openshell/proto/openshell.proto new file mode 100644 index 0000000000..7519ef7ab8 --- /dev/null +++ b/src/lib/adapters/openshell/proto/openshell.proto @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.v1; + +import "datamodel.proto"; + +service OpenShell { + rpc Health(HealthRequest) returns (HealthResponse); + rpc GetSandbox(GetSandboxRequest) returns (SandboxResponse); + rpc ExecSandbox(ExecSandboxRequest) returns (stream ExecSandboxEvent); + rpc ExecSandboxInteractive(stream ExecSandboxInput) returns (stream ExecSandboxEvent); + rpc GetSandboxLogs(GetSandboxLogsRequest) returns (GetSandboxLogsResponse); + rpc WatchSandbox(WatchSandboxRequest) returns (stream SandboxStreamEvent); + rpc CreateSshSession(CreateSshSessionRequest) returns (CreateSshSessionResponse); + rpc ForwardTcp(stream TcpForwardFrame) returns (stream TcpForwardFrame); +} + +message HealthRequest {} + +message HealthResponse { + ServiceStatus status = 1; + string version = 2; +} + +enum ServiceStatus { + SERVICE_STATUS_UNSPECIFIED = 0; + SERVICE_STATUS_HEALTHY = 1; + SERVICE_STATUS_DEGRADED = 2; + SERVICE_STATUS_UNHEALTHY = 3; +} + +message GetSandboxRequest { + string name = 1; +} + +message SandboxResponse { + Sandbox sandbox = 1; +} + +message Sandbox { + openshell.datamodel.v1.ObjectMeta metadata = 1; + SandboxStatus status = 3; + SandboxPhase phase = 4; + uint32 current_policy_version = 5; +} + +message SandboxStatus { + string sandbox_name = 1; + string agent_pod = 2; + string agent_fd = 3; + string sandbox_fd = 4; + repeated SandboxCondition conditions = 5; +} + +message SandboxCondition { + string type = 1; + string status = 2; + string reason = 3; + string message = 4; + string last_transition_time = 5; +} + +enum SandboxPhase { + SANDBOX_PHASE_UNSPECIFIED = 0; + SANDBOX_PHASE_PROVISIONING = 1; + SANDBOX_PHASE_READY = 2; + SANDBOX_PHASE_ERROR = 3; + SANDBOX_PHASE_DELETING = 4; + SANDBOX_PHASE_UNKNOWN = 5; +} + +message ExecSandboxRequest { + string sandbox_id = 1; + repeated string command = 2; + string workdir = 3; + map environment = 4; + uint32 timeout_seconds = 5; + bytes stdin = 6; + bool tty = 7; + uint32 cols = 8; + uint32 rows = 9; +} + +message ExecSandboxStdout { + bytes data = 1; +} + +message ExecSandboxStderr { + bytes data = 1; +} + +message ExecSandboxExit { + int32 exit_code = 1; +} + +message ExecSandboxEvent { + oneof payload { + ExecSandboxStdout stdout = 1; + ExecSandboxStderr stderr = 2; + ExecSandboxExit exit = 3; + } +} + +message ExecSandboxInput { + oneof payload { + ExecSandboxRequest start = 1; + bytes stdin = 2; + ExecSandboxWindowResize resize = 3; + } +} + +message ExecSandboxWindowResize { + uint32 cols = 1; + uint32 rows = 2; +} + +message GetSandboxLogsRequest { + string sandbox_id = 1; + uint32 lines = 2; + int64 since_ms = 3; + repeated string sources = 4; + string min_level = 5; +} + +message GetSandboxLogsResponse { + repeated SandboxLogLine logs = 1; + uint32 buffer_total = 2; +} + +message WatchSandboxRequest { + string id = 1; + bool follow_status = 2; + bool follow_logs = 3; + bool follow_events = 4; + uint32 log_tail_lines = 5; + uint32 event_tail = 6; + bool stop_on_terminal = 7; + int64 log_since_ms = 8; + repeated string log_sources = 9; + string log_min_level = 10; +} + +message SandboxStreamEvent { + oneof payload { + Sandbox sandbox = 1; + SandboxLogLine log = 2; + PlatformEvent event = 3; + SandboxStreamWarning warning = 4; + } +} + +message SandboxLogLine { + string sandbox_id = 1; + int64 timestamp_ms = 2; + string level = 3; + string target = 4; + string message = 5; + string source = 6; + map fields = 7; +} + +message PlatformEvent { + int64 timestamp_ms = 1; + string source = 2; + string type = 3; + string reason = 4; + string message = 5; + map metadata = 6; +} + +message SandboxStreamWarning { + string message = 1; +} + +message CreateSshSessionRequest { + string sandbox_id = 1; +} + +message CreateSshSessionResponse { + string sandbox_id = 1; + string token = 2; + string gateway_host = 3; + uint32 gateway_port = 4; + string gateway_scheme = 5; + string host_key_fingerprint = 7; + int64 expires_at_ms = 8; +} + +message TcpForwardInit { + string sandbox_id = 1; + string service_id = 4; + oneof target { + SshRelayTarget ssh = 5; + TcpRelayTarget tcp = 6; + } + string authorization_token = 7; +} + +message SshRelayTarget {} + +message TcpRelayTarget { + string host = 1; + uint32 port = 2; +} + +message TcpForwardFrame { + oneof payload { + TcpForwardInit init = 1; + bytes data = 2; + } +} diff --git a/src/lib/adapters/openshell/sync-runner.ts b/src/lib/adapters/openshell/sync-runner.ts new file mode 100644 index 0000000000..abbc42c4fd --- /dev/null +++ b/src/lib/adapters/openshell/sync-runner.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { spawnSync } from "node:child_process"; + +import { createSandboxGrpcClient, type SandboxExecOptions } from "./grpc"; + +type SyncRequest = { + op: "execText" | "execBinary" | "execInput"; + sandboxName: string; + argv: string[]; + inputBase64?: string; + opts?: SandboxExecOptions; +}; + +function readRequest(): SyncRequest { + const raw = fs.readFileSync(0, "utf-8"); + return JSON.parse(raw) as SyncRequest; +} + +function shellQuoteArg(value: string): string { + if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(value)) return value; + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +function legacyFakeRemoteCommand(argv: readonly string[]): string { + if ((argv[0] === "sh" || argv[0] === "bash") && (argv[1] === "-c" || argv[1] === "-lc")) { + return argv[2] ?? ""; + } + return argv.map(shellQuoteArg).join(" "); +} + +function runLegacyFakeSshTransport( + request: SyncRequest, + input: Buffer = Buffer.alloc(0), +): { status: number; stdout: Buffer; stderr: Buffer } { + if ( + process.env.NEMOCLAW_GRPC_TEST_TRANSPORT !== "1" || + process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH !== "1" + ) { + throw new Error("legacy fake transport is not enabled"); + } + + const sshBin = process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN || "ssh"; + const remoteCommand = legacyFakeRemoteCommand(request.argv); + const result = spawnSync(sshBin, [`openshell-${request.sandboxName}`, remoteCommand], { + input, + timeout: + request.opts?.timeoutMs && request.opts.timeoutMs > 0 + ? request.opts.timeoutMs + 5_000 + : undefined, + stdio: ["pipe", "pipe", "pipe"], + maxBuffer: 256 * 1024 * 1024, + env: process.env, + }); + if (result.error) throw result.error; + return { + status: result.status ?? 1, + stdout: Buffer.isBuffer(result.stdout) ? result.stdout : Buffer.from(result.stdout || ""), + stderr: Buffer.isBuffer(result.stderr) ? result.stderr : Buffer.from(result.stderr || ""), + }; +} + +async function main(): Promise { + const request = readRequest(); + if (process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1") { + const input = + request.op === "execInput" + ? Buffer.from(request.inputBase64 || "", "base64") + : Buffer.alloc(0); + const result = runLegacyFakeSshTransport(request, input); + if (request.op === "execText") { + process.stdout.write( + JSON.stringify({ + ok: true, + result: { + status: result.status, + stdout: result.stdout.toString("utf-8"), + stderr: result.stderr.toString("utf-8"), + }, + }), + ); + return; + } + process.stdout.write( + JSON.stringify({ + ok: true, + result: { + status: result.status, + stdoutBase64: result.stdout.toString("base64"), + stderrBase64: result.stderr.toString("base64"), + }, + }), + ); + return; + } + + const client = createSandboxGrpcClient(); + try { + if (request.op === "execText") { + const result = await client.execText(request.sandboxName, request.argv, request.opts ?? {}); + process.stdout.write(JSON.stringify({ ok: true, result })); + return; + } + if (request.op === "execBinary") { + const result = await client.execBinaryStream(request.sandboxName, request.argv, request.opts ?? {}); + process.stdout.write( + JSON.stringify({ + ok: true, + result: { + status: result.status, + stdoutBase64: result.stdout.toString("base64"), + stderrBase64: result.stderr.toString("base64"), + }, + }), + ); + return; + } + const input = Buffer.from(request.inputBase64 || "", "base64"); + const result = await client.execInputStream(request.sandboxName, request.argv, input, request.opts ?? {}); + process.stdout.write( + JSON.stringify({ + ok: true, + result: { + status: result.status, + stdoutBase64: result.stdout.toString("base64"), + stderrBase64: result.stderr.toString("base64"), + }, + }), + ); + } finally { + client.close(); + } +} + +main().catch((error) => { + process.stdout.write( + JSON.stringify({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + process.exit(0); +}); diff --git a/src/lib/adapters/openshell/timeouts.ts b/src/lib/adapters/openshell/timeouts.ts index 84ee8fc02e..bfff7b5238 100644 --- a/src/lib/adapters/openshell/timeouts.ts +++ b/src/lib/adapters/openshell/timeouts.ts @@ -9,13 +9,13 @@ * call sites and makes it easy to tune timeouts from a single location. * * Categories: - * PROBE — read-only queries that should return instantly (list, status, info, ssh-config) + * PROBE — read-only queries that should return instantly (list, status, info) * OPERATION — mutating commands (provider CRUD, forward start/stop, gateway select) * HEAVY — destructive or long-running (sandbox delete, gateway destroy, build) - * DOWNLOAD — file transfers over the sandbox SSH tunnel (config download) + * DOWNLOAD — bounded artifact transfers (config/log/state fetches) */ -/** Quick probe — sandbox list, status, gateway info, forward list, ssh-config */ +/** Quick probe — sandbox list, status, gateway info */ export const OPENSHELL_PROBE_TIMEOUT_MS = 15_000; /** In-sandbox inference.local route probe used during connect recovery */ @@ -27,5 +27,5 @@ export const OPENSHELL_OPERATION_TIMEOUT_MS = 30_000; /** Heavy operations — sandbox delete, gateway destroy, full build */ export const OPENSHELL_HEAVY_TIMEOUT_MS = 60_000; -/** Sandbox download (config file fetch over SSH) */ +/** Bounded artifact transfer */ export const OPENSHELL_DOWNLOAD_TIMEOUT_MS = 30_000; diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 63ab72c9e9..751d32c57f 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -1,10 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - import type { AgentDefinition } from "../agent/defs"; import { DASHBOARD_PORT } from "../core/ports"; import { buildChain, buildControlUiUrls } from "../dashboard/contract"; @@ -17,13 +13,13 @@ import { isLiveForwardStatus, } from "./dashboard-port"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { execBinaryStreamSync } from "../adapters/openshell/grpc"; import { - buildDetachedForwardStartSpawn, - buildForwardStartProgressLogger, - looksLikeForwardPortConflict, - runDetachedForwardStartWithPortReleaseRetries, -} from "./forward-start"; -import { bestEffortForwardStop, bestEffortForwardStopForSandbox } from "./forward-cleanup"; + forwardStatesAsListOutput, + startForwardBridgeDetached, + stopAllForwardBridges, + stopForwardBridge, +} from "../adapters/openshell/forward-bridge-state"; const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; export const CONTROL_UI_PORT = DASHBOARD_PORT; @@ -129,19 +125,20 @@ function getRunningForwardPorts(forwardListOutput: string | null | undefined): s return [...ports]; } -function findOpenclawJsonPath(dir: string): string | null { - if (!fs.existsSync(dir)) return null; - const entries = fs.readdirSync(dir, { withFileTypes: true }); - for (const entry of entries) { - const entryPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - const found: string | null = findOpenclawJsonPath(entryPath); - if (found) return found; - } else if (entry.name === "openclaw.json") { - return entryPath; - } +function parseForwardTarget(target: string): { bind: string; port: number } { + if (/^\d+$/.test(target.trim())) { + return { bind: "127.0.0.1", port: Number(target.trim()) }; } - return null; + const normalized = target.includes("://") ? target : `tcp://${target}`; + const url = new URL(normalized); + const port = Number(url.port || url.pathname.replace(/^\//, "")); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error(`Invalid dashboard forward target '${target}'.`); + } + return { + bind: url.hostname || "127.0.0.1", + port, + }; } function dashboardUrlForDisplay(url: string, deps: OnboardDashboardDeps): string { @@ -178,10 +175,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } function stopAllDashboardForwards(): void { - const forwardList = deps.runCaptureOpenshell(["forward", "list"], { ignoreError: true }); - for (const port of getRunningForwardPorts(forwardList)) { - bestEffortForwardStop(deps.runOpenshell, port); - } + stopAllForwardBridges(); } function buildOrphanedSandboxRollbackMessage( @@ -222,21 +216,15 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ): number { const { rollbackSandboxOnFailure = false } = options; const preferredPort = Number(getDashboardForwardPort(chatUiUrl)); - const stopForwardForSandbox = (port: string | number) => - bestEffortForwardStopForSandbox( - deps.runOpenshell, - (args, opts) => (deps.runCaptureOpenshell(args, opts) ?? "") as string, - port, - sandboxName, - ); - let existingForwards = deps.runCaptureOpenshell(["forward", "list"], { ignoreError: true }); + const stopForwardForSandbox = (port: string | number) => stopForwardBridge(sandboxName, port); + let existingForwards = forwardStatesAsListOutput(); const preferredEntry = findForwardEntry(existingForwards, String(preferredPort)); if ( preferredEntry && (preferredEntry.sandboxName === sandboxName || !isLiveForwardStatus(preferredEntry.status)) ) { stopForwardForSandbox(preferredPort); - existingForwards = deps.runCaptureOpenshell(["forward", "list"], { ignoreError: true }); + existingForwards = forwardStatesAsListOutput(); } let actualPort: number; try { @@ -270,28 +258,23 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa parsedUrl.port = String(actualPort); const actualTarget = getDashboardForwardTarget(parsedUrl.toString()); stopForwardForSandbox(actualPort); - const { ok: fwdOk, diagnostic: fwdDiagnostic } = runDetachedForwardStartWithPortReleaseRetries( - buildDetachedForwardStartSpawn( - deps.openshellArgv(["forward", "start", "--background", actualTarget, sandboxName]), - ), - () => - (deps.runCaptureOpenshell(["forward", "list"], { timeout: OPENSHELL_PROBE_TIMEOUT_MS }) ?? "") as string, - { port: actualPort, sandboxName }, - () => { - deps.sleep(1); - stopForwardForSandbox(actualPort); - }, - { onProgress: buildForwardStartProgressLogger(actualPort) }, - ); + const target = parseForwardTarget(actualTarget); + const { ok: fwdOk, diagnostic: fwdDiagnostic } = startForwardBridgeDetached(sandboxName, { + bind: target.bind, + port: actualPort, + targetHost: "127.0.0.1", + targetPort: target.port, + timeoutMs: 30_000, + }); if (!fwdOk) { - const looksLikePortConflict = looksLikeForwardPortConflict(fwdDiagnostic); + const looksLikePortConflict = /EADDRINUSE|address already in use|in use/i.test(fwdDiagnostic); if (rollbackSandboxOnFailure) { const err = new Error( looksLikePortConflict ? `Failed to start dashboard forward on port ${actualPort} — the host port ` + `is held by another process. Free it and run \`${deps.cliName()} onboard\` again, ` + `or pass \`--control-ui-port \` to pick a different dashboard port.` - : `Failed to start dashboard forward on port ${actualPort}: ${fwdDiagnostic.slice(0, 240)}`, + : `Failed to start dashboard gRPC forward on port ${actualPort}: ${fwdDiagnostic.slice(0, 240)}`, ); rollbackSandboxAndExit(sandboxName, err); } @@ -304,7 +287,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ); console.warn(` Free the port, then reconnect: ${deps.cliName()} ${sandboxName} connect`); } else { - console.warn(`! Port ${actualPort} forward did not start: ${fwdDiagnostic.slice(0, 240)}`); + console.warn(`! Port ${actualPort} gRPC forward did not start: ${fwdDiagnostic.slice(0, 240)}`); console.warn(` Reconnect after resolving the issue: ${deps.cliName()} ${sandboxName} connect`); } } @@ -323,27 +306,18 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } function fetchGatewayAuthTokenFromSandbox(sandboxName: string): string | null { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-token-")); try { - const destDir = `${tmpDir}${path.sep}`; - const result = deps.runOpenshell( - ["sandbox", "download", sandboxName, "/sandbox/.openclaw/openclaw.json", destDir], - { ignoreError: true, stdio: ["ignore", "ignore", "ignore"] }, + const result = execBinaryStreamSync( + sandboxName, + ["cat", "/sandbox/.openclaw/openclaw.json"], + { timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS }, ); if (result.status !== 0) return null; - const jsonPath = findOpenclawJsonPath(tmpDir); - if (!jsonPath) return null; - const cfg = JSON.parse(fs.readFileSync(jsonPath, "utf-8")); + const cfg = JSON.parse(result.stdout.toString("utf-8")); const token = cfg && cfg.gateway && cfg.gateway.auth && cfg.gateway.auth.token; return typeof token === "string" && token.length > 0 ? token : null; } catch { return null; - } finally { - try { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } catch { - // ignore cleanup errors - } } } diff --git a/src/lib/onboard/sandbox-verification-exec.ts b/src/lib/onboard/sandbox-verification-exec.ts index e186d79fae..6c8d28724b 100644 --- a/src/lib/onboard/sandbox-verification-exec.ts +++ b/src/lib/onboard/sandbox-verification-exec.ts @@ -8,9 +8,7 @@ * result, so we deliberately swallow spawn errors here. */ -import { spawnSync } from "node:child_process"; - -import { getOpenshellBinary } from "../adapters/openshell/runtime"; +import { execTextSync } from "../adapters/openshell/grpc"; const SANDBOX_EXEC_TIMEOUT_MS = 15000; @@ -19,16 +17,13 @@ export function executeSandboxCommandForVerification( script: string, ): { status: number; stdout: string; stderr: string } | null { try { - const result = spawnSync( - getOpenshellBinary(), - ["sandbox", "exec", "-n", sandboxName, "--", "sh", "-c", script], - { encoding: "utf-8", timeout: SANDBOX_EXEC_TIMEOUT_MS, stdio: ["ignore", "pipe", "pipe"] }, - ); - if (result.error) return null; + const result = execTextSync(sandboxName, ["sh", "-c", script], { + timeoutMs: SANDBOX_EXEC_TIMEOUT_MS, + }); return { - status: result.status ?? 1, - stdout: (result.stdout || "").trim(), - stderr: (result.stderr || "").trim(), + status: result.status, + stdout: result.stdout.trim(), + stderr: result.stderr.trim(), }; } catch { return null; diff --git a/src/lib/sandbox/version.test.ts b/src/lib/sandbox/version.test.ts index 8e2da099b0..8fd6d40d65 100644 --- a/src/lib/sandbox/version.test.ts +++ b/src/lib/sandbox/version.test.ts @@ -6,11 +6,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -// Mock heavy dependencies that pull in the full module graph -vi.mock("../adapters/openshell/resolve.js", () => ({ - resolveOpenshell: vi.fn(() => "/usr/local/bin/openshell"), -})); - vi.mock("../adapters/openshell/client.js", () => ({ parseVersionFromText: (value = "") => { const match = String(value).match(/([0-9]+\.[0-9]+\.[0-9]+)/); @@ -28,7 +23,10 @@ vi.mock("../adapters/openshell/client.js", () => ({ } return true; }, - captureSandboxSshConfigCommand: vi.fn(), +})); + +vi.mock("../adapters/openshell/grpc.js", () => ({ + execTextSync: vi.fn(), })); vi.mock("../agent/defs.js", () => ({ @@ -42,13 +40,7 @@ vi.mock("../agent/defs.js", () => ({ })), })); -vi.mock("child_process", async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, spawnSync: vi.fn() }; -}); - -import { spawnSync } from "child_process"; -import { captureSandboxSshConfigCommand } from "../adapters/openshell/client.js"; +import { execTextSync } from "../adapters/openshell/grpc.js"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts.js"; import * as registry from "../state/registry.js"; import { checkAgentVersion, formatStalenessWarning } from "./version.js"; @@ -110,31 +102,23 @@ describe("checkAgentVersion", () => { expect(result.isStale).toBe(false); }); - it("slow path: probes via SSH when no cached version", () => { + it("slow path: probes via gRPC when no cached version", () => { registry.registerSandbox({ name: "test-sb", agent: null }); - vi.mocked(captureSandboxSshConfigCommand).mockReturnValue({ - status: 0, - output: "Host openshell-test-sb\n HostName 127.0.0.1\n", - }); - - vi.mocked(spawnSync).mockReturnValue({ + vi.mocked(execTextSync).mockReturnValue({ status: 0, stdout: "OpenClaw 2026.5.22 (abc123)\n", stderr: "", - pid: 1234, - output: [], - signal: null, }); const result = checkAgentVersion("test-sb"); - expect(result.detectionMethod).toBe("ssh-exec"); + expect(result.detectionMethod).toBe("grpc-exec"); expect(result.sandboxVersion).toBe("2026.5.22"); expect(result.isStale).toBe(false); - expect(captureSandboxSshConfigCommand).toHaveBeenCalledWith( - "/usr/local/bin/openshell", + expect(execTextSync).toHaveBeenCalledWith( "test-sb", - { ignoreError: true, timeout: OPENSHELL_PROBE_TIMEOUT_MS }, + ["sh", "-c", "openclaw --version"], + { timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS }, ); // Should have cached the version in registry @@ -142,12 +126,11 @@ describe("checkAgentVersion", () => { expect(updated?.agentVersion).toBe("2026.5.22"); }); - it("returns unavailable when SSH config fails", () => { + it("returns unavailable when gRPC exec fails", () => { registry.registerSandbox({ name: "test-sb", agent: null }); - vi.mocked(captureSandboxSshConfigCommand).mockReturnValue({ - status: 1, - output: "", + vi.mocked(execTextSync).mockImplementation(() => { + throw new Error("gRPC unavailable"); }); const result = checkAgentVersion("test-sb"); @@ -157,16 +140,14 @@ describe("checkAgentVersion", () => { it("can skip live probing when no cached version is available", () => { registry.registerSandbox({ name: "test-sb", agent: null }); - vi.mocked(captureSandboxSshConfigCommand).mockClear(); - vi.mocked(spawnSync).mockClear(); + vi.mocked(execTextSync).mockClear(); const result = checkAgentVersion("test-sb", { skipProbe: true }); expect(result.detectionMethod).toBe("unavailable"); expect(result.sandboxVersion).toBeNull(); expect(result.isStale).toBe(false); - expect(captureSandboxSshConfigCommand).not.toHaveBeenCalled(); - expect(spawnSync).not.toHaveBeenCalled(); + expect(execTextSync).not.toHaveBeenCalled(); }); it("force probe bypasses cached version", () => { @@ -176,22 +157,14 @@ describe("checkAgentVersion", () => { agentVersion: "2026.3.11", }); - vi.mocked(captureSandboxSshConfigCommand).mockReturnValue({ - status: 0, - output: "Host openshell-test-sb\n HostName 127.0.0.1\n", - }); - - vi.mocked(spawnSync).mockReturnValue({ + vi.mocked(execTextSync).mockReturnValue({ status: 0, stdout: "OpenClaw 2026.5.22 (abc123)\n", stderr: "", - pid: 1234, - output: [], - signal: null, }); const result = checkAgentVersion("test-sb", { forceProbe: true }); - expect(result.detectionMethod).toBe("ssh-exec"); + expect(result.detectionMethod).toBe("grpc-exec"); expect(result.sandboxVersion).toBe("2026.5.22"); }); }); diff --git a/src/lib/sandbox/version.ts b/src/lib/sandbox/version.ts index 5e57aa1266..b36e9cc712 100644 --- a/src/lib/sandbox/version.ts +++ b/src/lib/sandbox/version.ts @@ -5,20 +5,11 @@ // // Compares the agent version running inside a sandbox against the version // this NemoClaw release was built for. Two code paths: -// Fast: registry lookup (no SSH, used when agentVersion is already cached) -// Slow: SSH exec into sandbox, run version_command, cache result in registry +// Fast: registry lookup (no sandbox RPC, used when agentVersion is already cached) +// Slow: gRPC exec into sandbox, run version_command, cache result in registry -import { spawnSync } from "child_process"; -import fs from "fs"; -import os from "os"; -import path from "path"; - -import { - captureSandboxSshConfigCommand, - parseVersionFromText, - versionGte, -} from "../adapters/openshell/client.js"; -import { resolveOpenshell } from "../adapters/openshell/resolve.js"; +import { parseVersionFromText, versionGte } from "../adapters/openshell/client.js"; +import { execTextSync } from "../adapters/openshell/grpc.js"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts.js"; import { loadAgent } from "../agent/defs.js"; import * as registry from "../state/registry.js"; @@ -27,7 +18,7 @@ export interface VersionCheckResult { sandboxVersion: string | null; expectedVersion: string | null; isStale: boolean; - detectionMethod: "registry" | "ssh-exec" | "unavailable"; + detectionMethod: "registry" | "grpc-exec" | "unavailable"; } /** @@ -41,43 +32,20 @@ function resolveAgentForSandbox(sandboxName: string): ReturnType` and return output. */ - getSshConfig: (sandboxName: string) => { status: number | null; output: string }; /** Ensure the sandbox is live, exit process if not. */ ensureLive: (sandboxName: string) => Promise; /** - * Check whether `remotePath` exists inside the sandbox via - * `openshell sandbox exec -n -- test -e `. Returns true when - * the path exists; false when it is missing, when the sandbox is unreachable, - * or when the exec itself fails. Used by `share mount` as a pre-flight - * before invoking `sshfs`, which exits non-zero with empty stderr on a - * missing remote path and leaves the user with nothing actionable. See #3414. + * Check whether `remotePath` exists inside the sandbox via gRPC exec. + * Returns true when the path exists; false when it is missing, when the + * sandbox is unreachable, or when the exec itself fails. */ checkSandboxPathExists: (sandboxName: string, remotePath: string) => boolean; /** NVIDIA-green ANSI code (empty string if color disabled). */ @@ -28,35 +24,23 @@ export interface ShareCommandDeps { } export function buildShareCommandDeps(): ShareCommandDeps { - const { captureOpenshell, captureSandboxSshConfig } = require("./adapters/openshell/runtime") as { - captureOpenshell: ( - args: string[], - opts?: { ignoreError?: boolean; timeout?: number }, - ) => { status: number | null; output: string }; - captureSandboxSshConfig: ( - sandboxName: string, - opts?: { ignoreError?: boolean; timeout?: number }, - ) => { status: number | null; output: string }; - }; const { ensureLiveSandboxOrExit } = require("./actions/sandbox/gateway-state") as { ensureLiveSandboxOrExit: (sandboxName: string) => Promise; }; return { - getSshConfig: (sandboxName: string) => - captureSandboxSshConfig(sandboxName, { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }), ensureLive: async (sandboxName: string) => { await ensureLiveSandboxOrExit(sandboxName); }, checkSandboxPathExists: (sandboxName: string, remotePath: string) => { - const result = captureOpenshell( - ["sandbox", "exec", "-n", sandboxName, "--", "test", "-e", remotePath], - { ignoreError: true, timeout: OPENSHELL_PROBE_TIMEOUT_MS }, - ); - return result.status === 0; + try { + const result = execTextSync(sandboxName, ["test", "-e", remotePath], { + timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS, + }); + return result.status === 0; + } catch { + return false; + } }, colorGreen: G, colorReset: R, diff --git a/src/lib/share-command.test.ts b/src/lib/share-command.test.ts index ce7cc28009..6a2e494fab 100644 --- a/src/lib/share-command.test.ts +++ b/src/lib/share-command.test.ts @@ -25,10 +25,6 @@ import type { ShareCommandDeps } from "./share-command-deps"; function makeDeps(overrides: Partial = {}): ShareCommandDeps { return { - getSshConfig: vi.fn(() => ({ - status: 0, - output: "Host openshell-alpha\n HostName 127.0.0.1\n", - })), ensureLive: vi.fn(async () => undefined), checkSandboxPathExists: vi.fn(() => true), colorGreen: "", @@ -136,91 +132,39 @@ describe("ShareCommand mount/status actions", () => { vi.restoreAllMocks(); }); - it("mounts through sshfs with a private temporary ssh config and cleans it up", async () => { + it("rejects live mounts under the gRPC-only transport", async () => { const deps = makeDeps(); const localMount = fs.mkdtempSync(path.join(process.cwd(), ".tmp-share-mount-")); - let sshfsConfigPath = ""; try { - spawnSyncMock.mockImplementation((cmd: string, args: string[]) => { - if (cmd === "sh" && args[1] === "command -v sshfs") { - return { status: 0, stdout: "/usr/bin/sshfs\n", stderr: "" }; - } - if (cmd === "mountpoint") return { status: 1, stdout: "", stderr: "" }; - if (cmd === "mount") return { status: 0, stdout: "", stderr: "" }; - if (cmd === "sshfs") { - const configFlagIndex = args.indexOf("-F"); - sshfsConfigPath = args[configFlagIndex + 1]; - expect(fs.statSync(sshfsConfigPath).mode & 0o777).toBe(0o600); - expect(args).toContain("sftp_server=/usr/lib/openssh/sftp-server"); - expect(args).toContain("openshell-alpha:/workspace"); - expect(args.at(-1)).toBe(localMount); - return { status: 0, stdout: "", stderr: "" }; - } - return { status: 1, stdout: "", stderr: `unexpected ${cmd} ${args.join(" ")}` }; - }); - - await runShareMount({ sandboxName: "alpha", remotePath: "/workspace", localMount }, deps); - + await expect( + runShareMount({ sandboxName: "alpha", remotePath: "/workspace", localMount }, deps), + ).rejects.toThrow(/Live sandbox filesystem mounts are no longer supported/); expect(deps.ensureLive).toHaveBeenCalledWith("alpha"); - expect(deps.getSshConfig).toHaveBeenCalledWith("alpha"); - expect(spawnSyncMock).toHaveBeenCalledWith( - "sshfs", - expect.arrayContaining(["openshell-alpha:/workspace", localMount]), - expect.objectContaining({ timeout: 30_000 }), - ); - expect(sshfsConfigPath).not.toBe(""); - expect(fs.existsSync(sshfsConfigPath)).toBe(false); - expect(fs.existsSync(path.dirname(sshfsConfigPath))).toBe(false); - expect( - logSpy.mock.calls.some((call: unknown[]) => String(call[0]).includes("Mounted /workspace")), - ).toBe(true); + expect(spawnSyncMock).not.toHaveBeenCalledWith("sshfs", expect.any(Array), expect.anything()); } finally { fs.rmSync(localMount, { recursive: true, force: true }); } }); - it("exits with an install hint when sshfs is missing", async () => { + it("does not require host mount tools before reporting unsupported mount", async () => { const deps = makeDeps(); - spawnSyncMock.mockImplementation((cmd: string, args: string[]) => { - if (cmd === "sh" && args[1] === "command -v sshfs") { - return { status: 1, stdout: "", stderr: "" }; - } - return { status: 1, stdout: "", stderr: "" }; - }); - await expect(runShareMount({ sandboxName: "alpha" }, deps)).rejects.toThrow( ShareCommandError, ); - - expect(deps.ensureLive).not.toHaveBeenCalled(); await expect(runShareMount({ sandboxName: "alpha" }, deps)).rejects.toThrow( - /sshfs is not installed/, + /OpenShell gRPC/, ); + expect(deps.ensureLive).toHaveBeenCalledWith("alpha"); }); - it("surfaces SFTP-specific remediation when sshfs fails after sandbox validation", async () => { + it("still checks sandbox liveness before reporting unsupported mount", async () => { const deps = makeDeps(); const localMount = fs.mkdtempSync(path.join(process.cwd(), ".tmp-share-sftp-")); try { - spawnSyncMock.mockImplementation((cmd: string, args: string[]) => { - if (cmd === "sh" && args[1] === "command -v sshfs") return { status: 0, stdout: "sshfs\n" }; - if (cmd === "mountpoint") return { status: 1, stdout: "", stderr: "" }; - if (cmd === "mount") return { status: 0, stdout: "", stderr: "" }; - if (cmd === "sshfs") - return { status: 1, stdout: "", stderr: "subsystem request failed: sftp" }; - return { status: 1, stdout: "", stderr: "" }; - }); - await expect( runShareMount({ sandboxName: "alpha", remotePath: "/sandbox", localMount }, deps), - ).rejects.toThrow(/SSHFS mount failed/); - - await expect( - runShareMount({ sandboxName: "alpha", remotePath: "/sandbox", localMount }, deps), - ).rejects.toThrow(/openssh-sftp-server/); - await expect( - runShareMount({ sandboxName: "alpha", remotePath: "/sandbox", localMount }, deps), - ).rejects.toThrow(/nemoclaw alpha rebuild --yes/); + ).rejects.toThrow(/share status/); + expect(deps.ensureLive).toHaveBeenCalledWith("alpha"); } finally { fs.rmSync(localMount, { recursive: true, force: true }); } diff --git a/src/lib/share-command.ts b/src/lib/share-command.ts index 6680bbc017..b9b0464c11 100644 --- a/src/lib/share-command.ts +++ b/src/lib/share-command.ts @@ -2,11 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 /** - * `nemoclaw share mount|unmount|status` — SSHFS-based sandbox file sharing. + * `nemoclaw share mount|unmount|status`. * - * Mounts the sandbox filesystem on the host via SSHFS, tunneled through - * OpenShell's existing SSH proxy. Requires `sshfs` on the host and - * `openssh-sftp-server` in the sandbox image. + * Live filesystem mounts depended on the legacy SSH filesystem transport. + * Under the gRPC-only sandbox lifecycle, mount is intentionally unsupported; + * status and unmount remain for cleaning up older local mounts. */ import { spawnSync } from "child_process"; @@ -66,13 +66,10 @@ export function defaultShareMountDir(sandboxName: string): string { /** * Pre-flight: confirm the remote source path actually exists inside the - * sandbox. sshfs exits non-zero with empty stderr when the remote path is - * missing (e.g. a typo), and the bare "SSHFS mount failed." line we used to - * emit left the user with nothing actionable. Returns normally when the path - * can be verified; emits a structured error and exits the process non-zero - * when it cannot. The success path has no return value. - * Exported so the behavior is testable without driving the full sshfs - * lifecycle. See #3414. + * sandbox. Retained for older tests and any future non-mount copy flow. + * Returns normally when the path can be verified; emits a structured error + * and exits the process non-zero when it cannot. The success path has no + * return value. */ export function assertSandboxPathExistsOrExit( deps: ShareCommandDeps, @@ -160,119 +157,12 @@ export async function runShareMount( deps: ShareCommandDeps = buildShareCommandDeps(), ): Promise { const { sandboxName } = options; - const remotePath = options.remotePath || "/sandbox"; - const localMount = options.localMount || defaultShareMountDir(sandboxName); - const G = deps.colorGreen; - const R = deps.colorReset; - - // Preflight: check sshfs binary - const sshfsCheck = spawnSync("sh", ["-c", "command -v sshfs"], { - encoding: "utf-8", - stdio: ["ignore", "pipe", "pipe"], - }); - if (sshfsCheck.status !== 0) { - shareFail([ - " sshfs is not installed.", - process.platform === "darwin" - ? " Install with: brew install macfuse && brew install sshfs" - : " Install with: sudo apt-get install sshfs (or: sudo dnf install fuse-sshfs)", - ]); - } - - // Check not already mounted - if (isMountPoint(localMount)) { - shareFail([ - ` ${localMount} is already mounted.`, - ` Run '${deps.cliName} ${sandboxName} share unmount' first.`, - ]); - } - - // Verify sandbox is running await deps.ensureLive(sandboxName); - - // Pre-flight: confirm the remote source path actually exists. See #3414. - assertSandboxPathExistsOrExit(deps, sandboxName, remotePath); - - // Get SSH config - const sshConfigResult = deps.getSshConfig(sandboxName); - if (sshConfigResult.status !== 0) { - shareFail(" Failed to obtain SSH configuration for the sandbox."); - } - - // Use a private temp directory to prevent symlink attacks on predictable paths. - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sshfs-")); - const tmpFile = path.join(tmpDir, `${sandboxName}.conf`); - fs.writeFileSync(tmpFile, sshConfigResult.output, { mode: 0o600, flag: "wx" }); - - const writable = checkLocalMountWritable(localMount); - if (!writable.writable) { - console.error(` Local mount path '${localMount}' is not usable: ${writable.reason}.`); - console.error(" share mount projects sandbox files onto a host directory via SSHFS,"); - console.error(" so the local target must be on a writable filesystem."); - console.error( - ` Pick a writable directory: ${deps.cliName} ${sandboxName} share mount ${remotePath} `, - ); - try { - fs.unlinkSync(tmpFile); - fs.rmdirSync(tmpDir); - } catch { - /* ignore */ - } - process.exit(1); - } - - let mountFailed = false; - try { - const result = spawnSync( - "sshfs", - [ - "-F", - tmpFile, - "-o", - "sftp_server=/usr/lib/openssh/sftp-server", - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - "reconnect", - "-o", - "ServerAliveInterval=15", - "-o", - "ServerAliveCountMax=3", - `openshell-${sandboxName}:${remotePath}`, - localMount, - ], - { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: 30000 }, - ); - if (result.status !== 0) { - const stderr = (result.stderr || "").trim(); - mountFailed = true; - const lines = [" SSHFS mount failed."]; - if (stderr) lines.push(` ${stderr}`); - if (/sftp/i.test(stderr)) { - lines.push(" The sandbox may lack openssh-sftp-server."); - lines.push( - ` If this sandbox uses the default base image, rebuild with: ${deps.cliName} ${sandboxName} rebuild --yes`, - ); - lines.push( - " If it was created from a custom `--from` image, add openssh-sftp-server at /usr/lib/openssh/sftp-server and rebuild.", - ); - } - shareFail(lines); - } else { - console.log(` ${G}✓${R} Mounted ${remotePath} → ${localMount}`); - console.log(` Edit files at ${localMount} — changes appear in the sandbox instantly.`); - } - } finally { - try { - fs.unlinkSync(tmpFile); - fs.rmdirSync(tmpDir); - } catch { - /* ignore */ - } - } - if (mountFailed) shareFail(" SSHFS mount failed."); + shareFail([ + " Live sandbox filesystem mounts are no longer supported.", + " NemoClaw now uses OpenShell gRPC for sandbox lifecycle operations, and OpenShell does not provide a live filesystem mount API on that transport.", + ` Existing legacy mounts can still be inspected or removed with '${deps.cliName} ${sandboxName} share status' and '${deps.cliName} ${sandboxName} share unmount'.`, + ]); } export function runShareUnmount( @@ -338,7 +228,7 @@ export function printShareUsageAndExit(exitCode = 1): never { const { cliName } = buildShareCommandDeps(); shareFail([ ` Usage: ${cliName} share `, - " mount [sandbox-path] [local-mount-point] Mount sandbox filesystem via SSHFS", + " mount [sandbox-path] [local-mount-point] Unsupported under gRPC-only transport", " unmount [local-mount-point] Unmount a previously mounted filesystem", " status [local-mount-point] Check current mount status", ], exitCode); diff --git a/src/lib/skill-install.ts b/src/lib/skill-install.ts index f83141edd6..ef504928dc 100644 --- a/src/lib/skill-install.ts +++ b/src/lib/skill-install.ts @@ -2,18 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 // // Skill install logic for `nemoclaw skill install `. -// Validates a local SKILL.md, uploads it to the sandbox via SSH, and +// Validates a local SKILL.md, uploads it to the sandbox via OpenShell gRPC, and // performs agent-specific post-install steps (session refresh for // OpenClaw). Non-OpenClaw agents get a "restart gateway" hint until a // generic refresh contract is defined in the manifest schema. -import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; // yaml is a production dependency (used by policies.ts, onboard.ts) import YAML from "yaml"; +import { execInputStreamSync, execTextSync } from "./adapters/openshell/grpc"; import { isRecord } from "./core/json-types"; // ── Frontmatter parsing ────────────────────────────────────────── @@ -131,10 +131,11 @@ export function validateRelativePath(rel: string): boolean { return segments.every((s) => s !== "" && s !== ".." && s !== "."); } -// ── SSH helpers ────────────────────────────────────────────────── +// ── Sandbox gRPC exec helpers ──────────────────────────────────── export interface SshContext { - configFile: string; + /** @deprecated retained for test fixtures that still construct the old SSH context shape. */ + configFile?: string; sandboxName: string; } @@ -145,8 +146,7 @@ export interface SshResult { } /** - * Run a command on the sandbox via SSH with optional stdin content. - * Uses the same SSH flags as executeSandboxCommand in sandbox-process-recovery-action.ts. + * Run a command in the sandbox with optional stdin content. */ export function sshExec( ctx: SshContext, @@ -154,33 +154,24 @@ export function sshExec( opts: { input?: string | Buffer; timeout?: number } = {}, ): SshResult | null { try { - const result = spawnSync( - "ssh", - [ - "-F", - ctx.configFile, - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - "ConnectTimeout=10", - "-o", - "LogLevel=ERROR", - `openshell-${ctx.sandboxName}`, - command, - ], - { - encoding: "utf-8", - stdio: [opts.input !== undefined ? "pipe" : "ignore", "pipe", "pipe"], - input: opts.input, - timeout: opts.timeout ?? 30_000, - }, - ); + const timeoutMs = opts.timeout ?? 30_000; + const result = + opts.input === undefined + ? execTextSync(ctx.sandboxName, ["sh", "-c", command], { timeoutMs }) + : (() => { + const streamed = execInputStreamSync(ctx.sandboxName, ["sh", "-c", command], opts.input, { + timeoutMs, + }); + return { + status: streamed.status, + stdout: streamed.stdout.toString("utf-8"), + stderr: streamed.stderr.toString("utf-8"), + }; + })(); return { - status: result.status ?? 1, - stdout: (result.stdout || "").trim(), - stderr: (result.stderr || "").trim(), + status: result.status, + stdout: result.stdout.trim(), + stderr: result.stderr.trim(), }; } catch { return null; @@ -188,7 +179,7 @@ export function sshExec( } /** - * Upload a file to the sandbox by piping its content through SSH stdin. + * Upload a file to the sandbox by piping its content through gRPC stdin. * Creates the target directory and writes the file in a single remote command. */ export function uploadFile( @@ -213,7 +204,7 @@ export interface CollectedFiles { * Collect files under `dir` recursively, returning paths relative to `dir`. * Dotfiles (names starting with `.`) are excluded by default and reported * separately so the caller can warn. Paths with unsafe characters are - * rejected to prevent shell injection when interpolated into SSH commands. + * rejected to prevent shell injection when interpolated into remote shell commands. */ export function collectFiles(dir: string): CollectedFiles { const files: string[] = []; diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 518d43e307..82b972a8df 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -25,9 +25,11 @@ import os from "node:os"; import path from "node:path"; import { spawnSync } from "child_process"; -import { captureSandboxSshConfigCommand } from "../adapters/openshell/client.js"; -import { resolveOpenshell } from "../adapters/openshell/resolve.js"; -import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts.js"; +import { + execBinaryStreamSync, + execInputStreamSync, + execTextSync, +} from "../adapters/openshell/grpc.js"; import type { AgentStateFile } from "../agent/defs.js"; import { loadAgent } from "../agent/defs.js"; import { isRecord, type UnknownRecord } from "../core/json-types.js"; @@ -461,40 +463,6 @@ export function safeTarExtract(tarBuffer: Buffer, targetDir: string): SafeExtrac // ── Helpers ──────────────────────────────────────────────────────── -function getSshConfig(sandboxName: string): string | null { - const openshellBinary = resolveOpenshell(); - if (!openshellBinary) return null; - - const result = captureSandboxSshConfigCommand(openshellBinary, sandboxName, { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - if (result.status !== 0) return null; - return result.output; -} - -function writeTempSshConfig(sshConfig: string): string { - const tmpFile = path.join(os.tmpdir(), `nemoclaw-state-${process.pid}-${Date.now()}.conf`); - writeFileSync(tmpFile, sshConfig, { mode: 0o600 }); - return tmpFile; -} - -function sshArgs(configFile: string, sandboxName: string): string[] { - return [ - "-F", - configFile, - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - "ConnectTimeout=10", - "-o", - "LogLevel=ERROR", - `openshell-${sandboxName}`, - ]; -} - function computeBlueprintDigest(): string | null { // Look for blueprint.yaml relative to the agent-defs ROOT const candidates = [ @@ -839,7 +807,6 @@ function buildStateFileBackupCommand(dir: string, spec: StateFileSpec): string { } function backupStateFile( - configFile: string, sandboxName: string, dir: string, spec: StateFileSpec, @@ -847,18 +814,22 @@ function backupStateFile( ): "backed_up" | "missing" | "failed" { const command = buildStateFileBackupCommand(dir, spec); _log(`Backing up state file ${spec.path} (${spec.strategy})`); - const result = spawnSync("ssh", [...sshArgs(configFile, sandboxName), command], { - stdio: ["ignore", "pipe", "pipe"], - timeout: 120000, - maxBuffer: 256 * 1024 * 1024, - }); + let result: ReturnType; + try { + result = execBinaryStreamSync(sandboxName, ["sh", "-c", command], { timeoutMs: 120000 }); + } catch (error) { + _log( + `FAILED: state file backup ${spec.path}: ${ + error instanceof Error ? error.message : String(error) + }`.substring(0, 240), + ); + return "failed"; + } if (result.status === 2) return "missing"; - if (result.status !== 0 || result.error || result.signal || !result.stdout) { + if (result.status !== 0 || !result.stdout) { const detail = - (result.stderr?.toString() || "").trim() || - result.error?.message || - (result.signal ? `signal ${result.signal}` : `exit ${String(result.status)}`); + result.stderr.toString().trim() || `exit ${String(result.status)}`; _log(`FAILED: state file backup ${spec.path}: ${detail.substring(0, 200)}`); return "failed"; } @@ -906,7 +877,6 @@ function buildStateFileRestoreCommand(dir: string, spec: StateFileSpec): string } function restoreStateFile( - configFile: string, sandboxName: string, dir: string, spec: StateFileSpec, @@ -917,18 +887,24 @@ function restoreStateFile( const command = buildStateFileRestoreCommand(dir, spec); _log(`Restoring state file ${spec.path} (${spec.strategy})`); - const result = spawnSync("ssh", [...sshArgs(configFile, sandboxName), command], { - input: readFileSync(localPath), - stdio: ["pipe", "pipe", "pipe"], - timeout: 120000, - }); + let result: ReturnType; + try { + result = execInputStreamSync(sandboxName, ["sh", "-c", command], readFileSync(localPath), { + timeoutMs: 120000, + }); + } catch (error) { + _log( + `FAILED: state file restore ${spec.path}: ${ + error instanceof Error ? error.message : String(error) + }`.substring(0, 240), + ); + return false; + } - if (result.status === 0 && !result.error && !result.signal) return true; + if (result.status === 0) return true; const detail = - (result.stderr?.toString() || "").trim() || - result.error?.message || - (result.signal ? `signal ${result.signal}` : `exit ${String(result.status)}`); + result.stderr.toString().trim() || `exit ${String(result.status)}`; _log(`FAILED: state file restore ${spec.path}: ${detail.substring(0, 200)}`); return false; } @@ -1028,23 +1004,6 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = return { success: true, manifest, backedUpDirs, failedDirs, backedUpFiles, failedFiles }; } - // SSH+tar single-roundtrip download - _log("Getting SSH config via openshell sandbox ssh-config"); - const sshConfig = getSshConfig(sandboxName); - if (!sshConfig) { - _log("FAILED: Could not get SSH config"); - return { - success: false, - manifest, - backedUpDirs, - failedDirs: [...stateDirs], - backedUpFiles, - failedFiles: stateFiles.map((f) => f.path), - }; - } - _log(`SSH config obtained (${sshConfig.length} bytes)`); - - const configFile = writeTempSshConfig(sshConfig); try { if (stateDirs.length > 0) { // Build tar command that only includes existing directories. @@ -1058,12 +1017,27 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = .join("; "); const workspaceGlobCmd = `for d in ${shellQuote(dir)}/workspace-*/; do [ -d "$d" ] && basename "$d"; done 2>/dev/null`; const fullCheckCmd = `{ ${existCheckCmd}; ${workspaceGlobCmd}; } 2>/dev/null | awk '!seen[$0]++'`; - _log(`Checking existing dirs via SSH: ${fullCheckCmd.substring(0, 100)}...`); - const existResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), fullCheckCmd], { - encoding: "utf-8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 30000, - }); + _log(`Checking existing dirs via gRPC exec: ${fullCheckCmd.substring(0, 100)}...`); + let existResult: ReturnType; + try { + existResult = execTextSync(sandboxName, ["sh", "-c", fullCheckCmd], { + timeoutMs: 30000, + }); + } catch (error) { + _log( + `FAILED: gRPC dir check failed — ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return { + success: false, + manifest, + backedUpDirs, + failedDirs: [...stateDirs], + backedUpFiles, + failedFiles: stateFiles.map((f) => f.path), + }; + } _log( `Dir check: exit=${existResult.status}, stdout=${(existResult.stdout || "").trim().substring(0, 200)}, stderr=${(existResult.stderr || "").trim().substring(0, 200)}`, ); @@ -1077,7 +1051,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = if (existResult.status !== 0) { _log( - `FAILED: SSH dir check exited ${existResult.status} — cannot determine which dirs exist`, + `FAILED: gRPC dir check exited ${existResult.status} — cannot determine which dirs exist`, ); return { success: false, @@ -1103,7 +1077,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // Per-dir `find` invocations are joined with `;` (not `&&`) and each // is tolerant of its own exit code via `|| true`. The base image bakes // a few state subdirs as root-owned (e.g. `extensions/`, - // `agents/`) and `find` walking those from the sandbox-user SSH + // `agents/`) and `find` walking those from the sandbox-user exec // session exits 1 on permission denied. The audit's real signal is // stdout (the printf-emitted symlink/hardlink/special-file rows); // letting one perm-denied subdir abort the whole chain blocks legitimate @@ -1115,15 +1089,27 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = ) .join("; "); _log(`Pre-backup audit: checking for symlinks, hard links, and special files`); - const auditResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), auditCmd], { - encoding: "utf-8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 30000, - }); + let auditResult: ReturnType; + try { + auditResult = execTextSync(sandboxName, ["sh", "-c", auditCmd], { + timeoutMs: 30000, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + _log(`FAILED: Pre-backup audit gRPC command failed — ${detail}`); + return { + success: false, + manifest, + backedUpDirs, + failedDirs: [...existingDirs], + backedUpFiles, + failedFiles: stateFiles.map((f) => f.path), + error: `Pre-backup audit failed: ${detail}`, + }; + } if (auditResult.status !== 0) { - const stderr = (auditResult.stderr || "").trim(); - const detail = - stderr || auditResult.error?.message || `exit ${String(auditResult.status)}`; + const stderr = auditResult.stderr.trim(); + const detail = stderr || `exit ${String(auditResult.status)}`; _log(`FAILED: Pre-backup audit command failed — ${detail}`); return { success: false, @@ -1180,19 +1166,27 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = } _log("Pre-backup audit passed — no unsafe symlinks, hard links, or special files found"); - // Download via SSH+tar + // Download via gRPC+tar // NC-2227-04: Removed -h flag (was following symlinks). State dirs are // now agent-writable and co-located with config — a compromised agent // could create symlinks to exfiltrate config contents via backup. const tarCmd = `tar -cf - -C ${shellQuote(dir)} -- ${existingDirs.map(shellQuote).join(" ")}`; - _log(`Downloading via SSH+tar: ${tarCmd}`); - const result = spawnSync("ssh", [...sshArgs(configFile, sandboxName), tarCmd], { - stdio: ["ignore", "pipe", "pipe"], - timeout: 120000, - maxBuffer: 256 * 1024 * 1024, - }); + _log(`Downloading via gRPC+tar: ${tarCmd}`); + let result: ReturnType; + try { + result = execBinaryStreamSync(sandboxName, ["sh", "-c", tarCmd], { + timeoutMs: 120000, + }); + } catch (error) { + _log( + `FAILED: gRPC tar download failed — ${ + error instanceof Error ? error.message : String(error) + }`, + ); + result = { status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; + } _log( - `SSH+tar download: exit=${result.status}, stdout=${result.stdout ? result.stdout.length + " bytes" : "null"}, stderr=${(result.stderr?.toString() || "").substring(0, 200)}`, + `gRPC+tar download: exit=${result.status}, stdout=${result.stdout ? result.stdout.length + " bytes" : "null"}, stderr=${result.stderr.toString().substring(0, 200)}`, ); // GNU tar exit codes: 0 = success, 1 = files changed during archive, @@ -1224,7 +1218,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = } } else { const tarFailedDirs = failedDirsFromTarStderr( - result.stderr?.toString() || "", + result.stderr.toString() || "", existingDirs, ); if (tarFailedDirs.size === 0) { @@ -1257,19 +1251,17 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = } for (const spec of stateFiles) { - const result = backupStateFile(configFile, sandboxName, dir, spec, backupPath); + const result = backupStateFile(sandboxName, dir, spec, backupPath); if (result === "backed_up") { backedUpFiles.push(spec.path); } else if (result === "failed") { failedFiles.push(spec.path); } } - } finally { - try { - require("node:fs").unlinkSync(configFile); - } catch { - /* ignore */ - } + } catch (error) { + _log(`FAILED: gRPC backup operation failed: ${error instanceof Error ? error.message : String(error)}`); + failedDirs.push(...stateDirs.filter((d) => !backedUpDirs.includes(d))); + failedFiles.push(...stateFiles.map((f) => f.path).filter((p) => !backedUpFiles.includes(p))); } // SECURITY: Strip credentials from the local backup @@ -1358,20 +1350,6 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re return { success: true, restoredDirs, failedDirs, restoredFiles, failedFiles }; } - _log("Getting SSH config for restore"); - const sshConfig = getSshConfig(sandboxName); - if (!sshConfig) { - _log("FAILED: Could not get SSH config for restore"); - return { - success: false, - restoredDirs, - failedDirs: [...localDirs], - restoredFiles, - failedFiles: localFiles.map((f) => f.path), - }; - } - - const configFile = writeTempSshConfig(sshConfig); try { if (localDirs.length > 0) { // Upload via tar pipe @@ -1408,16 +1386,25 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re // are cleared and restored from the backup. const rmCmd = buildRestoreCleanupCommand(dir, localDirs, preserveManagedExtensions); _log(`Cleaning target dirs before restore: ${rmCmd}`); - const rmResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), rmCmd], { - stdio: ["ignore", "pipe", "pipe"], - timeout: 30000, - }); - if (rmResult.status !== 0 || rmResult.error || rmResult.signal) { - const stderr = (rmResult.stderr?.toString() || "").trim(); - const detail = - stderr || - rmResult.error?.message || - (rmResult.signal ? `signal ${rmResult.signal}` : `exit ${String(rmResult.status)}`); + let rmResult: ReturnType; + try { + rmResult = execBinaryStreamSync(sandboxName, ["sh", "-c", rmCmd], { + timeoutMs: 30000, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + _log(`FAILED: pre-restore cleanup failed: ${detail.substring(0, 200)}`); + return { + success: false, + restoredDirs, + failedDirs: [...localDirs], + restoredFiles, + failedFiles: localFiles.map((f) => f.path), + }; + } + if (rmResult.status !== 0) { + const stderr = rmResult.stderr.toString().trim(); + const detail = stderr || `exit ${String(rmResult.status)}`; _log(`FAILED: pre-restore cleanup failed: ${detail.substring(0, 200)}`); return { success: false, @@ -1429,29 +1416,33 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re } const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`; - const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], { - input: tarResult.stdout, - stdio: ["pipe", "pipe", "pipe"], - timeout: 120000, - }); + let extractResult: ReturnType; + try { + extractResult = execInputStreamSync(sandboxName, ["sh", "-c", extractCmd], tarResult.stdout, { + timeoutMs: 120000, + }); + } catch (error) { + _log( + `FAILED: gRPC tar restore failed: ${ + error instanceof Error ? error.message : String(error) + }`.substring(0, 240), + ); + extractResult = { status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; + } - if (sshResult.status === 0) { + if (extractResult.status === 0) { const restoredPaths = localDirs.map((d) => `${dir}/${d}`); - // Best-effort only: OpenShell exec/SSH normally runs as the sandbox user, + // Best-effort only: OpenShell gRPC exec normally runs as the sandbox user, // which cannot chown even files it owns. The tar restore above runs as the // same user, so the real restore gate is whether the restored state dirs // are usable by that user. const chownCmd = `chown -R sandbox:sandbox -- ${restoredPaths.map(shellQuote).join(" ")} 2>/dev/null || true`; _log(`Best-effort ownership repair: ${chownCmd}`); - const chownResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), chownCmd], { - stdio: ["ignore", "pipe", "pipe"], - timeout: 30000, - }); - if (chownResult.error || chownResult.signal) { - const detail = - chownResult.error?.message || - (chownResult.signal ? `signal ${chownResult.signal}` : "unknown error"); + try { + execBinaryStreamSync(sandboxName, ["sh", "-c", chownCmd], { timeoutMs: 30000 }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); _log( `WARNING: post-restore ownership repair did not complete: ${detail.substring(0, 200)}`, ); @@ -1464,24 +1455,23 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re ) .join(" && "); _log(`Verifying restored state usability: ${usabilityCmd}`); - const usabilityResult = spawnSync( - "ssh", - [...sshArgs(configFile, sandboxName), usabilityCmd], - { - stdio: ["ignore", "pipe", "pipe"], - timeout: 30000, - }, - ); - if (usabilityResult.status === 0 && !usabilityResult.error && !usabilityResult.signal) { + let usabilityResult: ReturnType; + try { + usabilityResult = execBinaryStreamSync(sandboxName, ["sh", "-c", usabilityCmd], { + timeoutMs: 30000, + }); + } catch (error) { + usabilityResult = { + status: 1, + stdout: Buffer.alloc(0), + stderr: Buffer.from(error instanceof Error ? error.message : String(error)), + }; + } + if (usabilityResult.status === 0) { restoredDirs.push(...localDirs); } else { - const stderr = (usabilityResult.stderr?.toString() || "").trim(); - const detail = - stderr || - usabilityResult.error?.message || - (usabilityResult.signal - ? `signal ${usabilityResult.signal}` - : `exit ${String(usabilityResult.status)}`); + const stderr = usabilityResult.stderr.toString().trim(); + const detail = stderr || `exit ${String(usabilityResult.status)}`; _log(`FAILED: restored state usability check failed: ${detail.substring(0, 200)}`); failedDirs.push(...localDirs); } @@ -1491,18 +1481,16 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re } for (const spec of localFiles) { - if (restoreStateFile(configFile, sandboxName, dir, spec, backupPath)) { + if (restoreStateFile(sandboxName, dir, spec, backupPath)) { restoredFiles.push(spec.path); } else { failedFiles.push(spec.path); } } - } finally { - try { - require("node:fs").unlinkSync(configFile); - } catch { - /* ignore */ - } + } catch (error) { + _log(`FAILED: gRPC restore operation failed: ${error instanceof Error ? error.message : String(error)}`); + failedDirs.push(...localDirs.filter((d) => !restoredDirs.includes(d))); + failedFiles.push(...localFiles.map((f) => f.path).filter((p) => !restoredFiles.includes(p))); } return { diff --git a/src/lib/status-command-deps.ts b/src/lib/status-command-deps.ts index dcb20832cf..161f40da91 100644 --- a/src/lib/status-command-deps.ts +++ b/src/lib/status-command-deps.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; - import { getNamedGatewayLifecycleState } from "./gateway-runtime-action"; import { getLiveGatewayInference } from "./inference/live"; import type { GatewayHealth, MessagingBridgeHealth, ShowStatusCommandDeps } from "./inventory"; @@ -10,6 +8,7 @@ import { backfillMessagingChannels, findAllOverlaps } from "./messaging-conflict import type { CaptureOpenshellResult } from "./adapters/openshell/client"; import { captureOpenshellCommand, stripAnsi } from "./adapters/openshell/client"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "./adapters/openshell/timeouts"; +import { execTextSync } from "./adapters/openshell/grpc"; import * as registry from "./state/registry"; import { resolveOpenshell } from "./adapters/openshell/resolve"; import { createSystemDeps, parseSshProcesses } from "./state/sandbox-session"; @@ -32,7 +31,7 @@ function captureOpenshell( } function checkMessagingBridgeHealth( - rootDir: string, + _rootDir: string, sandboxName: string, channels: string[], ): MessagingBridgeHealth[] { @@ -40,16 +39,10 @@ function checkMessagingBridgeHealth( // gateway log. Discord/Slack have similar single-consumer constraints but // log differently; we can extend the regex when those patterns are known. if (!Array.isArray(channels) || !channels.includes("telegram")) return []; - const openshell = resolveOpenshell(); - if (!openshell) return []; const script = 'tail -n 200 /tmp/gateway.log 2>/dev/null | grep -cE "getUpdates conflict|409[[:space:]:]+Conflict" || true'; try { - const result = spawnSync( - openshell, - ["sandbox", "exec", "-n", sandboxName, "--", "sh", "-c", script], - { cwd: rootDir, encoding: "utf-8", timeout: 3000, stdio: ["ignore", "pipe", "pipe"] }, - ); + const result = execTextSync(sandboxName, ["sh", "-c", script], { timeoutMs: 3000 }); const count = Number.parseInt((result.stdout || "").trim(), 10); if (!Number.isFinite(count) || count === 0) return []; return [{ channel: "telegram", conflicts: count }]; @@ -108,24 +101,11 @@ function backfillAndFindOverlaps(rootDir: string) { } } -function readGatewayLog(rootDir: string, sandboxName: string): string | null { - const openshell = resolveOpenshell(); - if (!openshell) return null; +function readGatewayLog(_rootDir: string, sandboxName: string): string | null { try { - const result = spawnSync( - openshell, - [ - "sandbox", - "exec", - "-n", - sandboxName, - "--", - "sh", - "-c", - "tail -n 10 /tmp/gateway.log 2>/dev/null", - ], - { cwd: rootDir, encoding: "utf-8", timeout: 3000, stdio: ["ignore", "pipe", "pipe"] }, - ); + const result = execTextSync(sandboxName, ["sh", "-c", "tail -n 10 /tmp/gateway.log 2>/dev/null"], { + timeoutMs: 3000, + }); const output = (result.stdout || "").trim(); return output || null; } catch { diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index dbee2081c9..5a17eb3160 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -17,6 +17,7 @@ import { import { basename, join } from "node:path"; import { dockerSpawnSync } from "../adapters/docker"; import { resolveOpenshell } from "../adapters/openshell/resolve"; +import { execTextSync } from "../adapters/openshell/grpc"; import { renderBox } from "../cli/banner"; import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding"; import { isRecord } from "../core/json-types"; @@ -434,10 +435,10 @@ export function showStatus(opts: ServiceOptions = {}): void { * Stop the OpenClaw gateway (and its messaging channels) inside the sandbox. * * Uses the OpenShell gateway container's kubectl as the privileged path so it - * can signal the gateway process even when the sandbox SSH/exec user is + * can signal the gateway process even when the in-sandbox command user is * `sandbox` and the gateway process runs as the separate `gateway` user. The - * fallback `openshell sandbox exec` path uses the same verified script for - * older/non-root deployments where the exec user can signal the gateway. + * fallback gRPC exec path uses the same verified script for older/non-root + * deployments where the exec user can signal the gateway. * * The in-sandbox script intentionally does not rely on a bare `pkill -f` * result: `pkill -f openclaw[- ]gateway` can match the transient shell/pkill @@ -452,18 +453,25 @@ export function stopSandboxChannels(sandboxName: string): void { const privilegedResult = stopSandboxChannelsViaKubectl(sandboxName); if (reportStopResult(privilegedResult)) return; - const openshell = resolveOpenshell(); - if (!openshell) { - warn("openshell not found — cannot stop in-sandbox messaging channels."); - return; + try { + const fallbackResult = execTextSync(sandboxName, ["sh", "-lc", GATEWAY_STOP_SCRIPT], { + timeoutMs: 20000, + }); + reportStopResult({ + status: fallbackResult.status, + signal: null, + stdout: fallbackResult.stdout, + stderr: fallbackResult.stderr, + pid: 0, + output: [null, fallbackResult.stdout, fallbackResult.stderr], + } as StopAttemptResult); + } catch (error) { + warn( + `OpenShell gRPC exec could not stop in-sandbox messaging channels: ${ + error instanceof Error ? error.message : String(error) + }`, + ); } - - const fallbackResult = spawnSync( - openshell, - ["sandbox", "exec", "--name", sandboxName, "--", "sh", "-lc", GATEWAY_STOP_SCRIPT], - { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: 20000 }, - ); - reportStopResult(fallbackResult); } const GATEWAY_CLUSTER_CONTAINER = "openshell-cluster-nemoclaw"; diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index 56c0078324..64f42f56ae 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -290,7 +290,7 @@ export async function verifyDeployment( detail: dashboard.detail, hint: dashboard.reachable ? "" - : `Port forward on ${chain.port} is not working. Run: openshell forward start ${chain.forwardTarget} ${sandboxName}`, + : `Dashboard gRPC forward on ${chain.port} is not working. Retry: nemoclaw ${sandboxName} connect --probe-only`, }); // 4. Inference route diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index b2a71747b0..ef2c0d77af 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -290,6 +290,9 @@ function runRebuild( env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: path.join(fixture.tmpDir, "ssh"), NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", diff --git a/test/rebuild-shields-auto-unlock.test.ts b/test/rebuild-shields-auto-unlock.test.ts index ed7efe1649..656dc05988 100644 --- a/test/rebuild-shields-auto-unlock.test.ts +++ b/test/rebuild-shields-auto-unlock.test.ts @@ -297,6 +297,9 @@ function runRebuild(fixture: ReturnType) { env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: path.join(fixture.tmpDir, "ssh"), NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", diff --git a/test/share-command-deps-probe-argv.test.ts b/test/share-command-deps-probe-argv.test.ts index 20bdde44a0..1a301aff33 100644 --- a/test/share-command-deps-probe-argv.test.ts +++ b/test/share-command-deps-probe-argv.test.ts @@ -7,33 +7,38 @@ import { afterEach, describe, expect, it } from "vitest"; const require = createRequire(import.meta.url); const requireCache: Record = require.cache as any; -// Regression: `nemoclaw share mount` was passing the sandbox name as a bare -// positional to `openshell sandbox exec`, so OpenShell treated it as the -// command to run and the probe always returned a non-zero exit code even -// when `/sandbox` existed. The convention in this repo is to select the -// target sandbox with `-n` (or `--name`). +// Regression: `nemoclaw share mount` once passed the sandbox name as a bare +// positional to the sandbox exec path, so the path probe always returned a +// non-zero exit code even when `/sandbox` existed. The gRPC transport keeps +// the sandbox identity out of the command argv entirely. // See #3889 and #3954. describe("buildShareCommandDeps().checkSandboxPathExists probe argv", () => { afterEach(() => { - const openshellRuntimePath = require.resolve("../dist/lib/adapters/openshell/runtime"); + const grpcPath = require.resolve("../dist/lib/adapters/openshell/grpc"); const shareDepsPath = require.resolve("../dist/lib/share-command-deps"); - delete require.cache[openshellRuntimePath]; + delete require.cache[grpcPath]; delete require.cache[shareDepsPath]; }); - it("targets the sandbox with `-n ` so it is not parsed as the command", () => { - const openshellRuntimePath = require.resolve("../dist/lib/adapters/openshell/runtime"); + it("targets the sandbox name through the gRPC exec adapter", () => { + const grpcPath = require.resolve("../dist/lib/adapters/openshell/grpc"); const shareDepsPath = require.resolve("../dist/lib/share-command-deps"); - let recordedArgs: readonly string[] | undefined; - requireCache[openshellRuntimePath] = { - id: openshellRuntimePath, - filename: openshellRuntimePath, + let recordedCall: + | { sandboxName: string; argv: readonly string[]; opts: { timeoutMs?: number } } + | undefined; + requireCache[grpcPath] = { + id: grpcPath, + filename: grpcPath, loaded: true, exports: { - captureOpenshell: (args: readonly string[]) => { - recordedArgs = args; - return { status: 0, output: "" }; + execTextSync: ( + sandboxName: string, + argv: readonly string[], + opts: { timeoutMs?: number }, + ) => { + recordedCall = { sandboxName, argv, opts }; + return { status: 0, stdout: "", stderr: "" }; }, }, } as any; @@ -44,28 +49,21 @@ describe("buildShareCommandDeps().checkSandboxPathExists probe argv", () => { const exists = deps.checkSandboxPathExists("prachi-sbox", "/sandbox"); expect(exists).toBe(true); - expect(recordedArgs).toEqual([ - "sandbox", - "exec", - "-n", - "prachi-sbox", - "--", - "test", - "-e", - "/sandbox", - ]); + expect(recordedCall?.sandboxName).toBe("prachi-sbox"); + expect(recordedCall?.argv).toEqual(["test", "-e", "/sandbox"]); + expect(recordedCall?.opts.timeoutMs).toBeGreaterThan(0); }); it("reports the path as missing when the probe exits non-zero", () => { - const openshellRuntimePath = require.resolve("../dist/lib/adapters/openshell/runtime"); + const grpcPath = require.resolve("../dist/lib/adapters/openshell/grpc"); const shareDepsPath = require.resolve("../dist/lib/share-command-deps"); - requireCache[openshellRuntimePath] = { - id: openshellRuntimePath, - filename: openshellRuntimePath, + requireCache[grpcPath] = { + id: grpcPath, + filename: grpcPath, loaded: true, exports: { - captureOpenshell: () => ({ status: 1, output: "" }), + execTextSync: () => ({ status: 1, stdout: "", stderr: "" }), }, } as any; delete require.cache[shareDepsPath]; diff --git a/test/share-command-remote-path.test.ts b/test/share-command-remote-path.test.ts index 1791d0e132..f2d6e9fe13 100644 --- a/test/share-command-remote-path.test.ts +++ b/test/share-command-remote-path.test.ts @@ -14,7 +14,6 @@ class ProcessExitError extends Error { function makeDeps(overrides: Partial = {}): ShareCommandDeps { return { - getSshConfig: () => ({ status: 0, output: "" }), ensureLive: async () => undefined, checkSandboxPathExists: () => true, colorGreen: "", diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 0d58de7a59..7cfab3b45b 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -21,6 +21,11 @@ const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snap-naming-")) process.env.HOME = TMP_HOME; const REPO_ROOT = path.join(import.meta.dirname, ".."); +const GRPC_TEST_ENV_KEYS = [ + "NEMOCLAW_GRPC_TEST_TRANSPORT", + "NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH", + "NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN", +] as const; type BackupScalar = string | number | boolean | null | undefined; type BackupValue = BackupScalar | BackupManifestOverrides | BackupValue[]; @@ -81,13 +86,27 @@ afterAll(() => { } else { process.env.HOME = ORIGINAL_HOME; } + clearFakeGrpcTransport(); fs.rmSync(TMP_HOME, { recursive: true, force: true }); }); beforeEach(() => { + clearFakeGrpcTransport(); fs.rmSync(BACKUPS_ROOT, { recursive: true, force: true }); }); +function enableFakeGrpcTransportViaLegacySsh(binDir: string): void { + process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; + process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; + process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = path.join(binDir, "ssh"); +} + +function clearFakeGrpcTransport(): void { + for (const key of GRPC_TEST_ENV_KEYS) { + delete process.env[key]; + } +} + function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { mode: 0o755 }); } @@ -113,6 +132,7 @@ function writeOpenClawRegistry(sandboxName: string): void { } function writeFakeOpenshell(binDir: string): string { + enableFakeGrpcTransportViaLegacySsh(binDir); const openshell = path.join(binDir, "openshell"); writeExecutable( openshell, @@ -1270,6 +1290,7 @@ process.exit(0); process.env.NEMOCLAW_OPENSHELL_BIN = openshell; process.env.PATH = `${binDir}:${oldPath || ""}`; + enableFakeGrpcTransportViaLegacySsh(binDir); const backup = sandboxState.backupSandboxState("hermes", { name: "hermes-state" }); expect(backup.success).toBe(true); From e8da94ee088c20c684ca8ef074c4a25218dd4a65 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:29:59 -0700 Subject: [PATCH 02/26] Stabilize gRPC sandbox test transport --- .github/workflows/e2e-script.yaml | 6 + ci/env-var-doc-allowlist.json | 12 ++ src/lib/actions/sandbox/process-recovery.ts | 3 + .../openshell/forward-bridge-state.ts | 28 ++++- src/lib/adapters/openshell/sync-runner.ts | 5 +- src/lib/onboard/dashboard.ts | 5 +- test/cli.test.ts | 112 ++++++++++++------ test/helpers/grpc-fake-ssh.cjs | 60 ++++++++++ test/nemoclaw-cli-recovery.test.ts | 9 ++ test/onboard-dashboard.test.ts | 93 ++++++++++----- test/recover-port-forward.test.ts | 111 +++++++++-------- test/snapshot-gateway-guard.test.ts | 25 +++- 12 files changed, 340 insertions(+), 129 deletions(-) create mode 100755 test/helpers/grpc-fake-ssh.cjs diff --git a/.github/workflows/e2e-script.yaml b/.github/workflows/e2e-script.yaml index db07747be0..2f427c4267 100644 --- a/.github/workflows/e2e-script.yaml +++ b/.github/workflows/e2e-script.yaml @@ -125,6 +125,12 @@ jobs: print("::error::env_json must be a JSON object", file=sys.stderr) sys.exit(1) + # Nightly runners commonly lack CAP_SETPCAP inside OpenShell sandboxes. + # The sandbox image remains fail-closed by default; E2E jobs opt in so + # they can validate product behavior while gateway-isolation covers the + # fail-closed residual-capability posture. + values.setdefault("NEMOCLAW_ALLOW_RESIDUAL_CAPS", "1") + name_pattern = re.compile(r"^[A-Z_][A-Z0-9_]*$") reserved_prefixes = ("ACTIONS_", "GITHUB_", "INPUT_", "RUNNER_") reserved_names = {"CI", "HOME", "PATH", "PWD", "SHELL"} diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index c04df7defc..307d915280 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -31,6 +31,18 @@ "name": "NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE", "reason": "Internal installer sentinel exported only during OpenShell gateway replacement so onboard restores the pre-upgrade sandbox backup. Not user-facing." }, + { + "name": "NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN", + "reason": "Vitest-only compatibility shim used by the OpenShell gRPC sync runner to drive existing hermetic fake OpenShell fixtures. Never user-set in production." + }, + { + "name": "NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH", + "reason": "Vitest-only sentinel that enables the OpenShell gRPC sync runner's legacy fake transport while CLI tests migrate away from SSH-shaped fixtures. Never user-set in production." + }, + { + "name": "NEMOCLAW_GRPC_TEST_TRANSPORT", + "reason": "Vitest-only sentinel that prevents hermetic CLI tests from loading real OpenShell gateway metadata and routes sandbox exec calls through test fixtures. Never user-set in production." + }, { "name": "NEMOCLAW_TEST_NO_SLEEP", "reason": "Test sentinel that bypasses real-time sleep() calls in onboard inference probes. Set to '1' only by Vitest tests; never user-set." diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 82e76bfc1b..4501c6e131 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -120,6 +120,9 @@ async function executeSandboxExecCommandForStatus( sandboxName: string, command: string, ): Promise { + if (process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1") { + return executeSandboxExecCommand(sandboxName, command, OPENSHELL_PROBE_TIMEOUT_MS); + } const markedCommand = `printf '%s\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; const client = createSandboxGrpcClient(); try { diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index 06625595b8..9f7a1ad6dd 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -98,6 +98,7 @@ export function getForwardState(sandboxName: string, port: number | string): San } export function isPidAlive(pid: number): boolean { + if (isTestForwardPid(pid)) return true; if (!Number.isInteger(pid) || pid <= 0) return false; try { process.kill(pid, 0); @@ -109,7 +110,7 @@ export function isPidAlive(pid: number): boolean { export function stopForwardBridge(sandboxName: string, port: number | string): boolean { const state = readStateFile(statePath(sandboxName, port)); - if (state && isPidAlive(state.pid)) { + if (state && isPidAlive(state.pid) && !isTestForwardPid(state.pid)) { try { process.kill(state.pid, "SIGTERM"); } catch { @@ -146,6 +147,18 @@ function runnerCommand(): { command: string; args: string[] } { throw new Error("OpenShell gRPC forward bridge runner is not available. Run `npm run build:cli` first."); } +function useTestForwardBridge(): boolean { + return process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1" || process.env.VITEST_WORKER_ID !== undefined; +} + +function testForwardPid(): number { + return 0; +} + +function isTestForwardPid(pid: number): boolean { + return useTestForwardBridge() && pid === testForwardPid(); +} + export function startForwardBridgeDetached( sandboxName: string, options: ForwardBridgeStartOptions, @@ -154,6 +167,19 @@ export function startForwardBridgeDetached( const targetHost = options.targetHost || "127.0.0.1"; const timeoutMs = options.timeoutMs ?? 30_000; stopForwardBridge(sandboxName, options.port); + if (useTestForwardBridge()) { + const state: SandboxForwardState = { + sandboxName, + bind, + port: options.port, + targetHost, + targetPort: options.targetPort, + pid: testForwardPid(), + startedAt: new Date().toISOString(), + }; + writeForwardState(state); + return { ok: true, state, diagnostic: "" }; + } const { command, args } = runnerCommand(); ensureStateDir(); const diagnosticPath = path.join( diff --git a/src/lib/adapters/openshell/sync-runner.ts b/src/lib/adapters/openshell/sync-runner.ts index abbc42c4fd..c6b082dca4 100644 --- a/src/lib/adapters/openshell/sync-runner.ts +++ b/src/lib/adapters/openshell/sync-runner.ts @@ -44,7 +44,10 @@ function runLegacyFakeSshTransport( const sshBin = process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN || "ssh"; const remoteCommand = legacyFakeRemoteCommand(request.argv); - const result = spawnSync(sshBin, [`openshell-${request.sandboxName}`, remoteCommand], { + const helperArgs = [`openshell-${request.sandboxName}`, remoteCommand]; + const helperCommand = /\.(?:c|m)?js$/i.test(sshBin) ? process.execPath : sshBin; + const helperArgv = helperCommand === process.execPath ? [sshBin, ...helperArgs] : helperArgs; + const result = spawnSync(helperCommand, helperArgv, { input, timeout: request.opts?.timeoutMs && request.opts.timeoutMs > 0 diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 751d32c57f..efd5d2e4d3 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -41,6 +41,7 @@ export interface OnboardDashboardDeps { isWsl(): boolean; redact(value: unknown): string; sleep(seconds: number): void; + fetchGatewayAuthTokenFromSandbox?: (sandboxName: string) => string | null; printAgentDashboardUi( sandboxName: string, token: string | null, @@ -335,7 +336,9 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const showNim = shouldShowNimLine(nimContainer, nimStat.running); const nimLabel = nimStat.running ? "running" : "not running"; const providerLabel = deps.getProviderLabel(provider); - const token = fetchGatewayAuthTokenFromSandbox(sandboxName); + const token = (deps.fetchGatewayAuthTokenFromSandbox ?? fetchGatewayAuthTokenFromSandbox)( + sandboxName, + ); const chatUiUrl = process.env.CHAT_UI_URL || `http://127.0.0.1:${CONTROL_UI_PORT}`; const chain = buildChain({ chatUiUrl, diff --git a/test/cli.test.ts b/test/cli.test.ts index 5499e61f84..8fdd985ca5 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -12,6 +12,7 @@ import { execTimeout, testTimeout, testTimeoutOptions } from "./helpers/timeouts const CLI = path.join(import.meta.dirname, "..", "bin", "nemoclaw.js"); const HERMES_CLI = path.join(import.meta.dirname, "..", "bin", "nemohermes.js"); +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); const PARSER_EXIT_CODE = 2; type CliRunResult = { @@ -93,6 +94,9 @@ function runWithEnv( HOME: "/tmp/nemoclaw-cli-test-" + Date.now(), NEMOCLAW_HEALTH_POLL_COUNT: "1", NEMOCLAW_HEALTH_POLL_INTERVAL: "0", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, ...env, }, }); @@ -2638,9 +2642,9 @@ describe("CLI dispatch", () => { const log = fs.readFileSync(markerFile, "utf8"); expect(r.code).toBe(0); expect(log).toContain( - 'sandbox exec -n alpha -- sh -c tail -n 200 /tmp/gateway.log 2>/dev/null | grep -cE "getUpdates conflict|409[[:space:]:]+Conflict" || true', + 'sandbox exec --name alpha -- sh -c tail -n 200 /tmp/gateway.log 2>/dev/null | grep -cE "getUpdates conflict|409[[:space:]:]+Conflict" || true', ); - expect(log).toContain("sandbox exec -n alpha -- sh -c tail -n 10 /tmp/gateway.log 2>/dev/null"); + expect(log).toContain("sandbox exec --name alpha -- sh -c tail -n 10 /tmp/gateway.log 2>/dev/null"); expect(log).not.toContain("sandbox exec alpha sh -c"); }); @@ -3569,7 +3573,7 @@ describe("CLI dispatch", () => { expect(fs.existsSync(sshMarkerFile)).toBe(false); }); - it("connect --probe-only falls back to SSH when sandbox exec never starts", testTimeoutOptions(15_000), () => { + it("connect --probe-only does not fall back to SSH when sandbox exec never starts", testTimeoutOptions(15_000), () => { const home = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-exec-fallback-"), ); @@ -3637,18 +3641,16 @@ describe("CLI dispatch", () => { PATH: `${localBin}:${process.env.PATH || ""}`, }); - expect(r.code).toBe(0); - expect(r.out).toContain("Probe complete: recovered OpenClaw gateway"); + expect(r.code).toBe(1); + expect(r.out).toContain("Probe failed: could not inspect the OpenClaw gateway inside sandbox 'alpha'."); const openshellLog = fs.readFileSync(openshellCalls, "utf8"); - const sshLog = fs.readFileSync(sshCalls, "utf8"); expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); - expect(openshellLog).toContain("sandbox ssh-config alpha"); + expect(openshellLog).not.toContain("sandbox ssh-config alpha"); expect(openshellLog).not.toContain("sandbox connect"); - expect(sshLog).toContain('OPENCLAW="$(command -v openclaw)"'); - expect(sshLog).not.toMatch(/(^|\s)-tt?(\s|$)/); + expect(fs.existsSync(sshCalls)).toBe(false); }); - it("connect --probe-only falls back to SSH when sandbox exec times out after starting", () => { + it("connect --probe-only does not fall back to SSH when sandbox exec cannot prove health", () => { const home = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-exec-timeout-"), ); @@ -3676,7 +3678,6 @@ describe("CLI dispatch", () => { "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - " sleep 1", " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ] && [ "$3" = "alpha" ]; then', @@ -3714,19 +3715,17 @@ describe("CLI dispatch", () => { const r = runWithEnv("alpha connect --probe-only", { HOME: home, PATH: `${localBin}:${process.env.PATH || ""}`, - NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS: "50", }); - expect(r.code).toBe(0); - expect(r.out).toContain("Probe complete: recovered OpenClaw gateway"); + expect(r.code).toBe(1); + expect(r.out).toContain("Probe failed: could not inspect the OpenClaw gateway inside sandbox 'alpha'."); const openshellLog = fs.readFileSync(openshellCalls, "utf8"); - const sshLog = fs.readFileSync(sshCalls, "utf8"); expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); - expect(openshellLog).toContain("sandbox ssh-config alpha"); - expect(sshLog).toContain('OPENCLAW="$(command -v openclaw)"'); + expect(openshellLog).not.toContain("sandbox ssh-config alpha"); + expect(fs.existsSync(sshCalls)).toBe(false); }); - it("recovers non-OpenClaw agents over SSH instead of root sandbox exec", () => { + it("recovers non-OpenClaw agents over gRPC sandbox exec", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-agent-")); const localBin = path.join(home, "bin"); const openshellCalls = path.join(home, "openshell-calls"); @@ -3753,16 +3752,16 @@ describe("CLI dispatch", () => { "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', ' cmd="$8"', + ' if [[ "$cmd" == *"HERMES_HOME=/sandbox/.hermes"* || "$cmd" == *"AGENT_BIN="* ]]; then', + ' echo recovered > "$state_file"', + " echo 'GATEWAY_PID=789'", + " exit 0", + " fi", ' if [[ "$cmd" == *"curl -so"* ]]; then', " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', " exit 0", " fi", - ' if [[ "$cmd" == *"HERMES_HOME=/sandbox/.hermes"* || "$cmd" == *"AGENT_BIN="* ]]; then', - " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - " echo UNEXPECTED_ROOT_EXEC_RECOVERY", - " exit 1", - " fi", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ] && [ "$3" = "alpha" ]; then', " echo 'Host openshell-alpha'", @@ -3804,15 +3803,12 @@ describe("CLI dispatch", () => { expect(r.code).toBe(0); expect(r.out).toContain("Probe complete: recovered Hermes Agent gateway"); const openshellLog = fs.readFileSync(openshellCalls, "utf8"); - const sshLog = fs.readFileSync(sshCalls, "utf8"); expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); - expect(openshellLog).toContain("sandbox ssh-config alpha"); - expect(openshellLog).not.toContain("HERMES_HOME=/sandbox/.hermes"); - expect(openshellLog).not.toContain("AGENT_BIN="); + expect(openshellLog).not.toContain("sandbox ssh-config alpha"); + expect(openshellLog).toContain("HERMES_HOME=/sandbox/.hermes"); + expect(openshellLog).toContain("AGENT_BIN="); expect(openshellLog).not.toContain("sandbox connect"); - expect(sshLog).toContain("HERMES_HOME=/sandbox/.hermes"); - expect(sshLog).toContain("AGENT_BIN='/usr/local/bin/hermes'"); - expect(sshLog).not.toMatch(/(^|\s)-tt?(\s|$)/); + expect(fs.existsSync(sshCalls)).toBe(false); }); it("waits for sandbox readiness before connecting", () => { @@ -4849,14 +4845,8 @@ describe("CLI dispatch", () => { "#!/usr/bin/env bash", `state_file=${JSON.stringify(stateFile)}`, 'count=$(cat "$state_file" 2>/dev/null || echo 0)', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " count=$((count + 1))", - ' echo "$count" > "$state_file"', - ' if [ "$count" -eq 1 ]; then', - " echo 'Error: transport error: Connection refused' >&2", - " exit 1", - " fi", - " echo 'Sandbox: alpha'", + 'if [ "$1" = "--version" ]; then', + " echo 'openshell 0.0.44'", " exit 0", "fi", 'if [ "$1" = "status" ]; then', @@ -4866,12 +4856,22 @@ describe("CLI dispatch", () => { " echo ' Status: Connected'", " exit 0", "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', + 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', " echo 'Gateway Info'", " echo", " echo ' Gateway: nemoclaw'", " exit 0", "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + " count=$((count + 1))", + ' echo "$count" > "$state_file"', + ' if [ "$count" -eq 1 ]; then', + " echo 'Error: transport error: Connection refused' >&2", + " exit 1", + " fi", + " echo 'Sandbox: alpha'", + " exit 0", + "fi", "exit 0", ].join("\n"), { mode: 0o755 }, @@ -4913,6 +4913,23 @@ describe("CLI dispatch", () => { path.join(localBin, "openshell"), [ "#!/usr/bin/env bash", + 'if [ "$1" = "--version" ]; then', + " echo 'openshell 0.0.44'", + " exit 0", + "fi", + 'if [ "$1" = "status" ]; then', + " echo 'Server Status'", + " echo", + " echo ' Gateway: nemoclaw'", + " echo ' Status: Connected'", + " exit 0", + "fi", + 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', + " echo 'Gateway Info'", + " echo", + " echo ' Gateway: nemoclaw'", + " exit 0", + "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', " echo 'Sandbox: alpha'", " exit 0", @@ -5034,6 +5051,23 @@ describe("CLI dispatch", () => { [ "#!/usr/bin/env bash", `printf '%s\\n' "$*" >> ${JSON.stringify(markerFile)}`, + 'if [ "$1" = "--version" ]; then', + " echo 'openshell 0.0.44'", + " exit 0", + "fi", + 'if [ "$1" = "status" ]; then', + " echo 'Server Status'", + " echo", + " echo ' Gateway: nemoclaw'", + " echo ' Status: Connected'", + " exit 0", + "fi", + 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', + " echo 'Gateway Info'", + " echo", + " echo ' Gateway: nemoclaw'", + " exit 0", + "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', " echo 'Sandbox:'", " echo", diff --git a/test/helpers/grpc-fake-ssh.cjs b/test/helpers/grpc-fake-ssh.cjs new file mode 100755 index 0000000000..e8278ce679 --- /dev/null +++ b/test/helpers/grpc-fake-ssh.cjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const path = require("node:path"); + +function findFakeOpenshell() { + const homeEntries = process.env.HOME + ? [path.join(process.env.HOME, "bin"), path.join(process.env.HOME, ".local", "bin")] + : []; + const pathEntries = [ + ...homeEntries, + ...(process.env.PATH || "").split(path.delimiter).filter(Boolean), + ]; + for (const dir of pathEntries) { + const candidate = path.join(dir, "openshell"); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return null; +} + +const host = process.argv[2] || ""; +const remoteCommand = process.argv.slice(3).join(" "); +const match = /^openshell-(.+)$/.exec(host); + +if (!match || !remoteCommand) { + process.stderr.write("usage: grpc-fake-ssh openshell- \n"); + process.exit(64); +} + +const openshell = findFakeOpenshell(); +const home = process.env.HOME ? path.resolve(process.env.HOME) : ""; +if (!openshell || !home || !path.resolve(openshell).startsWith(`${home}${path.sep}`)) { + process.stderr.write("grpc fake transport could not find the hermetic fake openshell under HOME\n"); + process.exit(127); +} + +const result = spawnSync( + openshell, + ["sandbox", "exec", "--name", match[1], "--", "sh", "-c", remoteCommand], + { + input: fs.readFileSync(0), + stdio: ["pipe", "pipe", "pipe"], + env: process.env, + maxBuffer: 256 * 1024 * 1024, + }, +); + +if (result.error) { + process.stderr.write(`${result.error.message}\n`); + process.exit(1); +} + +process.stdout.write(result.stdout || ""); +process.stderr.write(result.stderr || ""); +process.exit(result.status ?? 1); diff --git a/test/nemoclaw-cli-recovery.test.ts b/test/nemoclaw-cli-recovery.test.ts index 8038f27d43..eb498e7f2d 100644 --- a/test/nemoclaw-cli-recovery.test.ts +++ b/test/nemoclaw-cli-recovery.test.ts @@ -15,6 +15,7 @@ describe("nemoclaw CLI runtime recovery", () => { testTimeoutOptions(), () => { const repoRoot = path.join(import.meta.dirname, ".."); + const grpcFakeSsh = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-recovery-")); const homeLocalBin = path.join(tmpDir, ".local", "bin"); const stateDir = path.join(tmpDir, "state"); @@ -83,6 +84,11 @@ if (args[0] === "sandbox" && args[1] === "get" && args[2] === "my-assistant") { process.exit(0); } +if (args[0] === "sandbox" && args[1] === "exec" && args[2] === "--name" && args[3] === "my-assistant") { + process.stdout.write("__NEMOCLAW_SANDBOX_EXEC_STARTED__\\nRUNNING\\n"); + process.exit(0); +} + if (args[0] === "logs") { process.exit(0); } @@ -101,6 +107,9 @@ process.exit(0); env: { ...process.env, HOME: tmpDir, + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: grpcFakeSsh, PATH: "/usr/bin:/bin", }, }, diff --git a/test/onboard-dashboard.test.ts b/test/onboard-dashboard.test.ts index 034d5e3a6e..bb4ca58e13 100644 --- a/test/onboard-dashboard.test.ts +++ b/test/onboard-dashboard.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { @@ -15,6 +16,26 @@ const { getPortConflictServiceHints } = require("../dist/lib/onboard") as { const { createOnboardDashboardHelpers } = require("../dist/lib/onboard/dashboard") as { createOnboardDashboardHelpers: (deps: OnboardDashboardDeps) => OnboardDashboardHelpers; }; +const { listForwardStates, writeForwardState } = require("../dist/lib/adapters/openshell/forward-bridge-state") as { + listForwardStates: () => Array<{ + sandboxName: string; + bind: string; + port: number; + targetHost: string; + targetPort: number; + pid: number; + startedAt: string; + }>; + writeForwardState: (state: { + sandboxName: string; + bind: string; + port: number; + targetHost: string; + targetPort: number; + pid: number; + startedAt: string; + }) => void; +}; describe("onboard dashboard helpers", () => { it("prints platform-appropriate service hints for port conflicts", () => { @@ -26,16 +47,31 @@ describe("onboard dashboard helpers", () => { }); it("uses sandbox-scoped forward stops for same-sandbox dashboard cleanup", () => { - const forwardList = - "SANDBOX BIND PORT PID STATUS\n" + - "my-sandbox 127.0.0.1 18789 12345 running\n" + - "my-sandbox 127.0.0.1 19000 12346 running"; + const originalHome = process.env.HOME; + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dashboard-forward-state-")); + process.env.HOME = home; + writeForwardState({ + sandboxName: "my-sandbox", + bind: "127.0.0.1", + port: 18789, + targetHost: "127.0.0.1", + targetPort: 18789, + pid: 0, + startedAt: new Date().toISOString(), + }); + writeForwardState({ + sandboxName: "my-sandbox", + bind: "127.0.0.1", + port: 19000, + targetHost: "127.0.0.1", + targetPort: 19000, + pid: 0, + startedAt: new Date().toISOString(), + }); const runOpenshell = vi.fn((_args: string[], _opts?: Record) => ({ status: 0, })); - const runCaptureOpenshell = vi.fn((args: string[], _opts?: Record) => - args.join(" ") === "forward list" ? forwardList : "", - ); + const runCaptureOpenshell = vi.fn(() => ""); const helpers = createOnboardDashboardHelpers({ runOpenshell, runCaptureOpenshell, @@ -50,37 +86,29 @@ describe("onboard dashboard helpers", () => { printAgentDashboardUi: vi.fn(), }); - expect(helpers.ensureDashboardForward("my-sandbox", "http://127.0.0.1:18789")).toBe(18789); - - const stopArgs = runOpenshell.mock.calls.map(([args]) => args); - expect(stopArgs).toContainEqual(["forward", "stop", "18789", "my-sandbox"]); - expect(stopArgs).toContainEqual(["forward", "stop", "19000", "my-sandbox"]); - expect( - stopArgs.some( - (args) => - Array.isArray(args) && - args[0] === "forward" && - args[1] === "stop" && - args.length === 3, - ), - ).toBe(false); + try { + expect(helpers.ensureDashboardForward("my-sandbox", "http://127.0.0.1:18789")).toBe(18789); + expect(runOpenshell).not.toHaveBeenCalledWith( + expect.arrayContaining(["forward", "stop"]), + expect.anything(), + ); + expect(listForwardStates()).toMatchObject([ + { + sandboxName: "my-sandbox", + port: 18789, + targetPort: 18789, + }, + ]); + } finally { + process.env.HOME = originalHome; + } }); it("prints the dashboard-url command instead of raw gateway-token guidance", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const nimStatus = vi.fn(() => ({ running: false, container: "nemoclaw-nim-test" })); const shouldShowNimLine = vi.fn(() => false); - const runOpenshell = vi.fn((args: string[], _opts?: Record) => { - if (args.join(" ").startsWith("sandbox download ")) { - const destDir = args[4]; - fs.mkdirSync(destDir, { recursive: true }); - fs.writeFileSync( - path.join(destDir, "openclaw.json"), - JSON.stringify({ gateway: { auth: { token: "secret-token" } } }), - ); - } - return { status: 0 }; - }); + const runOpenshell = vi.fn(() => ({ status: 0 })); const helpers = createOnboardDashboardHelpers({ runOpenshell, runCaptureOpenshell: vi.fn(() => ""), @@ -95,6 +123,7 @@ describe("onboard dashboard helpers", () => { isWsl: () => false, redact: (value: unknown) => String(value), sleep: vi.fn(), + fetchGatewayAuthTokenFromSandbox: vi.fn(() => "secret-token"), printAgentDashboardUi: vi.fn(), }); diff --git a/test/recover-port-forward.test.ts b/test/recover-port-forward.test.ts index ef5b0203d3..82eefc002c 100644 --- a/test/recover-port-forward.test.ts +++ b/test/recover-port-forward.test.ts @@ -10,6 +10,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { execTimeout, testTimeoutOptions } from "./helpers/timeouts"; const tmpFixtures: string[] = []; +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); // Each fixture grabs a unique high port. Sharing port 18789 across tests // collides with real nemoclaw installs on the developer's machine: the @@ -28,16 +29,14 @@ afterEach(() => { interface Fixture { tmpDir: string; sandboxName: string; + port: string; invocationLog: string; } function setupFixture(opts: { sandboxName: string; gatewayProbe: "RUNNING" | "STOPPED"; - forwardListStatus: "running" | "dead" | "missing"; - /** When false, `forward start` exits 0 but the post-restart probe keeps - * reporting the original dead/missing state — models a failed restart. */ - forwardStartHeals?: boolean; + forwardListStatus: "running" | "dead" | "missing" | "occupied"; port?: string; }): Fixture { const sandboxName = opts.sandboxName; @@ -48,9 +47,11 @@ function setupFixture(opts: { const registryDir = path.join(tmpDir, ".nemoclaw"); const openshellPath = path.join(homeLocalBin, "openshell"); const invocationLog = path.join(tmpDir, "openshell-calls.log"); + const forwardDir = path.join(registryDir, "forwards"); fs.mkdirSync(homeLocalBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); + fs.mkdirSync(forwardDir, { recursive: true }); fs.writeFileSync( path.join(registryDir, "sandboxes.json"), @@ -70,19 +71,35 @@ function setupFixture(opts: { { mode: 0o600 }, ); - const initialForwardListBody = - opts.forwardListStatus === "missing" - ? "" - : `${sandboxName} 127.0.0.1 ${port} 12345 ${opts.forwardListStatus}\n`; - const recoveredForwardListBody = `${sandboxName} 127.0.0.1 ${port} 99999 running\n`; - const forwardStateFile = path.join(tmpDir, "forward-state"); - fs.writeFileSync(forwardStateFile, "initial"); - - // Fake openshell: emits the requested gateway-probe and forward-list - // shapes, swallows mutating subcommands (forward stop / forward start) - // while logging every invocation so the test can assert the order. The - // forward state flips to "running" after `forward start` to model the - // post-recovery probe. + const writeForwardState = ( + owner: string, + pid: number, + ) => { + fs.writeFileSync( + path.join(forwardDir, `${owner}-${port}.json`), + JSON.stringify({ + sandboxName: owner, + bind: "127.0.0.1", + port: Number(port), + targetHost: "127.0.0.1", + targetPort: Number(port), + pid, + startedAt: new Date().toISOString(), + }), + { mode: 0o600 }, + ); + }; + if (opts.forwardListStatus === "running") { + writeForwardState(sandboxName, 0); + } else if (opts.forwardListStatus === "dead") { + writeForwardState(sandboxName, 999999); + } else if (opts.forwardListStatus === "occupied") { + writeForwardState("other-sandbox", 0); + } + + // Fake openshell: emits the requested gateway-probe while logging every + // invocation so the test can assert that SSH-backed OpenShell forwards are + // not used by the gRPC bridge path. fs.writeFileSync( openshellPath, `#!${process.execPath} @@ -121,26 +138,6 @@ if (args[0] === "sandbox" && args[1] === "exec") { process.exit(0); } -if (args[0] === "forward" && args[1] === "list") { - const state = fs.readFileSync(${JSON.stringify(forwardStateFile)}, "utf-8"); - process.stdout.write(state === "running" - ? ${JSON.stringify(recoveredForwardListBody)} - : ${JSON.stringify(initialForwardListBody)}); - process.exit(0); -} - -if (args[0] === "forward" && args[1] === "start") { - if (${opts.forwardStartHeals === false ? "false" : "true"}) { - fs.writeFileSync(${JSON.stringify(forwardStateFile)}, "running"); - } - process.exit(0); -} - -if (args[0] === "forward") { - // forward stop swallowed; forward state untouched. - process.exit(0); -} - if (args[0] === "policy" && args[1] === "get") { process.exit(1); } @@ -157,7 +154,7 @@ process.exit(0); { mode: 0o755 }, ); - return { tmpDir, sandboxName, invocationLog }; + return { tmpDir, sandboxName, port, invocationLog }; } function runRecover(fixture: Fixture) { @@ -173,12 +170,22 @@ function runRecover(fixture: Fixture) { HOME: fixture.tmpDir, PATH: "/usr/bin:/bin", NEMOCLAW_NO_CONNECT_HINT: "1", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, }, timeout: execTimeout(15_000), }, ); } +function recoverFailureMessage(fixture: Fixture, result: ReturnType): string { + const calls = fs.existsSync(fixture.invocationLog) + ? fs.readFileSync(fixture.invocationLog, "utf-8") + : ""; + return `${result.stderr || ""}${result.stdout || ""}\n--- calls ---\n${calls}`; +} + describe("nemoclaw recover", () => { it( "re-establishes the dashboard port-forward when the gateway is alive but the forward is dead", @@ -190,7 +197,7 @@ describe("nemoclaw recover", () => { forwardListStatus: "dead", }); const result = runRecover(fixture); - expect(result.status).toBe(0); + expect(result.status, recoverFailureMessage(fixture, result)).toBe(0); const combined = (result.stdout || "") + (result.stderr || ""); expect(combined).toContain( @@ -198,31 +205,32 @@ describe("nemoclaw recover", () => { ); const calls = fs.readFileSync(fixture.invocationLog, "utf-8").split("\n"); - const stopIdx = calls.findIndex((l) => l.startsWith("forward stop ")); - const startIdx = calls.findIndex((l) => l.startsWith("forward start ")); - expect(stopIdx).toBeGreaterThanOrEqual(0); - expect(startIdx).toBeGreaterThan(stopIdx); + expect(calls.some((l) => l.startsWith("forward "))).toBe(false); }, ); it( - "reports a failure when forward start succeeds but the post-restart probe still shows dead", + "leaves an occupied dashboard port-forward unchanged", testTimeoutOptions(20_000), () => { const fixture = setupFixture({ sandboxName: "stuck-sandbox", gatewayProbe: "RUNNING", - forwardListStatus: "dead", - forwardStartHeals: false, + forwardListStatus: "occupied", }); const result = runRecover(fixture); - expect(result.status).toBe(0); + expect(result.status, recoverFailureMessage(fixture, result)).toBe(0); const combined = (result.stdout || "") + (result.stderr || ""); - // Probe wrapper falls back to the plain "is running" line; the success - // suffix must not appear because the forward never came back. expect(combined).toContain("gateway is running in 'stuck-sandbox'"); expect(combined).not.toContain("restored dashboard port forward"); + const occupiedState = JSON.parse( + fs.readFileSync( + path.join(fixture.tmpDir, ".nemoclaw", "forwards", `other-sandbox-${fixture.port}.json`), + "utf-8", + ), + ); + expect(occupiedState.sandboxName).toBe("other-sandbox"); }, ); @@ -236,7 +244,7 @@ describe("nemoclaw recover", () => { forwardListStatus: "running", }); const result = runRecover(fixture); - expect(result.status).toBe(0); + expect(result.status, recoverFailureMessage(fixture, result)).toBe(0); const combined = (result.stdout || "") + (result.stderr || ""); expect(combined).toContain("gateway is running in 'healthy-sandbox'"); @@ -244,8 +252,7 @@ describe("nemoclaw recover", () => { expect(combined).not.toContain("restored dashboard port forward"); const calls = fs.readFileSync(fixture.invocationLog, "utf-8").split("\n"); - expect(calls.some((l) => l.startsWith("forward stop "))).toBe(false); - expect(calls.some((l) => l.startsWith("forward start "))).toBe(false); + expect(calls.some((l) => l.startsWith("forward "))).toBe(false); }, ); }); diff --git a/test/snapshot-gateway-guard.test.ts b/test/snapshot-gateway-guard.test.ts index 0342734360..307921d752 100644 --- a/test/snapshot-gateway-guard.test.ts +++ b/test/snapshot-gateway-guard.test.ts @@ -14,6 +14,7 @@ import path from "node:path"; import { execTimeout } from "./helpers/timeouts"; const CLI = path.join(import.meta.dirname, "..", "bin", "nemoclaw.js"); +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); type CliRunResult = { code: number; out: string }; @@ -26,6 +27,9 @@ function runCli(args: string, env: Record = {}): Cli ...process.env, NEMOCLAW_HEALTH_POLL_COUNT: "1", NEMOCLAW_HEALTH_POLL_INTERVAL: "0", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, ...env, }, }); @@ -121,16 +125,29 @@ function makeHealthyVmGatewayEnv(prefix: string): Record { // VM-driver snapshots should trust gateway metadata, not the legacy cluster // container probe. writeExecutable(path.join(localBin, "openshell"), [ + 'printf "%s\\n" "$*" >> "$HOME/openshell-calls.log"', 'case "$1 $2" in', ' "gateway info") printf "Gateway Info\\n\\nGateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080/\\n"; exit 0 ;;', ' "sandbox list") printf "NAME STATUS\\nalpha Ready\\n"; exit 0 ;;', - ' "sandbox ssh-config") printf "Host openshell-alpha\\n HostName 127.0.0.1\\n User sandbox\\n"; exit 0 ;;', "esac", + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + ' printf "Sandbox:\\n\\n Id: abc\\n Name: alpha\\n Namespace: openshell\\n Phase: Ready\\n"', + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ' cmd="$8"', + ' if printf "%s" "$cmd" | grep -q "awk .!seen"; then', + " exit 0", + " fi", + ' if printf "%s" "$cmd" | grep -q "find "; then', + " exit 0", + " fi", + " exit 2", + "fi", 'if [ "$1" = "status" ]; then exit 0; fi', "exit 0", ]); - writeExecutable(path.join(localBin, "ssh"), ["exit 0"]); writeExecutable(path.join(localBin, "docker"), [ 'if [ "$1" = "inspect" ]; then echo "false"; exit 0; fi', "exit 0", @@ -162,7 +179,9 @@ describe("snapshot VM-driver gateway guard", () => { it("snapshot create accepts healthy macOS VM-driver gateways without legacy cluster container", () => { const env = makeHealthyVmGatewayEnv("nemoclaw-snap-vm-gw-create-"); const r = runCli("alpha snapshot create --name baseline", env); - expect(r.code).toBe(0); + const callsPath = path.join(env.HOME, "openshell-calls.log"); + const calls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf-8") : ""; + expect(r.code, `${r.out}\n--- calls ---\n${calls}`).toBe(0); expect(r.out).toContain("Snapshot v1 name=baseline created"); expect(r.out).not.toContain("Failed to query live sandbox state"); }); From cafb69e78131174df78270e8ba65415bc3c66003 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:35:31 -0700 Subject: [PATCH 03/26] Honor validation probe curl retry budgets --- src/lib/inference/onboard-probes.ts | 37 +++++++++++++++++++---------- test/wsl2-probe-timeout.test.ts | 11 +++++---- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index a86a1afb95..acb5609d05 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -203,6 +203,14 @@ function getProbeProcessTimeoutMs(args) { return (getCurlMaxTimeSeconds(args) + 5) * 1000; } +function runValidationCurlProbe(args) { + return runCurlProbe(args, { timeoutMs: getProbeProcessTimeoutMs(args) }); +} + +function runValidationStreamingEventProbe(args) { + return runStreamingEventProbe(args, { timeoutMs: getProbeProcessTimeoutMs(args) }); +} + // 429 = Too Many Requests; 502/503/504 = upstream gateway/availability flakes // (NVIDIA Endpoints and other hosted providers periodically emit these for // minutes at a time). All four are transient — retry with backoff before @@ -278,7 +286,7 @@ function probeResponsesToolCalling(endpointUrl, model, apiKey, options = {}) { useQueryParam && normalizedKey ? `${baseUrl}/responses?key=${encodeURIComponent(normalizedKey)}` : `${baseUrl}/responses`; - const result = runCurlProbe([ + const args = [ "-sS", ...getValidationProbeCurlArgs(), "-H", @@ -306,7 +314,8 @@ function probeResponsesToolCalling(endpointUrl, model, apiKey, options = {}) { ], }), url, - ]); + ]; + const result = runValidationCurlProbe(args); if (!result.ok) { return result; @@ -335,7 +344,7 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { ? `${baseUrl}/chat/completions?key=${encodeURIComponent(normalizedKey)}` : `${baseUrl}/chat/completions`; const timingArgs = options.timingArgs ?? getValidationProbeCurlArgs(); - const result = runCurlProbe([ + const args = [ "-sS", ...timingArgs, "-H", @@ -401,7 +410,8 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { temperature: 0, }), url, - ]); + ]; + const result = runValidationCurlProbe(args); if (!result.ok) { return result; @@ -506,7 +516,7 @@ function runChatCompletionsProbe({ authHeader, model, url, isWsl: isWslOverride timeoutMs: getProbeProcessTimeoutMs(args), }); } - return runCurlProbe(args); + return runValidationCurlProbe(args); } function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { @@ -557,11 +567,11 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { execute: () => probeResponsesToolCalling(endpointUrl, model, apiKey, { authMode: options.authMode }), } - : { + : { name: "Responses API", api: "openai-responses", - execute: () => - runCurlProbe([ + execute: () => { + const args = [ "-sS", ...getValidationProbeCurlArgs(), "-H", @@ -573,7 +583,9 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { input: "Reply with exactly: OK", }), appendKey("/responses"), - ]), + ]; + return runValidationCurlProbe(args); + }, }; const chatCompletionsProbe = { @@ -608,7 +620,7 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { // streaming mode. Only run for /responses probes on custom endpoints // where probeStreaming was requested. if (probe.api === "openai-responses" && options.probeStreaming === true) { - const streamResult = runStreamingEventProbe([ + const streamArgs = [ "-sS", ...getValidationProbeCurlArgs(), "-H", @@ -621,7 +633,8 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { stream: true, }), appendKey("/responses"), - ]); + ]; + const streamResult = runValidationStreamingEventProbe(streamArgs); if (!streamResult.ok && streamResult.missingEvents.length > 0) { // Backend responds but lacks required streaming events — fall back // to /chat/completions silently. @@ -717,7 +730,7 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { authMode: options.authMode, timingArgs: doubledArgs, }) - : runCurlProbe(buildRetryArgs()); + : runValidationCurlProbe(buildRetryArgs()); let retryResult = runRetryProbe(); if (retryResult.ok) { return { ok: true, api: "openai-completions", label: "Chat Completions API" }; diff --git a/test/wsl2-probe-timeout.test.ts b/test/wsl2-probe-timeout.test.ts index b28c3fa786..2a9a15abb8 100644 --- a/test/wsl2-probe-timeout.test.ts +++ b/test/wsl2-probe-timeout.test.ts @@ -71,11 +71,11 @@ describe("WSL2 inference verification timeouts (issue #987)", () => { const platform = require(platformPath); const originalRunCurlProbe = httpProbe.runCurlProbe; const originalIsWsl = platform.isWsl; - const calls: string[][] = []; + const calls: { args: string[]; opts?: { timeoutMs?: number } }[] = []; let index = 0; platform.isWsl = () => false; - httpProbe.runCurlProbe = (args: string[]) => { - calls.push(args); + httpProbe.runCurlProbe = (args: string[], opts?: { timeoutMs?: number }) => { + calls.push({ args, opts }); const status = statuses[index++] ?? 0; if (status === 0) { return { @@ -121,7 +121,7 @@ describe("WSL2 inference verification timeouts (issue #987)", () => { const { result, calls } = runProbeWithCurlStatuses([28, 28, 0]); expect(result.ok).toBe(true); expect(calls.length).toBe(3); - expect(calls[2]).toEqual( + expect(calls[2].args).toEqual( expect.arrayContaining(["--connect-timeout", "20", "--max-time", "30"]), ); }); @@ -214,9 +214,10 @@ describe("WSL2 inference verification timeouts (issue #987)", () => { it("doubles timeout values for the retry attempt", () => { const { calls } = runProbeWithCurlStatuses([28, 28, 0]); - expect(calls[2]).toEqual( + expect(calls[2].args).toEqual( expect.arrayContaining(["--connect-timeout", "20", "--max-time", "30"]), ); + expect(calls[2].opts?.timeoutMs).toBe(35_000); }); it("appends WSL2 hint when retry fails on WSL2", () => { From 5a34f1a115211c1059c53e54ea11a6ee652d900e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:40:10 -0700 Subject: [PATCH 04/26] Tolerate optional workspace dir misses in gRPC backups --- src/lib/state/sandbox.ts | 17 +++++++---- test/snapshot.test.ts | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 82b972a8df..63b3a98064 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1010,13 +1010,14 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // First, check which declared state dirs actually exist in the sandbox, // then additionally discover per-agent `workspace-*` directories produced // by multi-agent OpenClaw deployments (see issue #1260) so they get - // snapshotted alongside the manifest-declared dirs. `awk '!seen[$0]++'` - // dedupes while preserving order. + // snapshotted alongside the manifest-declared dirs. The dynamic + // workspace probe is optional; no matches must not make the whole dir + // check fail. const existCheckCmd = stateDirs .map((d) => `[ -d ${shellQuote(`${dir}/${d}`)} ] && printf '%s\\n' ${shellQuote(d)}`) .join("; "); - const workspaceGlobCmd = `for d in ${shellQuote(dir)}/workspace-*/; do [ -d "$d" ] && basename "$d"; done 2>/dev/null`; - const fullCheckCmd = `{ ${existCheckCmd}; ${workspaceGlobCmd}; } 2>/dev/null | awk '!seen[$0]++'`; + const workspaceGlobCmd = `for d in ${shellQuote(dir)}/workspace-*/; do [ -d "$d" ] || continue; name="\${d%/}"; printf '%s\\n' "\${name##*/}"; done`; + const fullCheckCmd = `{ ${existCheckCmd}; ${workspaceGlobCmd}; } 2>/dev/null || true`; _log(`Checking existing dirs via gRPC exec: ${fullCheckCmd.substring(0, 100)}...`); let existResult: ReturnType; try { @@ -1041,10 +1042,16 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = _log( `Dir check: exit=${existResult.status}, stdout=${(existResult.stdout || "").trim().substring(0, 200)}, stderr=${(existResult.stderr || "").trim().substring(0, 200)}`, ); + const seenExistingDirs = new Set(); const existingDirs = (existResult.stdout || "") .trim() .split("\n") - .filter((d) => d.length > 0); + .map((d) => d.trim()) + .filter((d) => { + if (d.length === 0 || seenExistingDirs.has(d)) return false; + seenExistingDirs.add(d); + return true; + }); _log( `Existing dirs in sandbox: [${existingDirs.join(",")}] (${existingDirs.length}/${stateDirs.length})`, ); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 7cfab3b45b..fd4f5767e0 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -504,6 +504,67 @@ process.exit(0); } }); + it("keeps optional workspace glob misses from failing the gRPC dir check", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-workspace-glob-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const openclawDir = path.join(fixture, "sandbox-root", ".openclaw"); + const existingDirs = ["agents", "workspace", "skills"]; + fs.mkdirSync(binDir, { recursive: true }); + for (const dirName of existingDirs) { + fs.mkdirSync(path.join(openclawDir, dirName), { recursive: true }); + } + + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const cmd = process.argv[process.argv.length - 1] || ""; +const existingDirs = ${JSON.stringify(existingDirs)}; +if (cmd.includes("[ -d ")) { + process.stdout.write(existingDirs.join("\\n") + "\\n"); + // Simulate the real regression: the optional workspace-* glob can leave a + // non-zero shell status even though declared dirs were printed. + process.exit(cmd.includes("|| true") ? 0 : 1); +} +if (cmd.includes("find ")) { + process.exit(0); +} +if (cmd.includes("tar -cf -")) { + const r = spawnSync("tar", ["-cf", "-", "-C", ${JSON.stringify(openclawDir)}, ...existingDirs], { + stdio: ["ignore", "pipe", "pipe"], + }); + if (r.stdout) fs.writeSync(1, r.stdout); + if (r.stderr) fs.writeSync(2, r.stderr); + process.exit(r.status || 0); +} +process.exit(0); +`, + ); + + writeOpenClawRegistry("alpha"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}:${oldPath || ""}`; + + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + expect(backup.failedDirs).toEqual([]); + expect(backup.backedUpDirs).toEqual(existingDirs); + } finally { + if (oldOpenshell === undefined) { + delete process.env.NEMOCLAW_OPENSHELL_BIN; + } else { + process.env.NEMOCLAW_OPENSHELL_BIN = oldOpenshell; + } + process.env.PATH = oldPath; + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + it("excludes tar-failed directories from the restorable manifest", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-partial-tar-")); const oldPath = process.env.PATH; From 7ea65a1a7da1d215954dc8cc1ce61b5af8de76b9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:50:41 -0700 Subject: [PATCH 05/26] Harden gRPC recovery and Hermes E2E probes --- src/lib/actions/sandbox/process-recovery.ts | 5 +- test/cli.test.ts | 4 +- .../skill/verify-sandbox-skill-via-agent.sh | 6 +- test/e2e/test-hermes-e2e.sh | 66 +++++++++++-------- 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 4501c6e131..e6d153fdaf 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -243,7 +243,10 @@ function recoverSandboxProcesses(sandboxName: string): boolean { !!(result && result.status === 0 && hasRecoveryMarker(result)); if (agentScript) { - return recoveredCommand(executeSandboxCommand(sandboxName, agentScript)); + const execResult = executeSandboxCommand(sandboxName, agentScript); + if (hasRecoveryMarker(execResult)) return true; + if (execResult !== null) return false; + return false; } const script = agentRuntime.buildOpenClawRecoveryScript(dashboardPort); diff --git a/test/cli.test.ts b/test/cli.test.ts index 8fdd985ca5..4985e03e28 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -3725,7 +3725,7 @@ describe("CLI dispatch", () => { expect(fs.existsSync(sshCalls)).toBe(false); }); - it("recovers non-OpenClaw agents over gRPC sandbox exec", () => { + it("recovers non-OpenClaw agents over gRPC sandbox exec when the recovery marker is present", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-agent-")); const localBin = path.join(home, "bin"); const openshellCalls = path.join(home, "openshell-calls"); @@ -3755,7 +3755,7 @@ describe("CLI dispatch", () => { ' if [[ "$cmd" == *"HERMES_HOME=/sandbox/.hermes"* || "$cmd" == *"AGENT_BIN="* ]]; then', ' echo recovered > "$state_file"', " echo 'GATEWAY_PID=789'", - " exit 0", + " exit 42", " fi", ' if [[ "$cmd" == *"curl -so"* ]]; then', " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", diff --git a/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh b/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh index a88e31bea6..8807212dca 100755 --- a/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh +++ b/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh @@ -59,6 +59,10 @@ command -v base64 >/dev/null 2>&1 || die "base64 not on PATH" prompt_b64=$(printf '%s' "$PROMPT" | base64 | tr -d '\n') nv_b64=$(printf '%s' "$NVIDIA_API_KEY" | base64 | tr -d '\n') +remote_env_prefix='[ -r /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh; ' +if [ "${NEMOCLAW_ALLOW_RESIDUAL_CAPS:-0}" = "1" ]; then + remote_env_prefix+='export NEMOCLAW_ALLOW_RESIDUAL_CAPS=1; ' +fi ssh_config="$(mktemp)" trap 'rm -f "$ssh_config"' EXIT @@ -75,7 +79,7 @@ _lock_rm="" if [ "${SKILL_VERIFY_NO_CLEAR_LOCK:-0}" != "1" ]; then _lock_rm="rm -f '/sandbox/.openclaw/agents/main/sessions/${SESSION_ID}.jsonl.lock' 2>/dev/null || true; " fi -remote_cmd="pm=\$(printf '%s' '${prompt_b64}' | base64 -d) || exit 1; nv=\$(printf '%s' '${nv_b64}' | base64 -d) || exit 1; export NVIDIA_API_KEY=\"\$nv\"; ${_lock_rm}${AGENT_LAUNCHER}openclaw agent --agent main --local -m \"\$pm\" --session-id '${SESSION_ID}'" +remote_cmd="${remote_env_prefix}pm=\$(printf '%s' '${prompt_b64}' | base64 -d) || exit 1; nv=\$(printf '%s' '${nv_b64}' | base64 -d) || exit 1; export NVIDIA_API_KEY=\"\$nv\"; ${_lock_rm}${AGENT_LAUNCHER}openclaw agent --agent main --local -m \"\$pm\" --session-id '${SESSION_ID}'" info "Running openclaw agent in sandbox '${SANDBOX_NAME}' (session ${SESSION_ID})..." diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index 6829d01e77..a76986942c 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -470,37 +470,49 @@ fi # agent runtime or openclaw's HTTP client. See NemoClaw #2490 for the # openclaw 4.9 SSRF regression that was invisible to assertions of this shape. info "[ROUTING] inference.local DNS + OpenShell proxy reachable from Hermes sandbox..." -ssh_config="$(mktemp)" +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" +sandbox_content="" sandbox_response="" - -if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then - # Use timeout if available (Linux, Homebrew), fall back to plain ssh - TIMEOUT_CMD="" - command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" - command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" - sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=10 \ - -o LogLevel=ERROR \ - "openshell-${SANDBOX_NAME}" \ - "curl -s --max-time 60 https://inference.local/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ - 2>&1) || true -fi -rm -f "$ssh_config" - -if [ -n "$sandbox_response" ]; then - sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true - if grep -qi "PONG" <<<"$sandbox_content"; then - pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG" - info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise the Hermes agent runtime or openclaw HTTP client)" +sandbox_detail="" +pong_ok=false +for pong_attempt in 1 2 3; do + ssh_config="$(mktemp)" + sandbox_response="" + if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; fi; curl -sS --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true + fi + rm -f "$ssh_config" + if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + sandbox_detail="${sandbox_content:-$sandbox_response}" else - fail "[ROUTING] inference.local: expected PONG, got: ${sandbox_content:0:200}" + sandbox_content="" + sandbox_detail="empty response" fi + if grep -qi "PONG" <<<"$sandbox_content"; then + pong_ok=true + break + fi + info "Sandbox inference attempt ${pong_attempt}/3: got '${sandbox_detail:0:120}'" + [ "$pong_attempt" -lt 3 ] && sleep 5 +done + +if $pong_ok; then + pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG" + info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise the Hermes agent runtime or openclaw HTTP client)" else - fail "[ROUTING] inference.local: no response from inference.local inside Hermes sandbox" + fail "[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_detail:0:200}" fi # ══════════════════════════════════════════════════════════════════ From 28548cf562a9d6bd03a5126fb9fb262da76e5c05 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:53:56 -0700 Subject: [PATCH 06/26] Harden launchable endpoint preflight --- test/e2e/test-launchable-smoke.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/e2e/test-launchable-smoke.sh b/test/e2e/test-launchable-smoke.sh index 0511b1565b..110b68f26d 100755 --- a/test/e2e/test-launchable-smoke.sh +++ b/test/e2e/test-launchable-smoke.sh @@ -184,8 +184,18 @@ else exit 1 fi -if curl -sf --max-time 10 https://integrate.api.nvidia.com/v1/models >/dev/null 2>&1; then - pass "Network access to integrate.api.nvidia.com" +models_http_code="000" +for attempt in 1 2 3; do + models_http_code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 \ + -H "Authorization: Bearer $NVIDIA_API_KEY" \ + https://integrate.api.nvidia.com/v1/models 2>/dev/null || printf '000') + if [ "$models_http_code" != "000" ]; then + break + fi + [ "$attempt" -lt 3 ] && sleep 5 +done +if [ "$models_http_code" != "000" ]; then + pass "Network access to integrate.api.nvidia.com (HTTP ${models_http_code})" else fail "Cannot reach integrate.api.nvidia.com" exit 1 From 6083811e81f6c342e59884aeaf570eb6e3a75ff8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:59:04 -0700 Subject: [PATCH 07/26] Revert "Harden launchable endpoint preflight" This reverts commit 28548cf562a9d6bd03a5126fb9fb262da76e5c05. --- test/e2e/test-launchable-smoke.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/e2e/test-launchable-smoke.sh b/test/e2e/test-launchable-smoke.sh index 110b68f26d..0511b1565b 100755 --- a/test/e2e/test-launchable-smoke.sh +++ b/test/e2e/test-launchable-smoke.sh @@ -184,18 +184,8 @@ else exit 1 fi -models_http_code="000" -for attempt in 1 2 3; do - models_http_code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 \ - -H "Authorization: Bearer $NVIDIA_API_KEY" \ - https://integrate.api.nvidia.com/v1/models 2>/dev/null || printf '000') - if [ "$models_http_code" != "000" ]; then - break - fi - [ "$attempt" -lt 3 ] && sleep 5 -done -if [ "$models_http_code" != "000" ]; then - pass "Network access to integrate.api.nvidia.com (HTTP ${models_http_code})" +if curl -sf --max-time 10 https://integrate.api.nvidia.com/v1/models >/dev/null 2>&1; then + pass "Network access to integrate.api.nvidia.com" else fail "Cannot reach integrate.api.nvidia.com" exit 1 From a1031c973d2d1efb23bb147a11adf1b2e3424da7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:59:12 -0700 Subject: [PATCH 08/26] Revert non-migration E2E probe changes --- .../skill/verify-sandbox-skill-via-agent.sh | 6 +- test/e2e/test-hermes-e2e.sh | 66 ++++++++----------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh b/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh index 8807212dca..a88e31bea6 100755 --- a/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh +++ b/test/e2e/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh @@ -59,10 +59,6 @@ command -v base64 >/dev/null 2>&1 || die "base64 not on PATH" prompt_b64=$(printf '%s' "$PROMPT" | base64 | tr -d '\n') nv_b64=$(printf '%s' "$NVIDIA_API_KEY" | base64 | tr -d '\n') -remote_env_prefix='[ -r /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh; ' -if [ "${NEMOCLAW_ALLOW_RESIDUAL_CAPS:-0}" = "1" ]; then - remote_env_prefix+='export NEMOCLAW_ALLOW_RESIDUAL_CAPS=1; ' -fi ssh_config="$(mktemp)" trap 'rm -f "$ssh_config"' EXIT @@ -79,7 +75,7 @@ _lock_rm="" if [ "${SKILL_VERIFY_NO_CLEAR_LOCK:-0}" != "1" ]; then _lock_rm="rm -f '/sandbox/.openclaw/agents/main/sessions/${SESSION_ID}.jsonl.lock' 2>/dev/null || true; " fi -remote_cmd="${remote_env_prefix}pm=\$(printf '%s' '${prompt_b64}' | base64 -d) || exit 1; nv=\$(printf '%s' '${nv_b64}' | base64 -d) || exit 1; export NVIDIA_API_KEY=\"\$nv\"; ${_lock_rm}${AGENT_LAUNCHER}openclaw agent --agent main --local -m \"\$pm\" --session-id '${SESSION_ID}'" +remote_cmd="pm=\$(printf '%s' '${prompt_b64}' | base64 -d) || exit 1; nv=\$(printf '%s' '${nv_b64}' | base64 -d) || exit 1; export NVIDIA_API_KEY=\"\$nv\"; ${_lock_rm}${AGENT_LAUNCHER}openclaw agent --agent main --local -m \"\$pm\" --session-id '${SESSION_ID}'" info "Running openclaw agent in sandbox '${SANDBOX_NAME}' (session ${SESSION_ID})..." diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index a76986942c..6829d01e77 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -470,49 +470,37 @@ fi # agent runtime or openclaw's HTTP client. See NemoClaw #2490 for the # openclaw 4.9 SSRF regression that was invisible to assertions of this shape. info "[ROUTING] inference.local DNS + OpenShell proxy reachable from Hermes sandbox..." -TIMEOUT_CMD="" -command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" -command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" -sandbox_content="" +ssh_config="$(mktemp)" sandbox_response="" -sandbox_detail="" -pong_ok=false -for pong_attempt in 1 2 3; do - ssh_config="$(mktemp)" - sandbox_response="" - if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then - sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=10 \ - -o LogLevel=ERROR \ - "openshell-${SANDBOX_NAME}" \ - "if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; fi; curl -sS --max-time 60 https://inference.local/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ - 2>&1) || true - fi - rm -f "$ssh_config" - if [ -n "$sandbox_response" ]; then - sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true - sandbox_detail="${sandbox_content:-$sandbox_response}" - else - sandbox_content="" - sandbox_detail="empty response" - fi + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + # Use timeout if available (Linux, Homebrew), fall back to plain ssh + TIMEOUT_CMD="" + command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" + command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" + sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true +fi +rm -f "$ssh_config" + +if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true if grep -qi "PONG" <<<"$sandbox_content"; then - pong_ok=true - break + pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG" + info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise the Hermes agent runtime or openclaw HTTP client)" + else + fail "[ROUTING] inference.local: expected PONG, got: ${sandbox_content:0:200}" fi - info "Sandbox inference attempt ${pong_attempt}/3: got '${sandbox_detail:0:120}'" - [ "$pong_attempt" -lt 3 ] && sleep 5 -done - -if $pong_ok; then - pass "[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG" - info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → NVIDIA Endpoints (does not exercise the Hermes agent runtime or openclaw HTTP client)" else - fail "[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_detail:0:200}" + fail "[ROUTING] inference.local: no response from inference.local inside Hermes sandbox" fi # ══════════════════════════════════════════════════════════════════ From 1bbb6faaac5a57a6e1a07fcffcb07974a7bcab8b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 10:59:15 -0700 Subject: [PATCH 09/26] Revert "Honor validation probe curl retry budgets" This reverts commit cafb69e78131174df78270e8ba65415bc3c66003. --- src/lib/inference/onboard-probes.ts | 37 ++++++++++------------------- test/wsl2-probe-timeout.test.ts | 11 ++++----- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index acb5609d05..a86a1afb95 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -203,14 +203,6 @@ function getProbeProcessTimeoutMs(args) { return (getCurlMaxTimeSeconds(args) + 5) * 1000; } -function runValidationCurlProbe(args) { - return runCurlProbe(args, { timeoutMs: getProbeProcessTimeoutMs(args) }); -} - -function runValidationStreamingEventProbe(args) { - return runStreamingEventProbe(args, { timeoutMs: getProbeProcessTimeoutMs(args) }); -} - // 429 = Too Many Requests; 502/503/504 = upstream gateway/availability flakes // (NVIDIA Endpoints and other hosted providers periodically emit these for // minutes at a time). All four are transient — retry with backoff before @@ -286,7 +278,7 @@ function probeResponsesToolCalling(endpointUrl, model, apiKey, options = {}) { useQueryParam && normalizedKey ? `${baseUrl}/responses?key=${encodeURIComponent(normalizedKey)}` : `${baseUrl}/responses`; - const args = [ + const result = runCurlProbe([ "-sS", ...getValidationProbeCurlArgs(), "-H", @@ -314,8 +306,7 @@ function probeResponsesToolCalling(endpointUrl, model, apiKey, options = {}) { ], }), url, - ]; - const result = runValidationCurlProbe(args); + ]); if (!result.ok) { return result; @@ -344,7 +335,7 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { ? `${baseUrl}/chat/completions?key=${encodeURIComponent(normalizedKey)}` : `${baseUrl}/chat/completions`; const timingArgs = options.timingArgs ?? getValidationProbeCurlArgs(); - const args = [ + const result = runCurlProbe([ "-sS", ...timingArgs, "-H", @@ -410,8 +401,7 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { temperature: 0, }), url, - ]; - const result = runValidationCurlProbe(args); + ]); if (!result.ok) { return result; @@ -516,7 +506,7 @@ function runChatCompletionsProbe({ authHeader, model, url, isWsl: isWslOverride timeoutMs: getProbeProcessTimeoutMs(args), }); } - return runValidationCurlProbe(args); + return runCurlProbe(args); } function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { @@ -567,11 +557,11 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { execute: () => probeResponsesToolCalling(endpointUrl, model, apiKey, { authMode: options.authMode }), } - : { + : { name: "Responses API", api: "openai-responses", - execute: () => { - const args = [ + execute: () => + runCurlProbe([ "-sS", ...getValidationProbeCurlArgs(), "-H", @@ -583,9 +573,7 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { input: "Reply with exactly: OK", }), appendKey("/responses"), - ]; - return runValidationCurlProbe(args); - }, + ]), }; const chatCompletionsProbe = { @@ -620,7 +608,7 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { // streaming mode. Only run for /responses probes on custom endpoints // where probeStreaming was requested. if (probe.api === "openai-responses" && options.probeStreaming === true) { - const streamArgs = [ + const streamResult = runStreamingEventProbe([ "-sS", ...getValidationProbeCurlArgs(), "-H", @@ -633,8 +621,7 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { stream: true, }), appendKey("/responses"), - ]; - const streamResult = runValidationStreamingEventProbe(streamArgs); + ]); if (!streamResult.ok && streamResult.missingEvents.length > 0) { // Backend responds but lacks required streaming events — fall back // to /chat/completions silently. @@ -730,7 +717,7 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { authMode: options.authMode, timingArgs: doubledArgs, }) - : runValidationCurlProbe(buildRetryArgs()); + : runCurlProbe(buildRetryArgs()); let retryResult = runRetryProbe(); if (retryResult.ok) { return { ok: true, api: "openai-completions", label: "Chat Completions API" }; diff --git a/test/wsl2-probe-timeout.test.ts b/test/wsl2-probe-timeout.test.ts index 2a9a15abb8..b28c3fa786 100644 --- a/test/wsl2-probe-timeout.test.ts +++ b/test/wsl2-probe-timeout.test.ts @@ -71,11 +71,11 @@ describe("WSL2 inference verification timeouts (issue #987)", () => { const platform = require(platformPath); const originalRunCurlProbe = httpProbe.runCurlProbe; const originalIsWsl = platform.isWsl; - const calls: { args: string[]; opts?: { timeoutMs?: number } }[] = []; + const calls: string[][] = []; let index = 0; platform.isWsl = () => false; - httpProbe.runCurlProbe = (args: string[], opts?: { timeoutMs?: number }) => { - calls.push({ args, opts }); + httpProbe.runCurlProbe = (args: string[]) => { + calls.push(args); const status = statuses[index++] ?? 0; if (status === 0) { return { @@ -121,7 +121,7 @@ describe("WSL2 inference verification timeouts (issue #987)", () => { const { result, calls } = runProbeWithCurlStatuses([28, 28, 0]); expect(result.ok).toBe(true); expect(calls.length).toBe(3); - expect(calls[2].args).toEqual( + expect(calls[2]).toEqual( expect.arrayContaining(["--connect-timeout", "20", "--max-time", "30"]), ); }); @@ -214,10 +214,9 @@ describe("WSL2 inference verification timeouts (issue #987)", () => { it("doubles timeout values for the retry attempt", () => { const { calls } = runProbeWithCurlStatuses([28, 28, 0]); - expect(calls[2].args).toEqual( + expect(calls[2]).toEqual( expect.arrayContaining(["--connect-timeout", "20", "--max-time", "30"]), ); - expect(calls[2].opts?.timeoutMs).toBe(35_000); }); it("appends WSL2 hint when retry fails on WSL2", () => { From 0d9c6a97d940433f97a26783a0ab9e1f8aa22a53 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 11:05:57 -0700 Subject: [PATCH 10/26] Address gRPC forward CodeQL findings --- src/lib/actions/sandbox/process-recovery.ts | 2 -- .../openshell/forward-bridge-state.ts | 21 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index e6d153fdaf..73c5d54c73 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -239,8 +239,6 @@ function recoverSandboxProcesses(sandboxName: string): boolean { result && (result.stdout.includes("GATEWAY_PID=") || result.stdout.includes("ALREADY_RUNNING")) ); - const recoveredCommand = (result: SandboxCommandResult | null) => - !!(result && result.status === 0 && hasRecoveryMarker(result)); if (agentScript) { const execResult = executeSandboxCommand(sandboxName, agentScript); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index 9f7a1ad6dd..cabe554fc2 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -159,6 +159,14 @@ function isTestForwardPid(pid: number): boolean { return useTestForwardBridge() && pid === testForwardPid(); } +function readOpenFileDescriptor(fd: number): string { + const stat = fs.fstatSync(fd); + if (stat.size <= 0) return ""; + const buffer = Buffer.alloc(Math.min(stat.size, 64 * 1024)); + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0); + return buffer.subarray(0, bytesRead).toString("utf-8"); +} + export function startForwardBridgeDetached( sandboxName: string, options: ForwardBridgeStartOptions, @@ -182,11 +190,13 @@ export function startForwardBridgeDetached( } const { command, args } = runnerCommand(); ensureStateDir(); + const safeSandbox = sandboxName.replace(/[^A-Za-z0-9._-]/g, "_"); + const diagnosticDir = fs.mkdtempSync(path.join(stateDir(), `${safeSandbox}-${options.port}-`)); const diagnosticPath = path.join( - stateDir(), - `${sandboxName.replace(/[^A-Za-z0-9._-]/g, "_")}-${options.port}.log`, + diagnosticDir, + "bridge.log", ); - const out = fs.openSync(diagnosticPath, "w", 0o600); + const out = fs.openSync(diagnosticPath, "w+", 0o600); const child = spawn( command, [ @@ -202,12 +212,12 @@ export function startForwardBridgeDetached( { detached: true, stdio: ["ignore", out, out], env: process.env }, ); child.unref(); - fs.closeSync(out); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const state = getForwardState(sandboxName, options.port); if (state?.pid === child.pid) { + fs.closeSync(out); return { ok: true, state: state ?? undefined, diagnostic: "" }; } sleepMs(250); @@ -215,10 +225,11 @@ export function startForwardBridgeDetached( let diagnostic = ""; try { - diagnostic = fs.readFileSync(diagnosticPath, "utf-8").trim(); + diagnostic = readOpenFileDescriptor(out).trim(); } catch { /* ignore */ } + fs.closeSync(out); try { if (child.pid) process.kill(child.pid, "SIGTERM"); } catch { From e17f7905e9254bea3fdeb4de0972e3173182d000 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 11:44:17 -0700 Subject: [PATCH 11/26] test: update fixtures for gRPC sandbox transport --- src/lib/status-command-deps.test.ts | 43 +++++++- src/lib/tunnel/services-sandbox.test.ts | 112 +++++++++++++-------- test/helpers/grpc-fake-ssh.cjs | 8 +- test/onboard.test.ts | 127 ++++++++++++++++-------- test/repro-2201.test.ts | 3 + test/sandbox-connect-inference.test.ts | 5 + test/shellquote-sandbox.test.ts | 31 +++++- test/snapshot-gateway-guard.test.ts | 35 +++++-- 8 files changed, 271 insertions(+), 93 deletions(-) diff --git a/src/lib/status-command-deps.test.ts b/src/lib/status-command-deps.test.ts index 77563c2e34..34aa80751c 100644 --- a/src/lib/status-command-deps.test.ts +++ b/src/lib/status-command-deps.test.ts @@ -10,6 +10,7 @@ import path from "node:path"; const require = createRequire(import.meta.url); const { buildStatusCommandDeps } = require("../../dist/lib/status-command-deps.js") as typeof import("../../dist/lib/status-command-deps"); +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "..", "..", "test", "helpers", "grpc-fake-ssh.cjs"); function writeExecutable(target: string, body: string): void { fs.writeFileSync(target, body, { mode: 0o755 }); @@ -17,16 +18,31 @@ function writeExecutable(target: string, body: string): void { describe("buildStatusCommandDeps", () => { let previousOverride: string | undefined; + let previousHome: string | undefined; + let previousPath: string | undefined; + let previousGrpcTransport: string | undefined; + let previousGrpcLegacy: string | undefined; + let previousGrpcFakeSsh: string | undefined; let tmp: string; let callsFile: string; let openshell: string; beforeEach(() => { previousOverride = process.env.NEMOCLAW_OPENSHELL_BIN; + previousHome = process.env.HOME; + previousPath = process.env.PATH; + previousGrpcTransport = process.env.NEMOCLAW_GRPC_TEST_TRANSPORT; + previousGrpcLegacy = process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH; + previousGrpcFakeSsh = process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN; tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-status-deps-")); callsFile = path.join(tmp, "openshell.calls"); openshell = path.join(tmp, "openshell"); process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.HOME = tmp; + process.env.PATH = `${tmp}${path.delimiter}${previousPath || ""}`; + process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; + process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; + process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = GRPC_FAKE_SSH; }); afterEach(() => { @@ -35,6 +51,31 @@ describe("buildStatusCommandDeps", () => { } else { process.env.NEMOCLAW_OPENSHELL_BIN = previousOverride; } + if (previousHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = previousHome; + } + if (previousPath === undefined) { + delete process.env.PATH; + } else { + process.env.PATH = previousPath; + } + if (previousGrpcTransport === undefined) { + delete process.env.NEMOCLAW_GRPC_TEST_TRANSPORT; + } else { + process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = previousGrpcTransport; + } + if (previousGrpcLegacy === undefined) { + delete process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH; + } else { + process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = previousGrpcLegacy; + } + if (previousGrpcFakeSsh === undefined) { + delete process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN; + } else { + process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = previousGrpcFakeSsh; + } fs.rmSync(tmp, { recursive: true, force: true }); }); @@ -57,7 +98,7 @@ exit 0 { channel: "telegram", conflicts: 7 }, ]); expect(fs.readFileSync(callsFile, "utf-8")).toContain( - "sandbox exec -n alpha -- sh -c tail -n 200 /tmp/gateway.log", + "sandbox exec --name alpha -- sh -c tail -n 200 /tmp/gateway.log", ); }); diff --git a/src/lib/tunnel/services-sandbox.test.ts b/src/lib/tunnel/services-sandbox.test.ts index a4e2299276..cf7924bce8 100644 --- a/src/lib/tunnel/services-sandbox.test.ts +++ b/src/lib/tunnel/services-sandbox.test.ts @@ -20,6 +20,42 @@ import { stopAll, } from "../../../dist/lib/tunnel/services"; +function syncRunnerResult(status = 0, stdout = "", stderr = "") { + return { + status: 0, + stdout: JSON.stringify({ + ok: true, + result: { status, stdout, stderr }, + }), + stderr: "", + }; +} + +function syncRunnerFailure(error: string) { + return { + status: 0, + stdout: JSON.stringify({ ok: false, error }), + stderr: "", + }; +} + +function readSyncRunnerPayload(spawnSyncSpy: ReturnType, callIndex = 0) { + const syncCalls = spawnSyncSpy.mock.calls.filter((call: unknown[]) => { + const [command, args] = call; + return ( + command === process.execPath && + Array.isArray(args) && + args.some((arg) => String(arg).includes("sync-runner")) + ); + }); + const call = syncCalls[callIndex]; + expect(call, "expected gRPC sync runner spawn").toBeTruthy(); + return JSON.parse(String((call[2] as { input?: string }).input || "{}")) as { + sandboxName?: string; + argv?: string[]; + }; +} + // --------------------------------------------------------------------------- // stopSandboxChannels // --------------------------------------------------------------------------- @@ -99,18 +135,17 @@ describe("stopSandboxChannels", () => { logSpy.mockRestore(); }); - it("falls back to openshell sandbox exec when the gateway container is unavailable", () => { + it("falls back to OpenShell gRPC exec when the gateway container is unavailable", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - spawnSyncSpy.mockReturnValueOnce({ status: 1, stdout: "" }).mockReturnValueOnce({ status: 0 }); + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerResult()); stopSandboxChannels("my-sandbox"); - expect(spawnSyncSpy).toHaveBeenNthCalledWith( - 2, - "/usr/local/bin/openshell", - ["sandbox", "exec", "--name", "my-sandbox", "--", "sh", "-lc", expect.any(String)], - expect.objectContaining({ timeout: 20000 }), - ); + const payload = readSyncRunnerPayload(spawnSyncSpy); + expect(payload.sandboxName).toBe("my-sandbox"); + expect(payload.argv?.slice(0, 2)).toEqual(["sh", "-lc"]); const output = logSpy.mock.calls.map((c) => c[0]).join("\n"); expect(output).toContain("OpenClaw gateway stopped inside sandbox"); logSpy.mockRestore(); @@ -157,29 +192,32 @@ describe("stopSandboxChannels", () => { logSpy.mockRestore(); }); - it("warns when privileged shutdown is unavailable and openshell is not found", () => { + it("warns when privileged shutdown is unavailable and gRPC exec fails", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - resolveOpenshellModule.resolveOpenshell = vi.fn(() => null); - spawnSyncSpy.mockReturnValueOnce({ status: 1, stdout: "" }); + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerFailure("sync runner missing")); stopSandboxChannels("my-sandbox"); - expect(spawnSyncSpy).toHaveBeenCalledTimes(1); + expect(spawnSyncSpy).toHaveBeenCalledTimes(2); const output = logSpy.mock.calls.map((c) => c[0]).join("\n"); - expect(output).toContain("openshell not found"); + expect(output).toContain("OpenShell gRPC exec could not stop"); + expect(output).toContain("sync runner missing"); logSpy.mockRestore(); }); - it("uses --name flag for fallback sandbox selection (not positional)", () => { + it("passes the sandbox name in the gRPC fallback payload", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - spawnSyncSpy.mockReturnValueOnce({ status: 1, stdout: "" }).mockReturnValueOnce({ status: 0 }); + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerResult()); stopSandboxChannels("my-sandbox"); - const args = spawnSyncSpy.mock.calls[1][1] as string[]; - expect(args[1]).toBe("exec"); - expect(args[2]).toBe("--name"); - expect(args[3]).toBe("my-sandbox"); + const payload = readSyncRunnerPayload(spawnSyncSpy); + expect(payload.sandboxName).toBe("my-sandbox"); + expect(payload.argv?.slice(0, 2)).toEqual(["sh", "-lc"]); logSpy.mockRestore(); }); @@ -284,6 +322,9 @@ describe("stopAll with sandbox channels", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const saved = process.env.NEMOCLAW_SANDBOX; process.env.NEMOCLAW_SANDBOX = "env-sandbox"; + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerResult()); try { stopAll({ pidDir }); @@ -295,11 +336,7 @@ describe("stopAll with sandbox channels", () => { } } - expect(spawnSyncSpy).toHaveBeenCalledWith( - "/usr/local/bin/openshell", - expect.arrayContaining(["env-sandbox"]), - expect.any(Object), - ); + expect(readSyncRunnerPayload(spawnSyncSpy).sandboxName).toBe("env-sandbox"); logSpy.mockRestore(); }); @@ -309,6 +346,9 @@ describe("stopAll with sandbox channels", () => { const savedNemoclawName = process.env.NEMOCLAW_SANDBOX_NAME; delete process.env.NEMOCLAW_SANDBOX; process.env.NEMOCLAW_SANDBOX_NAME = "named-sandbox"; + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerResult()); try { stopAll({ pidDir }); @@ -325,11 +365,7 @@ describe("stopAll with sandbox channels", () => { } } - expect(spawnSyncSpy).toHaveBeenCalledWith( - "/usr/local/bin/openshell", - expect.arrayContaining(["named-sandbox"]), - expect.any(Object), - ); + expect(readSyncRunnerPayload(spawnSyncSpy).sandboxName).toBe("named-sandbox"); logSpy.mockRestore(); }); @@ -341,6 +377,9 @@ describe("stopAll with sandbox channels", () => { process.env.NEMOCLAW_SANDBOX_NAME = "name-sandbox"; process.env.NEMOCLAW_SANDBOX = "other-sandbox"; delete process.env.SANDBOX_NAME; + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerResult()); try { stopAll({ pidDir }); @@ -353,11 +392,7 @@ describe("stopAll with sandbox channels", () => { else delete process.env.SANDBOX_NAME; } - expect(spawnSyncSpy).toHaveBeenCalledWith( - "/usr/local/bin/openshell", - expect.arrayContaining(["name-sandbox"]), - expect.any(Object), - ); + expect(readSyncRunnerPayload(spawnSyncSpy).sandboxName).toBe("name-sandbox"); logSpy.mockRestore(); }); @@ -375,15 +410,14 @@ describe("stopAll with sandbox channels", () => { process.env.NEMOCLAW_SANDBOX_NAME = "name-sandbox"; process.env.NEMOCLAW_SANDBOX = "other-sandbox"; delete process.env.SANDBOX_NAME; + spawnSyncSpy + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce(syncRunnerResult()); try { stopAll(); - expect(spawnSyncSpy).toHaveBeenCalledWith( - "/usr/local/bin/openshell", - expect.arrayContaining(["name-sandbox"]), - expect.any(Object), - ); + expect(readSyncRunnerPayload(spawnSyncSpy).sandboxName).toBe("name-sandbox"); expect(existsSync(join(effectivePidDir, "cloudflared.pid"))).toBe(false); expect(existsSync(join(lowerPriorityPidDir, "cloudflared.pid"))).toBe(true); } finally { diff --git a/test/helpers/grpc-fake-ssh.cjs b/test/helpers/grpc-fake-ssh.cjs index e8278ce679..8cf03c5167 100755 --- a/test/helpers/grpc-fake-ssh.cjs +++ b/test/helpers/grpc-fake-ssh.cjs @@ -55,6 +55,10 @@ if (result.error) { process.exit(1); } -process.stdout.write(result.stdout || ""); -process.stderr.write(result.stderr || ""); +if (result.stdout && result.stdout.length > 0) { + fs.writeSync(1, result.stdout); +} +if (result.stderr && result.stderr.length > 0) { + fs.writeSync(2, result.stderr); +} process.exit(result.status ?? 1); diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 5cc6513fd0..834a840265 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -10,6 +10,8 @@ import { describe, expect, it } from "vitest"; import { stageOptimizedSandboxBuildContext } from "../dist/lib/sandbox/build-context.js"; import { testTimeoutOptions } from "./helpers/timeouts"; +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); + type ShimScalar = string | number | boolean | null | undefined; type ShimCallable = (...args: readonly string[]) => ShimValue; type ShimValue = ShimScalar | { [key: string]: ShimValue } | ShimValue[] | ShimCallable; @@ -59,6 +61,14 @@ function stripMessagingEnv(source: NodeJS.ProcessEnv): Record { + return { + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + }; +} + type OnboardTestInternalsCandidate = Partial | null; function isOnboardTestInternals( @@ -89,6 +99,9 @@ const { } = onboardTestInternals; const repoRoot = path.join(import.meta.dirname, ".."); +const forwardBridgeStatePath = JSON.stringify( + path.join(repoRoot, "dist", "lib", "adapters", "openshell", "forward-bridge-state.js"), +); const onboardScriptMocksPath = JSON.stringify( path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), ); @@ -1681,10 +1694,8 @@ console.log(JSON.stringify({ fs.writeFileSync( fakeOpenshell, `#!/usr/bin/env bash -if [ "$1" = "sandbox" ] && [ "$2" = "download" ]; then - dest="\${@: -1}" - mkdir -p "$dest/sandbox/.openclaw" - cat > "$dest/sandbox/.openclaw/openclaw.json" <<'EOF' +if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && printf "%s\\n" "$*" | grep -q "/sandbox/.openclaw/openclaw.json"; then + cat <<'EOF' {"gateway":{"auth":{"token":"test-token"}}} EOF exit 0 @@ -1709,7 +1720,9 @@ console.log(JSON.stringify({ encoding: "utf-8", env: { ...process.env, + HOME: tmpDir, PATH: `${tmpDir}:${process.env.PATH || ""}`, + ...grpcTestEnv(), }, }); @@ -1795,7 +1808,7 @@ console.log(JSON.stringify({ mode: 0o755, }); - const script = String.raw` +const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); @@ -2268,6 +2281,7 @@ console.log(JSON.stringify({ liveExists, sandbox: registry.getSandbox("my-assist const script = String.raw` const runner = require(${runnerPath}); +const { listForwardStates } = require(${forwardBridgeStatePath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -2328,7 +2342,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox(null, "gpt-5.4"); - console.log(JSON.stringify({ sandboxName, commands, registerCalls, updateCalls, defaultCalls })); + console.log(JSON.stringify({ sandboxName, commands, registerCalls, updateCalls, defaultCalls, forwardStates: listForwardStates() })); })().catch((error) => { console.error(error); process.exit(1); @@ -2344,6 +2358,7 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + ...grpcTestEnv(), }, }); @@ -2384,14 +2399,17 @@ const { createSandbox } = require(${onboardPath}); assert.doesNotMatch(createCommand.command, /NVIDIA_API_KEY=/); assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); - assert.ok( - payload.commands.some( - (entry: CommandEntry) => - entry.command.includes("forward start --background 18789 my-assistant") || - entry.command.includes("forward start --background 0.0.0.0:18789 my-assistant"), - ), - "expected dashboard forward (loopback or WSL 0.0.0.0)", - ); + assert.deepEqual(payload.forwardStates, [ + { + sandboxName: "my-assistant", + bind: "127.0.0.1", + port: 18789, + targetHost: "127.0.0.1", + targetPort: 18789, + pid: 0, + startedAt: payload.forwardStates[0].startedAt, + }, + ]); }, ); @@ -2764,6 +2782,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +const { listForwardStates } = require(${forwardBridgeStatePath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -2813,7 +2832,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.CHAT_UI_URL = "https://chat.example.com"; await createSandbox(null, "gpt-5.4"); - console.log(JSON.stringify(commands)); + console.log(JSON.stringify({ commands, forwardStates: listForwardStates() })); })().catch((error) => { console.error(error); process.exit(1); @@ -2829,17 +2848,28 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + ...grpcTestEnv(), }, }); assert.equal(result.status, 0, result.stderr); - const commands = parseStdoutJson(result.stdout); - assert.ok( - commands.some((entry: CommandEntry) => - entry.command.includes("forward start --background 0.0.0.0:18789 my-assistant"), - ), - "expected remote dashboard forward target", - ); + const payload = parseStdoutJson<{ + commands: CommandEntry[]; + forwardStates: Array<{ sandboxName: string; bind: string; port: number; targetPort: number }>; + }>(result.stdout); + assert.deepEqual(payload.forwardStates.map(({ sandboxName, bind, port, targetPort }) => ({ + sandboxName, + bind, + port, + targetPort, + })), [ + { + sandboxName: "my-assistant", + bind: "0.0.0.0", + port: 18789, + targetPort: 18789, + }, + ]); }); it("injects NEMOCLAW_DASHBOARD_PORT into sandbox create envArgs when set (#1925)", async () => { @@ -2860,6 +2890,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +const { listForwardStates } = require(${forwardBridgeStatePath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -2913,7 +2944,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox(null, "gpt-5.4"); - console.log(JSON.stringify({ sandboxName, commands })); + console.log(JSON.stringify({ sandboxName, commands, forwardStates: listForwardStates() })); })().catch((error) => { console.error(error); process.exit(1); @@ -2935,6 +2966,7 @@ const { createSandbox } = require(${onboardPath}); PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_DASHBOARD_PORT: "19000", + ...grpcTestEnv(), }, }); @@ -2956,14 +2988,21 @@ const { createSandbox } = require(${onboardPath}); // overriding whatever value the Docker image had baked in. assert.match(createCommand.command, /NEMOCLAW_DASHBOARD_PORT=19000/); // Forward must use same-port mapping (openshell does not support asymmetric) - assert.ok( - payload.commands.some( - (entry: CommandEntry) => - entry.command.includes("forward start --background 19000 my-assistant") || - entry.command.includes("forward start --background 0.0.0.0:19000 my-assistant"), - ), - "expected dashboard forward for port 19000", - ); + assert.deepEqual(payload.forwardStates.map( + ({ sandboxName, bind, port, targetPort }: { sandboxName: string; bind: string; port: number; targetPort: number }) => ({ + sandboxName, + bind, + port, + targetPort, + }), + ), [ + { + sandboxName: "my-assistant", + bind: "127.0.0.1", + port: 19000, + targetPort: 19000, + }, + ]); assert.ok( !payload.commands.some((entry: CommandEntry) => entry.command.includes("19000:18789")), "forward must not use asymmetric 19000:18789 mapping", @@ -2993,7 +3032,7 @@ const { createSandbox } = require(${onboardPath}); mode: 0o755, }); - const script = String.raw` +const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); @@ -4271,6 +4310,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +const { listForwardStates } = require(${forwardBridgeStatePath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const childProcess = require("node:child_process"); @@ -4306,7 +4346,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.CHAT_UI_URL = "https://chat.example.com"; const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); - console.log(JSON.stringify({ sandboxName, commands })); + console.log(JSON.stringify({ sandboxName, commands, forwardStates: listForwardStates() })); })().catch((error) => { console.error(error); process.exit(1); @@ -4322,6 +4362,7 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + ...grpcTestEnv(), }, }); @@ -4329,14 +4370,22 @@ const { createSandbox } = require(${onboardPath}); const payload = parseStdoutJson<{ sandboxName: string; commands: CommandEntry[]; + forwardStates: Array<{ sandboxName: string; bind: string; port: number; targetPort: number }>; }>(result.stdout); assert.equal(payload.sandboxName, "my-assistant"); - assert.ok( - payload.commands.some((entry: CommandEntry) => - entry.command.includes("forward start --background 0.0.0.0:18789 my-assistant"), - ), - "expected dashboard forward restore on sandbox reuse", - ); + assert.deepEqual(payload.forwardStates.map(({ sandboxName, bind, port, targetPort }) => ({ + sandboxName, + bind, + port, + targetPort, + })), [ + { + sandboxName: "my-assistant", + bind: "0.0.0.0", + port: 18789, + targetPort: 18789, + }, + ]); assert.ok( payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox create")), "did not expect sandbox create when reusing existing sandbox", diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index 5333347b8a..ed56abc6c7 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -240,6 +240,9 @@ function runRebuild(fixture: ReturnType) { PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: path.join(fixture.tmpDir, "ssh"), NO_COLOR: "1", }, timeout: 30_000, diff --git a/test/sandbox-connect-inference.test.ts b/test/sandbox-connect-inference.test.ts index caef5d06d9..2a2485fb8c 100644 --- a/test/sandbox-connect-inference.test.ts +++ b/test/sandbox-connect-inference.test.ts @@ -8,6 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { execTimeout, testTimeoutOptions } from "./helpers/timeouts"; +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); + /** * Tests for #1248 — inference route swap on sandbox connect. * @@ -386,6 +388,9 @@ function runConnect( NEMOCLAW_NO_CONNECT_HINT: "1", NEMOCLAW_OLLAMA_PORT: "11434", NEMOCLAW_OLLAMA_PROXY_PORT: "11435", + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, ...extraEnv, }, timeout: execTimeout(15_000), diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index dcf948dd64..8350e6ec50 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -9,6 +9,8 @@ import { pathToFileURL } from "url"; import { spawnSync } from "child_process"; import { describe, it, expect } from "vitest"; +const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); + describe("sandboxName command hardening in onboard.js", () => { it("re-validates sandboxName at the createSandbox boundary", async () => { const onboardModule = await import("../dist/lib/onboard.js"); @@ -52,9 +54,21 @@ describe("sandboxName command hardening in onboard.js", () => { ); fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); + fs.writeFileSync( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then + if printf "%s\\n" "$*" | grep -q "/health"; then + printf "200" + fi + exit 0 +fi +exit 0 +`, + { + mode: 0o755, + }, + ); fs.writeFileSync( scriptPath, String.raw` @@ -68,6 +82,9 @@ for (const key of Object.keys(process.env)) { delete process.env[key]; } } +process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; +process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; +process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = ${JSON.stringify(GRPC_FAKE_SSH)}; const commands = []; const asText = (command) => Array.isArray(command) ? command.join(" ") : String(command); runner.run = (command, opts = {}) => { @@ -119,7 +136,13 @@ try { const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", - env: { HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}` }, + env: { + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_GRPC_TEST_TRANSPORT: "1", + NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", + NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + }, timeout: 30_000, }); expect(result.status, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); diff --git a/test/snapshot-gateway-guard.test.ts b/test/snapshot-gateway-guard.test.ts index 307921d752..f528509b79 100644 --- a/test/snapshot-gateway-guard.test.ts +++ b/test/snapshot-gateway-guard.test.ts @@ -119,7 +119,29 @@ function makeStoppedGatewayEnv(prefix: string): Record { function makeHealthyVmGatewayEnv(prefix: string): Record { const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); const localBin = path.join(home, "bin"); + const fakeOpenClawRoot = path.join(home, "fake-openclaw-root"); fs.mkdirSync(localBin, { recursive: true }); + const stateDirs = [ + "agents", + "extensions", + "workspace", + "skills", + "hooks", + "identity", + "devices", + "canvas", + "cron", + "memory", + "telegram", + "wechat", + "whatsapp", + "credentials", + ]; + for (const dir of stateDirs) { + const fullPath = path.join(fakeOpenClawRoot, dir); + fs.mkdirSync(fullPath, { recursive: true }); + fs.writeFileSync(path.join(fullPath, ".keep"), ""); + } writeSandboxRegistry(home, "alpha", { openshellDriver: "vm" }); // VM-driver snapshots should trust gateway metadata, not the legacy cluster @@ -135,14 +157,11 @@ function makeHealthyVmGatewayEnv(prefix: string): Record { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', - ' cmd="$8"', - ' if printf "%s" "$cmd" | grep -q "awk .!seen"; then', - " exit 0", - " fi", - ' if printf "%s" "$cmd" | grep -q "find "; then', - " exit 0", - " fi", - " exit 2", + ' cmd=""', + ' for arg in "$@"; do cmd="$arg"; done', + ` translated="$(printf "%s" "$cmd" | sed 's#/sandbox/.openclaw#${fakeOpenClawRoot}#g')"`, + ' sh -c "$translated"', + " exit $?", "fi", 'if [ "$1" = "status" ]; then exit 0; fi', "exit 0", From 347c995b1423d4e67c9f92a91fc9a2cec1016dd8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 15:02:22 -0700 Subject: [PATCH 12/26] Fix OpenShell gRPC sandbox migration failures --- src/lib/actions/sandbox/destroy.ts | 13 ++ src/lib/actions/sandbox/process-recovery.ts | 55 +++-- .../openshell/forward-bridge-state.test.ts | 27 +++ .../openshell/forward-bridge-state.ts | 32 ++- src/lib/adapters/openshell/grpc.test.ts | 52 +++++ src/lib/adapters/openshell/grpc.ts | 18 +- src/lib/onboard/dashboard.ts | 29 ++- src/lib/state/sandbox.ts | 202 +++++++++++++----- src/lib/tunnel/services-sandbox.test.ts | 6 +- src/lib/tunnel/services.ts | 14 +- test/destroy-cleanup-sandbox-services.test.ts | 44 +++- test/onboard-dashboard.test.ts | 31 +++ test/snapshot.test.ts | 76 ++++++- 13 files changed, 507 insertions(+), 92 deletions(-) create mode 100644 src/lib/adapters/openshell/forward-bridge-state.test.ts create mode 100644 src/lib/adapters/openshell/grpc.test.ts diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index e39fbcd1c6..b1599c1541 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -5,6 +5,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { + listForwardStates, + stopForwardBridge, +} from "../../adapters/openshell/forward-bridge-state"; import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { CLI_NAME } from "../../cli/branding"; @@ -45,6 +49,8 @@ type RemoveSandboxImageDeps = { type RemoveSandboxRegistryEntryDeps = { removeImage?: (sandboxName: string) => void; removeSandbox?: typeof registry.removeSandbox; + listForwardStates?: typeof listForwardStates; + stopForwardBridge?: typeof stopForwardBridge; }; type RunOpenshell = ( @@ -383,7 +389,14 @@ export function removeSandboxRegistryEntry( ): boolean { const removeImage = deps.removeImage ?? removeSandboxImage; const removeSandbox = deps.removeSandbox ?? registry.removeSandbox; + const listForwards = deps.listForwardStates ?? listForwardStates; + const stopForward = deps.stopForwardBridge ?? stopForwardBridge; removeImage(sandboxName); + for (const state of listForwards()) { + if (state.sandboxName === sandboxName) { + stopForward(sandboxName, state.port); + } + } return removeSandbox(sandboxName); } diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 73c5d54c73..a49d4e849b 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -3,7 +3,11 @@ import { spawnSync } from "node:child_process"; -import { createSandboxGrpcClient, execTextSync } from "../../adapters/openshell/grpc"; +import { + createSandboxGrpcClient, + execInputStreamSync, + execTextSync, +} from "../../adapters/openshell/grpc"; import { forwardStatesAsListOutput, startForwardBridgeDetached, @@ -72,17 +76,27 @@ function getSandboxHealthProbeUrl(sandboxName: string): string { } /** Run a shell command inside the sandbox over OpenShell gRPC. */ +function executeSandboxShellSync( + sandboxName: string, + command: string, + timeoutMs: number, +): SandboxCommandResult { + const result = /[\r\n]/.test(command) + ? execInputStreamSync(sandboxName, ["sh", "-s"], command, { timeoutMs }) + : execTextSync(sandboxName, ["sh", "-c", command], { timeoutMs }); + return { + status: result.status, + stdout: result.stdout.toString().trim(), + stderr: result.stderr.toString().trim(), + }; +} + export function executeSandboxCommand( sandboxName: string, command: string, ): SandboxCommandResult | null { try { - const result = execTextSync(sandboxName, ["sh", "-c", command], { timeoutMs: 15_000 }); - return { - status: result.status, - stdout: result.stdout.trim(), - stderr: result.stderr.trim(), - }; + return executeSandboxShellSync(sandboxName, command, 15_000); } catch { return null; } @@ -93,14 +107,12 @@ export function executeSandboxExecCommand( command: string, timeout = 15000, ): SandboxCommandResult | null { - const markedCommand = `printf '%s\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; + const markedCommand = `printf '%s\\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; const timeoutOverride = Number(process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS || ""); const effectiveTimeout = Number.isFinite(timeoutOverride) && timeoutOverride > 0 ? timeoutOverride : timeout; try { - const result = execTextSync(sandboxName, ["sh", "-c", markedCommand], { - timeoutMs: effectiveTimeout, - }); + const result = executeSandboxShellSync(sandboxName, markedCommand, effectiveTimeout); const stdout = result.stdout.trim(); const stdoutLines = stdout.split(/\r?\n/); const markerIndex = stdoutLines.indexOf(SANDBOX_EXEC_STARTED_MARKER); @@ -123,13 +135,21 @@ async function executeSandboxExecCommandForStatus( if (process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1") { return executeSandboxExecCommand(sandboxName, command, OPENSHELL_PROBE_TIMEOUT_MS); } - const markedCommand = `printf '%s\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; + const markedCommand = `printf '%s\\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; const client = createSandboxGrpcClient(); try { - const result = await client.execText(sandboxName, ["sh", "-c", markedCommand], { - timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS, - }); - const stdout = result.stdout.trim(); + const result = /[\r\n]/.test(markedCommand) + ? await client.execInputStream( + sandboxName, + ["sh", "-s"], + markedCommand, + { timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS }, + ) + : await client.execText(sandboxName, ["sh", "-c", markedCommand], { + timeoutMs: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const stdout = + typeof result.stdout === "string" ? result.stdout.trim() : result.stdout.toString("utf-8").trim(); const stdoutLines = stdout.split(/\r?\n/); const markerIndex = stdoutLines.indexOf(SANDBOX_EXEC_STARTED_MARKER); if (markerIndex === -1) return null; @@ -137,7 +157,8 @@ async function executeSandboxExecCommandForStatus( return { status: result.status, stdout: commandStdoutLines.join("\n").trim(), - stderr: result.stderr.trim(), + stderr: + typeof result.stderr === "string" ? result.stderr.trim() : result.stderr.toString("utf-8").trim(), }; } catch { return null; diff --git a/src/lib/adapters/openshell/forward-bridge-state.test.ts b/src/lib/adapters/openshell/forward-bridge-state.test.ts new file mode 100644 index 0000000000..6478011034 --- /dev/null +++ b/src/lib/adapters/openshell/forward-bridge-state.test.ts @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; + +import { describe, expect, it } from "vitest"; + +import { __forwardBridgeTestHooks } from "./forward-bridge-state"; + +async function unusedLocalPort(): Promise { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + return port; +} + +describe("OpenShell gRPC forward bridge readiness", () => { + it("does not treat state-file presence as readiness when the local port is unreachable", async () => { + const port = await unusedLocalPort(); + expect(__forwardBridgeTestHooks.probeForwardReady("127.0.0.1", port)).toBe(false); + }); +}); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index cabe554fc2..f119c3601a 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -167,6 +167,26 @@ function readOpenFileDescriptor(fd: number): string { return buffer.subarray(0, bytesRead).toString("utf-8"); } +function probeForwardReady(bind: string, port: number): boolean { + const host = bind === "0.0.0.0" || bind === "::" ? "127.0.0.1" : bind; + const script = + "const net=require('node:net');" + + `const socket=net.createConnection({host:${JSON.stringify(host)},port:${port}});` + + "let data='';let done=false;" + + "const finish=(code)=>{if(done)return;done=true;socket.destroy();process.exit(code);};" + + "socket.setTimeout(1000);" + + "socket.on('connect',()=>socket.write('GET /health HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n'));" + + "socket.on('data',(chunk)=>{data+=chunk.toString('utf8');if(/^HTTP\\//.test(data))finish(0);});" + + "socket.on('error',()=>finish(1));" + + "socket.on('timeout',()=>finish(1));" + + "socket.on('end',()=>finish(/^HTTP\\//.test(data)?0:1));"; + const result = spawnSync(process.execPath, ["-e", script], { + stdio: "ignore", + timeout: 1500, + }); + return result.status === 0; +} + export function startForwardBridgeDetached( sandboxName: string, options: ForwardBridgeStartOptions, @@ -217,8 +237,10 @@ export function startForwardBridgeDetached( while (Date.now() < deadline) { const state = getForwardState(sandboxName, options.port); if (state?.pid === child.pid) { - fs.closeSync(out); - return { ok: true, state: state ?? undefined, diagnostic: "" }; + if (probeForwardReady(bind, options.port)) { + fs.closeSync(out); + return { ok: true, state: state ?? undefined, diagnostic: "" }; + } } sleepMs(250); } @@ -241,3 +263,7 @@ export function startForwardBridgeDetached( diagnostic || `forward bridge did not become ready within ${String(timeoutMs)}ms`, }; } + +export const __forwardBridgeTestHooks = { + probeForwardReady, +}; diff --git a/src/lib/adapters/openshell/grpc.test.ts b/src/lib/adapters/openshell/grpc.test.ts new file mode 100644 index 0000000000..ee572a5e4d --- /dev/null +++ b/src/lib/adapters/openshell/grpc.test.ts @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EventEmitter } from "node:events"; + +import { describe, expect, it } from "vitest"; + +import { __grpcTestHooks } from "./grpc"; + +function collect(events: unknown[]) { + const stream = new EventEmitter() as EventEmitter & { cancel: () => void }; + stream.cancel = () => undefined; + const result = __grpcTestHooks.collectExecStream(stream as any, "test exec", 0); + queueMicrotask(() => { + for (const event of events) stream.emit("data", event); + stream.emit("end"); + }); + return result; +} + +describe("OpenShell gRPC exec stream parsing", () => { + it("parses keepCase exit_code events", async () => { + await expect( + collect([ + { stdout: { data: Buffer.from("ok\n") } }, + { exit: { exit_code: 0 } }, + ]), + ).resolves.toMatchObject({ + status: 0, + stdout: Buffer.from("ok\n"), + }); + }); + + it("parses camelCase exitCode events", async () => { + await expect( + collect([ + { stderr: { data: Buffer.from("boom\n") } }, + { exit: { exitCode: 42 } }, + ]), + ).resolves.toMatchObject({ + status: 42, + stderr: Buffer.from("boom\n"), + }); + }); + + it("keeps missing exit events non-successful so call sites must prove completion", async () => { + await expect(collect([{ stdout: { data: Buffer.from("partial") } }])).resolves.toMatchObject({ + status: 1, + stdout: Buffer.from("partial"), + }); + }); +}); diff --git a/src/lib/adapters/openshell/grpc.ts b/src/lib/adapters/openshell/grpc.ts index 1df5869a56..a42923ac87 100644 --- a/src/lib/adapters/openshell/grpc.ts +++ b/src/lib/adapters/openshell/grpc.ts @@ -177,6 +177,13 @@ function bufferFromData(data: unknown): Buffer { return Buffer.alloc(0); } +function exitCodeFromEvent(event: any): number | null { + const raw = event?.exit?.exit_code ?? event?.exit?.exitCode; + if (raw === undefined || raw === null) return null; + const parsed = Number(raw); + return Number.isFinite(parsed) ? parsed : 1; +} + function stdinBuffer(input: Buffer | string | undefined): Buffer | undefined { if (input === undefined) return undefined; return Buffer.isBuffer(input) ? input : Buffer.from(input); @@ -225,10 +232,8 @@ function collectExecStream( stream.on("data", (event: any) => { if (event?.stdout?.data !== undefined) stdout.push(bufferFromData(event.stdout.data)); if (event?.stderr?.data !== undefined) stderr.push(bufferFromData(event.stderr.data)); - if (event?.exit?.exit_code !== undefined) { - const parsed = Number(event.exit.exit_code); - exitCode = Number.isFinite(parsed) ? parsed : 1; - } + const parsedExitCode = exitCodeFromEvent(event); + if (parsedExitCode !== null) exitCode = parsedExitCode; }); stream.on("error", (error) => { settle(() => reject(new Error(`${label} failed: ${formatGrpcError(error)}`))); @@ -663,5 +668,10 @@ export function execInputStreamSync( }; } +export const __grpcTestHooks = { + collectExecStream, + exitCodeFromEvent, +}; + // Keep CommonJS transpilation happy if this file is ever executed through ESM-aware tooling. void fileURLToPath; diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index efd5d2e4d3..3fcc50de5c 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -2,10 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import type { AgentDefinition } from "../agent/defs"; +import * as agentRuntime from "../agent/runtime"; import { DASHBOARD_PORT } from "../core/ports"; import { buildChain, buildControlUiUrls } from "../dashboard/contract"; import * as nim from "../inference/nim"; import { runCapture as defaultRunCapture } from "../runner"; +import * as registry from "../state/registry"; import * as dashboardAccess from "./dashboard-access"; import { findAvailableDashboardPort, @@ -42,6 +44,7 @@ export interface OnboardDashboardDeps { redact(value: unknown): string; sleep(seconds: number): void; fetchGatewayAuthTokenFromSandbox?: (sandboxName: string) => string | null; + resolveSandboxDashboardTargetPort?: (sandboxName: string, fallbackPort: number) => number; printAgentDashboardUi( sandboxName: string, token: string | null, @@ -142,6 +145,23 @@ function parseForwardTarget(target: string): { bind: string; port: number } { }; } +function isValidPort(value: unknown): value is number { + return ( + typeof value === "number" && + Number.isInteger(value) && + value >= 1 && + value <= 65535 + ); +} + +function resolveSandboxDashboardTargetPort(sandboxName: string, fallbackPort: number): number { + const agent = agentRuntime.getSessionAgent(sandboxName); + if (agent && isValidPort(agent.forwardPort)) return agent.forwardPort; + const sandbox = registry.getSandbox(sandboxName); + if (isValidPort(sandbox?.dashboardPort)) return sandbox.dashboardPort; + return fallbackPort; +} + function dashboardUrlForDisplay(url: string, deps: OnboardDashboardDeps): string { return dashboardAccess.dashboardUrlForDisplay(url, deps.redact); } @@ -255,16 +275,17 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } } - const parsedUrl = new URL(chatUiUrl.includes("://") ? chatUiUrl : `http://${chatUiUrl}`); - parsedUrl.port = String(actualPort); - const actualTarget = getDashboardForwardTarget(parsedUrl.toString()); + const actualTarget = getDashboardForwardTarget(chatUiUrl); stopForwardForSandbox(actualPort); const target = parseForwardTarget(actualTarget); + const targetPort = ( + deps.resolveSandboxDashboardTargetPort ?? resolveSandboxDashboardTargetPort + )(sandboxName, target.port); const { ok: fwdOk, diagnostic: fwdDiagnostic } = startForwardBridgeDetached(sandboxName, { bind: target.bind, port: actualPort, targetHost: "127.0.0.1", - targetPort: target.port, + targetPort, timeoutMs: 30_000, }); if (!fwdOk) { diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 63b3a98064..f348aba9d6 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -41,6 +41,14 @@ const HOME_DIR = path.resolve(process.env.HOME || os.homedir()); const REBUILD_BACKUPS_DIR = path.join(HOME_DIR, ".nemoclaw", "rebuild-backups"); const MANIFEST_VERSION = 1; +const DIR_DISCOVERY_STATUS_MARKER = "__NEMOCLAW_DIR_DISCOVERY_STATUS__"; +const PRE_BACKUP_AUDIT_STATUS_MARKER = "__NEMOCLAW_PRE_BACKUP_AUDIT_STATUS__"; +const TAR_BACKUP_STATUS_MARKER = "__NEMOCLAW_TAR_BACKUP_STATUS__"; +const STATE_FILE_BACKUP_STATUS_MARKER = "__NEMOCLAW_STATE_FILE_BACKUP_STATUS__"; +const RESTORE_CLEANUP_STATUS_MARKER = "__NEMOCLAW_RESTORE_CLEANUP_STATUS__"; +const RESTORE_EXTRACT_STATUS_MARKER = "__NEMOCLAW_RESTORE_EXTRACT_STATUS__"; +const RESTORE_USABILITY_STATUS_MARKER = "__NEMOCLAW_RESTORE_USABILITY_STATUS__"; +const STATE_FILE_RESTORE_STATUS_MARKER = "__NEMOCLAW_STATE_FILE_RESTORE_STATUS__"; function parseJson(text: string): T { return JSON.parse(text); @@ -634,6 +642,57 @@ function existingBackupDirs(backupPath: string, dirNames: string[]): string[] { return existing; } +function pythonCommand(script: string): string { + const encoded = Buffer.from(script, "utf-8").toString("base64"); + return `python3 -c ${shellQuote(`import base64; exec(base64.b64decode(${JSON.stringify(encoded)}))`)}`; +} + +function wrapShellCommandWithStatusMarker(command: string, marker: string): string { + return `( ${command} ); __nemoclaw_status=$?; printf '\\n%s:%s\\n' ${shellQuote(marker)} "$__nemoclaw_status" >&2; exit "$__nemoclaw_status"`; +} + +function markerRegex(marker: string): RegExp { + return new RegExp(`(?:^|\\r?\\n)${marker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}:(-?\\d+)(?=\\r?\\n|$)`); +} + +function statusFromMarker( + result: { status: number; stderr: string | Buffer }, + marker: string, +): { status: number; stderr: string; found: boolean } { + const stderr = Buffer.isBuffer(result.stderr) ? result.stderr.toString("utf-8") : result.stderr; + const match = markerRegex(marker).exec(stderr); + if (!match) return { status: result.status, stderr, found: false }; + const parsed = Number(match[1]); + const cleanStderr = stderr.replace(markerRegex(marker), "\n").trim(); + return { + status: Number.isFinite(parsed) ? parsed : result.status, + stderr: cleanStderr, + found: true, + }; +} + +function parseExistingStateDirs( + output: string, + declaredStateDirs: readonly string[], +): { ok: true; dirs: string[] } | { ok: false; error: string } { + const declared = new Set(declaredStateDirs); + const seen = new Set(); + const dirs: string[] = []; + for (const raw of output.split(/\r?\n/)) { + const dirName = raw.trim(); + if (!dirName || seen.has(dirName)) continue; + if ( + !isSafeStateDirPath(dirName) || + (!declared.has(dirName) && !/^workspace-[A-Za-z0-9._-]+$/.test(dirName)) + ) { + return { ok: false, error: `unsafe or undeclared state directory '${dirName}'` }; + } + seen.add(dirName); + dirs.push(dirName); + } + return { ok: true, dirs }; +} + function shouldPreserveOpenClawManagedExtensions( manifest: RebuildManifest, dir: string, @@ -791,7 +850,7 @@ function buildStateFileBackupCommand(dir: string, spec: StateFileSpec): string { '[ "${hardlink_count:-0}" = "0" ] || { echo "hard-linked sqlite state file rejected: $src" >&2; exit 11; }', 'tmp="$(mktemp /tmp/nemoclaw-sqlite-backup.XXXXXX)"', 'trap \'rm -f "$tmp"\' EXIT', - `python3 -c ${shellQuote(SQLITE_BACKUP_PY)} "$src" "$tmp"`, + `${pythonCommand(SQLITE_BACKUP_PY)} "$src" "$tmp"`, 'cat -- "$tmp"', ].join("; "); } @@ -812,7 +871,10 @@ function backupStateFile( spec: StateFileSpec, backupPath: string, ): "backed_up" | "missing" | "failed" { - const command = buildStateFileBackupCommand(dir, spec); + const command = wrapShellCommandWithStatusMarker( + buildStateFileBackupCommand(dir, spec), + STATE_FILE_BACKUP_STATUS_MARKER, + ); _log(`Backing up state file ${spec.path} (${spec.strategy})`); let result: ReturnType; try { @@ -826,10 +888,11 @@ function backupStateFile( return "failed"; } - if (result.status === 2) return "missing"; - if (result.status !== 0 || !result.stdout) { + const marked = statusFromMarker(result, STATE_FILE_BACKUP_STATUS_MARKER); + if (marked.status === 2) return "missing"; + if (marked.status !== 0 || !result.stdout) { const detail = - result.stderr.toString().trim() || `exit ${String(result.status)}`; + marked.stderr.trim() || `exit ${String(marked.status)}`; _log(`FAILED: state file backup ${spec.path}: ${detail.substring(0, 200)}`); return "failed"; } @@ -858,7 +921,7 @@ function buildStateFileRestoreCommand(dir: string, spec: StateFileSpec): string 'trap \'rm -f "$tmp"\' EXIT', 'cat > "$tmp"', 'chmod 600 "$tmp"', - `umask 0007; python3 -c ${shellQuote(SQLITE_RESTORE_PY)} "$tmp" "$dst"`, + `umask 0007; ${pythonCommand(SQLITE_RESTORE_PY)} "$tmp" "$dst"`, ].join("; "); } @@ -885,7 +948,10 @@ function restoreStateFile( const localPath = path.join(backupPath, spec.path); if (!existsSync(localPath)) return true; - const command = buildStateFileRestoreCommand(dir, spec); + const command = wrapShellCommandWithStatusMarker( + buildStateFileRestoreCommand(dir, spec), + STATE_FILE_RESTORE_STATUS_MARKER, + ); _log(`Restoring state file ${spec.path} (${spec.strategy})`); let result: ReturnType; try { @@ -901,10 +967,11 @@ function restoreStateFile( return false; } - if (result.status === 0) return true; + const marked = statusFromMarker(result, STATE_FILE_RESTORE_STATUS_MARKER); + if (marked.status === 0) return true; const detail = - result.stderr.toString().trim() || `exit ${String(result.status)}`; + marked.stderr.trim() || `exit ${String(marked.status)}`; _log(`FAILED: state file restore ${spec.path}: ${detail.substring(0, 200)}`); return false; } @@ -1017,7 +1084,10 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = .map((d) => `[ -d ${shellQuote(`${dir}/${d}`)} ] && printf '%s\\n' ${shellQuote(d)}`) .join("; "); const workspaceGlobCmd = `for d in ${shellQuote(dir)}/workspace-*/; do [ -d "$d" ] || continue; name="\${d%/}"; printf '%s\\n' "\${name##*/}"; done`; - const fullCheckCmd = `{ ${existCheckCmd}; ${workspaceGlobCmd}; } 2>/dev/null || true`; + const fullCheckCmd = wrapShellCommandWithStatusMarker( + `{ ${existCheckCmd}; ${workspaceGlobCmd}; } 2>/dev/null || true`, + DIR_DISCOVERY_STATUS_MARKER, + ); _log(`Checking existing dirs via gRPC exec: ${fullCheckCmd.substring(0, 100)}...`); let existResult: ReturnType; try { @@ -1042,23 +1112,29 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = _log( `Dir check: exit=${existResult.status}, stdout=${(existResult.stdout || "").trim().substring(0, 200)}, stderr=${(existResult.stderr || "").trim().substring(0, 200)}`, ); - const seenExistingDirs = new Set(); - const existingDirs = (existResult.stdout || "") - .trim() - .split("\n") - .map((d) => d.trim()) - .filter((d) => { - if (d.length === 0 || seenExistingDirs.has(d)) return false; - seenExistingDirs.add(d); - return true; - }); + const markedDirCheck = statusFromMarker(existResult, DIR_DISCOVERY_STATUS_MARKER); + const parsedExistingDirs = parseExistingStateDirs(existResult.stdout || "", stateDirs); + if (!parsedExistingDirs.ok) { + _log(`FAILED: gRPC dir check produced invalid output — ${parsedExistingDirs.error}`); + return { + success: false, + manifest, + backedUpDirs, + failedDirs: [...stateDirs], + backedUpFiles, + failedFiles: stateFiles.map((f) => f.path), + error: `Directory discovery failed: ${parsedExistingDirs.error}`, + }; + } + const existingDirs = parsedExistingDirs.dirs; _log( `Existing dirs in sandbox: [${existingDirs.join(",")}] (${existingDirs.length}/${stateDirs.length})`, ); - if (existResult.status !== 0) { + if (markedDirCheck.status !== 0) { + const detail = markedDirCheck.stderr.trim() || `exit ${String(markedDirCheck.status)}`; _log( - `FAILED: gRPC dir check exited ${existResult.status} — cannot determine which dirs exist`, + `FAILED: gRPC dir check exited ${markedDirCheck.status} — cannot determine which dirs exist`, ); return { success: false, @@ -1067,6 +1143,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = failedDirs: [...stateDirs], backedUpFiles, failedFiles: stateFiles.map((f) => f.path), + error: `Directory discovery failed: ${detail}`, }; } @@ -1089,12 +1166,15 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // stdout (the printf-emitted symlink/hardlink/special-file rows); // letting one perm-denied subdir abort the whole chain blocks legitimate // rebuilds. - const auditCmd = existingDirs - .map( - (d) => - `{ find ${shellQuote(`${dir}/${d}`)} \\( -type l -o \\( -type f -a -links +1 \\) -o \\( ! -type f -a ! -type d \\) \\) -printf "%y\\t%p\\t%l\\n" 2>/dev/null || true; }`, - ) - .join("; "); + const auditCmd = wrapShellCommandWithStatusMarker( + existingDirs + .map( + (d) => + `{ find ${shellQuote(`${dir}/${d}`)} \\( -type l -o \\( -type f -a -links +1 \\) -o \\( ! -type f -a ! -type d \\) \\) -printf "%y\\t%p\\t%l\\n" 2>/dev/null || true; }`, + ) + .join("; "), + PRE_BACKUP_AUDIT_STATUS_MARKER, + ); _log(`Pre-backup audit: checking for symlinks, hard links, and special files`); let auditResult: ReturnType; try { @@ -1114,9 +1194,9 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = error: `Pre-backup audit failed: ${detail}`, }; } - if (auditResult.status !== 0) { - const stderr = auditResult.stderr.trim(); - const detail = stderr || `exit ${String(auditResult.status)}`; + const markedAudit = statusFromMarker(auditResult, PRE_BACKUP_AUDIT_STATUS_MARKER); + if (markedAudit.status !== 0) { + const detail = markedAudit.stderr.trim() || `exit ${String(markedAudit.status)}`; _log(`FAILED: Pre-backup audit command failed — ${detail}`); return { success: false, @@ -1177,7 +1257,10 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // NC-2227-04: Removed -h flag (was following symlinks). State dirs are // now agent-writable and co-located with config — a compromised agent // could create symlinks to exfiltrate config contents via backup. - const tarCmd = `tar -cf - -C ${shellQuote(dir)} -- ${existingDirs.map(shellQuote).join(" ")}`; + const tarCmd = wrapShellCommandWithStatusMarker( + `tar -cf - -C ${shellQuote(dir)} -- ${existingDirs.map(shellQuote).join(" ")}`, + TAR_BACKUP_STATUS_MARKER, + ); _log(`Downloading via gRPC+tar: ${tarCmd}`); let result: ReturnType; try { @@ -1195,17 +1278,20 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = _log( `gRPC+tar download: exit=${result.status}, stdout=${result.stdout ? result.stdout.length + " bytes" : "null"}, stderr=${result.stderr.toString().substring(0, 200)}`, ); + const markedTar = statusFromMarker(result, TAR_BACKUP_STATUS_MARKER); // GNU tar exit codes: 0 = success, 1 = files changed during archive, // 2 = errors (e.g. permission denied) but archive still written to stdout. // Accept exit 0, 1, or 2 when stdout has data — extract what tar produced // and determine per-dir success from tar's reported read errors. const tarExitedWithData = - result.stdout && result.stdout.length > 0 && (result.status === 0 || result.status === 1 || result.status === 2); + result.stdout && + result.stdout.length > 0 && + (markedTar.status === 0 || markedTar.status === 1 || markedTar.status === 2); - if (result.status !== 0 && result.stdout && result.stdout.length > 0) { + if (markedTar.status !== 0 && result.stdout && result.stdout.length > 0) { _log( - `tar exited ${result.status} but produced ${result.stdout.length} bytes — attempting partial extraction`, + `tar exited ${markedTar.status} but produced ${result.stdout.length} bytes — attempting partial extraction`, ); } @@ -1214,7 +1300,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = const extractResult = safeTarExtract(result.stdout, backupPath); if (extractResult.success) { const extractedDirs = new Set(existingBackupDirs(backupPath, existingDirs)); - if (result.status === 0) { + if (markedTar.status === 0) { for (const d of existingDirs) { if (extractedDirs.has(d)) { backedUpDirs.push(d); @@ -1225,12 +1311,12 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = } } else { const tarFailedDirs = failedDirsFromTarStderr( - result.stderr.toString() || "", + markedTar.stderr || "", existingDirs, ); if (tarFailedDirs.size === 0) { _log( - `tar exited ${result.status} without attributable failed dirs — marking all dirs failed`, + `tar exited ${markedTar.status} without attributable failed dirs — marking all dirs failed`, ); failedDirs.push(...existingDirs); } else { @@ -1391,7 +1477,10 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re // image-managed extensions are preserved from the freshly built image and // excluded from the restore tar; only user/non-managed extension entries // are cleared and restored from the backup. - const rmCmd = buildRestoreCleanupCommand(dir, localDirs, preserveManagedExtensions); + const rmCmd = wrapShellCommandWithStatusMarker( + buildRestoreCleanupCommand(dir, localDirs, preserveManagedExtensions), + RESTORE_CLEANUP_STATUS_MARKER, + ); _log(`Cleaning target dirs before restore: ${rmCmd}`); let rmResult: ReturnType; try { @@ -1409,9 +1498,9 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re failedFiles: localFiles.map((f) => f.path), }; } - if (rmResult.status !== 0) { - const stderr = rmResult.stderr.toString().trim(); - const detail = stderr || `exit ${String(rmResult.status)}`; + const markedCleanup = statusFromMarker(rmResult, RESTORE_CLEANUP_STATUS_MARKER); + if (markedCleanup.status !== 0) { + const detail = markedCleanup.stderr.trim() || `exit ${String(markedCleanup.status)}`; _log(`FAILED: pre-restore cleanup failed: ${detail.substring(0, 200)}`); return { success: false, @@ -1422,7 +1511,10 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re }; } - const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`; + const extractCmd = wrapShellCommandWithStatusMarker( + `tar --no-same-owner -xf - -C ${shellQuote(dir)}`, + RESTORE_EXTRACT_STATUS_MARKER, + ); let extractResult: ReturnType; try { extractResult = execInputStreamSync(sandboxName, ["sh", "-c", extractCmd], tarResult.stdout, { @@ -1437,7 +1529,8 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re extractResult = { status: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; } - if (extractResult.status === 0) { + const markedExtract = statusFromMarker(extractResult, RESTORE_EXTRACT_STATUS_MARKER); + if (markedExtract.status === 0) { const restoredPaths = localDirs.map((d) => `${dir}/${d}`); // Best-effort only: OpenShell gRPC exec normally runs as the sandbox user, @@ -1455,12 +1548,15 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re ); } - const usabilityCmd = restoredPaths - .map( - (p) => - `[ -d ${shellQuote(p)} ] && [ ! -L ${shellQuote(p)} ] && [ -r ${shellQuote(p)} ] && [ -w ${shellQuote(p)} ]`, - ) - .join(" && "); + const usabilityCmd = wrapShellCommandWithStatusMarker( + restoredPaths + .map( + (p) => + `[ -d ${shellQuote(p)} ] && [ ! -L ${shellQuote(p)} ] && [ -r ${shellQuote(p)} ] && [ -w ${shellQuote(p)} ]`, + ) + .join(" && "), + RESTORE_USABILITY_STATUS_MARKER, + ); _log(`Verifying restored state usability: ${usabilityCmd}`); let usabilityResult: ReturnType; try { @@ -1474,11 +1570,11 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re stderr: Buffer.from(error instanceof Error ? error.message : String(error)), }; } - if (usabilityResult.status === 0) { + const markedUsability = statusFromMarker(usabilityResult, RESTORE_USABILITY_STATUS_MARKER); + if (markedUsability.status === 0) { restoredDirs.push(...localDirs); } else { - const stderr = usabilityResult.stderr.toString().trim(); - const detail = stderr || `exit ${String(usabilityResult.status)}`; + const detail = markedUsability.stderr.trim() || `exit ${String(markedUsability.status)}`; _log(`FAILED: restored state usability check failed: ${detail.substring(0, 200)}`); failedDirs.push(...localDirs); } diff --git a/src/lib/tunnel/services-sandbox.test.ts b/src/lib/tunnel/services-sandbox.test.ts index cf7924bce8..ecde9dc534 100644 --- a/src/lib/tunnel/services-sandbox.test.ts +++ b/src/lib/tunnel/services-sandbox.test.ts @@ -145,7 +145,8 @@ describe("stopSandboxChannels", () => { const payload = readSyncRunnerPayload(spawnSyncSpy); expect(payload.sandboxName).toBe("my-sandbox"); - expect(payload.argv?.slice(0, 2)).toEqual(["sh", "-lc"]); + expect(payload.argv).toEqual(["sh", "-s"]); + expect(payload.argv?.some((arg) => /[\r\n]/.test(arg))).toBe(false); const output = logSpy.mock.calls.map((c) => c[0]).join("\n"); expect(output).toContain("OpenClaw gateway stopped inside sandbox"); logSpy.mockRestore(); @@ -217,7 +218,8 @@ describe("stopSandboxChannels", () => { const payload = readSyncRunnerPayload(spawnSyncSpy); expect(payload.sandboxName).toBe("my-sandbox"); - expect(payload.argv?.slice(0, 2)).toEqual(["sh", "-lc"]); + expect(payload.argv).toEqual(["sh", "-s"]); + expect(payload.argv?.some((arg) => /[\r\n]/.test(arg))).toBe(false); logSpy.mockRestore(); }); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 5a17eb3160..53d2a278d4 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -17,7 +17,7 @@ import { import { basename, join } from "node:path"; import { dockerSpawnSync } from "../adapters/docker"; import { resolveOpenshell } from "../adapters/openshell/resolve"; -import { execTextSync } from "../adapters/openshell/grpc"; +import { execInputStreamSync } from "../adapters/openshell/grpc"; import { renderBox } from "../cli/banner"; import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding"; import { isRecord } from "../core/json-types"; @@ -454,16 +454,20 @@ export function stopSandboxChannels(sandboxName: string): void { if (reportStopResult(privilegedResult)) return; try { - const fallbackResult = execTextSync(sandboxName, ["sh", "-lc", GATEWAY_STOP_SCRIPT], { + const fallbackResult = execInputStreamSync(sandboxName, ["sh", "-s"], GATEWAY_STOP_SCRIPT, { timeoutMs: 20000, }); reportStopResult({ status: fallbackResult.status, signal: null, - stdout: fallbackResult.stdout, - stderr: fallbackResult.stderr, + stdout: fallbackResult.stdout.toString("utf-8"), + stderr: fallbackResult.stderr.toString("utf-8"), pid: 0, - output: [null, fallbackResult.stdout, fallbackResult.stderr], + output: [ + null, + fallbackResult.stdout.toString("utf-8"), + fallbackResult.stderr.toString("utf-8"), + ], } as StopAttemptResult); } catch (error) { warn( diff --git a/test/destroy-cleanup-sandbox-services.test.ts b/test/destroy-cleanup-sandbox-services.test.ts index 7da529daf4..b4eda98a20 100644 --- a/test/destroy-cleanup-sandbox-services.test.ts +++ b/test/destroy-cleanup-sandbox-services.test.ts @@ -11,7 +11,10 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { CleanupSandboxServicesDeps } from "../dist/lib/actions/sandbox/destroy.js"; -import { cleanupSandboxServices } from "../dist/lib/actions/sandbox/destroy.js"; +import { + cleanupSandboxServices, + removeSandboxRegistryEntry, +} from "../dist/lib/actions/sandbox/destroy.js"; type SandboxLike = { provider?: string | null } | null; @@ -113,3 +116,42 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { ]); }); }); + +describe("removeSandboxRegistryEntry forward cleanup", () => { + it("stops NemoClaw-managed gRPC forward bridges for the removed sandbox", () => { + const stopForwardBridge = vi.fn(); + const removeSandbox = vi.fn(() => true); + + expect( + removeSandboxRegistryEntry("alpha", { + removeImage: vi.fn(), + removeSandbox, + listForwardStates: vi.fn(() => [ + { + sandboxName: "alpha", + bind: "127.0.0.1", + port: 18789, + targetHost: "127.0.0.1", + targetPort: 18789, + pid: 123, + startedAt: "2026-05-27T00:00:00.000Z", + }, + { + sandboxName: "beta", + bind: "127.0.0.1", + port: 18790, + targetHost: "127.0.0.1", + targetPort: 18790, + pid: 456, + startedAt: "2026-05-27T00:00:00.000Z", + }, + ]), + stopForwardBridge, + }), + ).toBe(true); + + expect(stopForwardBridge).toHaveBeenCalledWith("alpha", 18789); + expect(stopForwardBridge).toHaveBeenCalledTimes(1); + expect(removeSandbox).toHaveBeenCalledWith("alpha"); + }); +}); diff --git a/test/onboard-dashboard.test.ts b/test/onboard-dashboard.test.ts index bb4ca58e13..2f6bad9e89 100644 --- a/test/onboard-dashboard.test.ts +++ b/test/onboard-dashboard.test.ts @@ -104,6 +104,37 @@ describe("onboard dashboard helpers", () => { } }); + it("keeps the sandbox dashboard target port separate from the reallocated host port", () => { + const originalHome = process.env.HOME; + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dashboard-target-port-")); + process.env.HOME = home; + const helpers = createOnboardDashboardHelpers({ + runOpenshell: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => ""), + openshellArgv: (args: string[]) => [process.execPath, "-e", "", ...args], + cliName: () => "nemoclaw", + agentProductName: () => "NemoClaw", + getProviderLabel: (provider: string) => provider, + note: vi.fn(), + isWsl: () => false, + redact: (value: unknown) => String(value), + sleep: vi.fn(), + resolveSandboxDashboardTargetPort: vi.fn(() => 18789), + printAgentDashboardUi: vi.fn(), + }); + + try { + const hostPort = helpers.ensureDashboardForward("my-sandbox", "http://127.0.0.1:18790"); + const state = listForwardStates().find((entry) => entry.sandboxName === "my-sandbox"); + expect(state).toMatchObject({ + port: hostPort, + targetPort: 18789, + }); + } finally { + process.env.HOME = originalHome; + } + }); + it("prints the dashboard-url command instead of raw gateway-token guidance", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const nimStatus = vi.fn(() => ({ running: false, container: "nemoclaw-nim-test" })); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index fd4f5767e0..abe151dca7 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -565,6 +565,68 @@ process.exit(0); } }); + it("trusts gRPC status sentinels when OpenShell omits successful exit events", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-grpc-status-marker-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const openclawDir = path.join(fixture, "sandbox-root", ".openclaw"); + const existingDirs = ["agents", "workspace", "skills"]; + fs.mkdirSync(binDir, { recursive: true }); + for (const dirName of existingDirs) { + fs.mkdirSync(path.join(openclawDir, dirName), { recursive: true }); + } + fs.writeFileSync(path.join(openclawDir, "workspace", "marker.txt"), "marker\n"); + + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const cmd = process.argv[process.argv.length - 1] || ""; +const existingDirs = ${JSON.stringify(existingDirs)}; +if (cmd.includes("[ -d ")) { + process.stdout.write(existingDirs.join("\\n") + "\\n"); + process.stderr.write("\\n__NEMOCLAW_DIR_DISCOVERY_STATUS__:0\\n"); + process.exit(1); +} +if (cmd.includes("find ")) { + process.stderr.write("\\n__NEMOCLAW_PRE_BACKUP_AUDIT_STATUS__:0\\n"); + process.exit(1); +} +if (cmd.includes("tar -cf -")) { + const r = spawnSync("tar", ["-cf", "-", "-C", ${JSON.stringify(openclawDir)}, ...existingDirs], { + stdio: ["ignore", "pipe", "pipe"], + }); + if (r.stdout) fs.writeSync(1, r.stdout); + process.stderr.write("\\n__NEMOCLAW_TAR_BACKUP_STATUS__:0\\n"); + process.exit(1); +} +process.exit(0); +`, + ); + + writeOpenClawRegistry("alpha"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}:${oldPath || ""}`; + + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + expect(backup.failedDirs).toEqual([]); + expect(backup.backedUpDirs).toEqual(existingDirs); + } finally { + if (oldOpenshell === undefined) { + delete process.env.NEMOCLAW_OPENSHELL_BIN; + } else { + process.env.NEMOCLAW_OPENSHELL_BIN = oldOpenshell; + } + process.env.PATH = oldPath; + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + it("excludes tar-failed directories from the restorable manifest", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-partial-tar-")); const oldPath = process.env.PATH; @@ -1382,9 +1444,17 @@ process.exit(0); ); const loggedCommands = fs.readFileSync(sshLog, "utf-8"); - expect(loggedCommands).toContain("sqlite3.connect"); - expect(loggedCommands).toContain("src_conn.backup(dst_conn)"); - expect(loggedCommands).toContain("PRAGMA quick_check"); + const commandText = loggedCommands + .trim() + .split("\n") + .map((line) => JSON.parse(line).cmd as string) + .join("\n"); + const decodedPython = [...commandText.matchAll(/base64\.b64decode\("([^"]+)"\)/g)] + .map((match) => Buffer.from(match[1], "base64").toString("utf-8")) + .join("\n"); + expect(decodedPython).toContain("sqlite3.connect"); + expect(decodedPython).toContain("src_conn.backup(dst_conn)"); + expect(decodedPython).toContain("PRAGMA quick_check"); } finally { if (oldOpenshell === undefined) { delete process.env.NEMOCLAW_OPENSHELL_BIN; From c4bb2cabb18bc64553957308412543b86b7dc654 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 15:23:22 -0700 Subject: [PATCH 13/26] Fix gRPC sandbox stream edge cases --- .../openshell/forward-bridge-state.test.ts | 79 ++++++++++++++++++- .../openshell/forward-bridge-state.ts | 18 +++++ src/lib/adapters/openshell/grpc.test.ts | 46 ++++++++++- src/lib/adapters/openshell/grpc.ts | 14 +++- src/lib/state/sandbox.ts | 47 +++++++++-- test/snapshot.test.ts | 62 +++++++++++++++ 6 files changed, 257 insertions(+), 9 deletions(-) diff --git a/src/lib/adapters/openshell/forward-bridge-state.test.ts b/src/lib/adapters/openshell/forward-bridge-state.test.ts index 6478011034..d428e1620d 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.test.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.test.ts @@ -1,11 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import fs from "node:fs"; import net from "node:net"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { __forwardBridgeTestHooks } from "./forward-bridge-state"; +import { + __forwardBridgeTestHooks, + stopForwardBridge, + writeForwardState, +} from "./forward-bridge-state"; async function unusedLocalPort(): Promise { const server = net.createServer(); @@ -19,9 +27,78 @@ async function unusedLocalPort(): Promise { return port; } +async function waitForReady(child: ChildProcessWithoutNullStreams): Promise { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("server did not become ready")), 2_000); + child.once("error", (error) => { + clearTimeout(timer); + reject(error); + }); + child.stdout.once("data", () => { + clearTimeout(timer); + resolve(); + }); + }); +} + +async function canListen(port: number): Promise { + const server = net.createServer(); + return new Promise((resolve) => { + server.once("error", () => resolve(false)); + server.listen(port, "127.0.0.1", () => { + server.close(() => resolve(true)); + }); + }); +} + describe("OpenShell gRPC forward bridge readiness", () => { it("does not treat state-file presence as readiness when the local port is unreachable", async () => { const port = await unusedLocalPort(); expect(__forwardBridgeTestHooks.probeForwardReady("127.0.0.1", port)).toBe(false); }); + + it("waits for a stopped bridge process to release its local port", async () => { + const originalHome = process.env.HOME; + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-forward-stop-")); + process.env.HOME = home; + const port = await unusedLocalPort(); + const child = spawn(process.execPath, [ + "-e", + [ + "const net=require('node:net');", + `const srv=net.createServer((socket)=>socket.end('ok')).listen(${port}, '127.0.0.1', () => console.log('ready'));`, + "setInterval(()=>{}, 1000);", + ].join(""), + ]); + try { + await waitForReady(child); + expect(await canListen(port)).toBe(false); + writeForwardState({ + sandboxName: "alpha", + bind: "127.0.0.1", + port, + targetHost: "127.0.0.1", + targetPort: 8642, + pid: child.pid ?? -1, + startedAt: new Date().toISOString(), + }); + + expect(stopForwardBridge("alpha", port)).toBe(true); + expect(await canListen(port)).toBe(true); + } finally { + if (child.pid) { + try { + process.kill(child.pid, "SIGKILL"); + } catch { + /* ignore */ + } + } + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + fs.rmSync(home, { recursive: true, force: true }); + } + }); }); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index f119c3601a..4a3a4c4e45 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -108,6 +108,15 @@ export function isPidAlive(pid: number): boolean { } } +function waitForPidExit(pid: number, timeoutMs = 2_000): boolean { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isPidAlive(pid)) return true; + sleepMs(50); + } + return !isPidAlive(pid); +} + export function stopForwardBridge(sandboxName: string, port: number | string): boolean { const state = readStateFile(statePath(sandboxName, port)); if (state && isPidAlive(state.pid) && !isTestForwardPid(state.pid)) { @@ -116,6 +125,14 @@ export function stopForwardBridge(sandboxName: string, port: number | string): b } catch { /* ignore */ } + if (!waitForPidExit(state.pid)) { + try { + process.kill(state.pid, "SIGKILL"); + } catch { + /* ignore */ + } + waitForPidExit(state.pid, 1_000); + } } removeForwardState(sandboxName, port); return Boolean(state); @@ -266,4 +283,5 @@ export function startForwardBridgeDetached( export const __forwardBridgeTestHooks = { probeForwardReady, + waitForPidExit, }; diff --git a/src/lib/adapters/openshell/grpc.test.ts b/src/lib/adapters/openshell/grpc.test.ts index ee572a5e4d..2ac7d491b4 100644 --- a/src/lib/adapters/openshell/grpc.test.ts +++ b/src/lib/adapters/openshell/grpc.test.ts @@ -2,10 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { __grpcTestHooks } from "./grpc"; +import { __grpcTestHooks, execBinaryStreamSync } from "./grpc"; function collect(events: unknown[]) { const stream = new EventEmitter() as EventEmitter & { cancel: () => void }; @@ -49,4 +52,45 @@ describe("OpenShell gRPC exec stream parsing", () => { stdout: Buffer.from("partial"), }); }); + + it("uses unary stdin for modest non-tty input so script uploads do not hang waiting for interactive EOF", () => { + expect(__grpcTestHooks.shouldInlineExecInput(Buffer.alloc(1024), {})).toBe(true); + expect(__grpcTestHooks.shouldInlineExecInput(Buffer.alloc(1024), { tty: true })).toBe(false); + expect(__grpcTestHooks.shouldInlineExecInput(Buffer.alloc(16 * 1024 * 1024 + 1), {})).toBe(false); + }); + + it("preserves large sync-runner binary stdout without hitting spawnSync ENOBUFS", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-grpc-large-stdout-")); + const oldEnv = { + transport: process.env.NEMOCLAW_GRPC_TEST_TRANSPORT, + legacy: process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH, + fakeSsh: process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN, + }; + try { + const fakeSsh = path.join(fixture, "ssh"); + fs.writeFileSync( + fakeSsh, + `#!/usr/bin/env node +require("node:fs").writeSync(1, Buffer.alloc(2 * 1024 * 1024, 0x61)); +process.exit(0); +`, + { mode: 0o755 }, + ); + process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; + process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; + process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = fakeSsh; + + const result = execBinaryStreamSync("alpha", ["cat", "/tmp/large"], { timeoutMs: 15_000 }); + expect(result.status).toBe(0); + expect(result.stdout.length).toBe(2 * 1024 * 1024); + } finally { + if (oldEnv.transport === undefined) delete process.env.NEMOCLAW_GRPC_TEST_TRANSPORT; + else process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = oldEnv.transport; + if (oldEnv.legacy === undefined) delete process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH; + else process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = oldEnv.legacy; + if (oldEnv.fakeSsh === undefined) delete process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN; + else process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = oldEnv.fakeSsh; + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); }); diff --git a/src/lib/adapters/openshell/grpc.ts b/src/lib/adapters/openshell/grpc.ts index a42923ac87..4e75c8d535 100644 --- a/src/lib/adapters/openshell/grpc.ts +++ b/src/lib/adapters/openshell/grpc.ts @@ -78,6 +78,7 @@ type GrpcClient = grpc.Client & Record any>; const DEFAULT_GRPC_TIMEOUT_MS = 30_000; const STREAM_CHUNK_SIZE = 64 * 1024; +const INLINE_STDIN_LIMIT = 16 * 1024 * 1024; let packageDefinition: grpc.GrpcObject | null = null; @@ -189,6 +190,10 @@ function stdinBuffer(input: Buffer | string | undefined): Buffer | undefined { return Buffer.isBuffer(input) ? input : Buffer.from(input); } +function shouldInlineExecInput(input: Buffer, opts: SandboxExecOptions = {}): boolean { + return opts.tty !== true && input.length <= INLINE_STDIN_LIMIT; +} + function sandboxIdFromResponse(response: any, sandboxName: string): string { const sandbox = response?.sandbox; const id = sandbox?.metadata?.id || sandbox?.id; @@ -344,6 +349,11 @@ export class SandboxGrpcClient { input: Buffer | string, opts: SandboxExecOptions = {}, ): Promise { + const inputBuffer = stdinBuffer(input) ?? Buffer.alloc(0); + if (shouldInlineExecInput(inputBuffer, opts)) { + return this.execBinaryStream(sandboxName, argv, { ...opts, stdin: inputBuffer }); + } + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); const execInteractive = method<() => grpc.ClientDuplexStream>( this.client, @@ -356,7 +366,7 @@ export class SandboxGrpcClient { opts.timeoutMs, ); stream.write({ start: execRequest(sandboxId, argv, opts) }); - writeInputChunks(stream, stdinBuffer(input) ?? Buffer.alloc(0)); + writeInputChunks(stream, inputBuffer); stream.end(); return result; } @@ -602,6 +612,7 @@ function runSyncRunner( encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], timeout: timeoutMs && timeoutMs > 0 ? timeoutMs + 5_000 : undefined, + maxBuffer: 512 * 1024 * 1024, env: process.env, }); if (result.error) { @@ -671,6 +682,7 @@ export function execInputStreamSync( export const __grpcTestHooks = { collectExecStream, exitCodeFromEvent, + shouldInlineExecInput, }; // Keep CommonJS transpilation happy if this file is ever executed through ESM-aware tooling. diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index f348aba9d6..7859da7cb2 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -671,6 +671,26 @@ function statusFromMarker( }; } +function tarStatusForBackup( + result: { status: number; stdout: Buffer; stderr: Buffer }, + marked: { status: number; stderr: string; found: boolean }, +): number { + // OpenShell 0.0.44 can occasionally finish a binary ExecSandbox stream + // without the terminal exit event or the final stderr status marker, while + // still delivering a complete tar archive on stdout. In that specific shape, + // a valid, safely extracted archive is stronger evidence than the adapter's + // conservative "missing exit means 1" default. + if ( + !marked.found && + result.status === 1 && + result.stdout.length > 0 && + marked.stderr.trim().length === 0 + ) { + return 0; + } + return marked.status; +} + function parseExistingStateDirs( output: string, declaredStateDirs: readonly string[], @@ -1279,6 +1299,12 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = `gRPC+tar download: exit=${result.status}, stdout=${result.stdout ? result.stdout.length + " bytes" : "null"}, stderr=${result.stderr.toString().substring(0, 200)}`, ); const markedTar = statusFromMarker(result, TAR_BACKUP_STATUS_MARKER); + const tarStatus = tarStatusForBackup(result, markedTar); + if (tarStatus !== markedTar.status) { + _log( + `gRPC+tar download missing exit marker but produced ${result.stdout.length} bytes; validating archive as success candidate`, + ); + } // GNU tar exit codes: 0 = success, 1 = files changed during archive, // 2 = errors (e.g. permission denied) but archive still written to stdout. @@ -1287,11 +1313,11 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = const tarExitedWithData = result.stdout && result.stdout.length > 0 && - (markedTar.status === 0 || markedTar.status === 1 || markedTar.status === 2); + (tarStatus === 0 || tarStatus === 1 || tarStatus === 2); - if (markedTar.status !== 0 && result.stdout && result.stdout.length > 0) { + if (tarStatus !== 0 && result.stdout && result.stdout.length > 0) { _log( - `tar exited ${markedTar.status} but produced ${result.stdout.length} bytes — attempting partial extraction`, + `tar exited ${tarStatus} but produced ${result.stdout.length} bytes; attempting partial extraction`, ); } @@ -1300,7 +1326,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = const extractResult = safeTarExtract(result.stdout, backupPath); if (extractResult.success) { const extractedDirs = new Set(existingBackupDirs(backupPath, existingDirs)); - if (markedTar.status === 0) { + if (tarStatus === 0) { for (const d of existingDirs) { if (extractedDirs.has(d)) { backedUpDirs.push(d); @@ -1316,7 +1342,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = ); if (tarFailedDirs.size === 0) { _log( - `tar exited ${markedTar.status} without attributable failed dirs — marking all dirs failed`, + `tar exited ${tarStatus} without attributable failed dirs; marking all dirs failed`, ); failedDirs.push(...existingDirs); } else { @@ -1530,7 +1556,16 @@ export function restoreSandboxState(sandboxName: string, backupPath: string): Re } const markedExtract = statusFromMarker(extractResult, RESTORE_EXTRACT_STATUS_MARKER); - if (markedExtract.status === 0) { + const extractStatus = + !markedExtract.found && + extractResult.status === 1 && + markedExtract.stderr.trim().length === 0 + ? 0 + : markedExtract.status; + if (extractStatus !== markedExtract.status) { + _log("gRPC tar restore missing exit marker; validating restored dirs with usability probe"); + } + if (extractStatus === 0) { const restoredPaths = localDirs.map((d) => `${dir}/${d}`); // Best-effort only: OpenShell gRPC exec normally runs as the sandbox user, diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index abe151dca7..c3f1ccf6d3 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -627,6 +627,68 @@ process.exit(0); } }); + it("accepts a valid tar archive when OpenShell omits the binary exit marker", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-grpc-tar-no-marker-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const openclawDir = path.join(fixture, "sandbox-root", ".openclaw"); + const existingDirs = ["agents", "workspace", "skills"]; + fs.mkdirSync(binDir, { recursive: true }); + for (const dirName of existingDirs) { + fs.mkdirSync(path.join(openclawDir, dirName), { recursive: true }); + } + fs.writeFileSync(path.join(openclawDir, "workspace", "marker.txt"), "marker\n"); + + const openshell = writeFakeOpenshell(binDir); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const cmd = process.argv[process.argv.length - 1] || ""; +const existingDirs = ${JSON.stringify(existingDirs)}; +if (cmd.includes("[ -d ")) { + process.stdout.write(existingDirs.join("\\n") + "\\n"); + process.stderr.write("\\n__NEMOCLAW_DIR_DISCOVERY_STATUS__:0\\n"); + process.exit(1); +} +if (cmd.includes("find ")) { + process.stderr.write("\\n__NEMOCLAW_PRE_BACKUP_AUDIT_STATUS__:0\\n"); + process.exit(1); +} +if (cmd.includes("tar -cf -")) { + const r = spawnSync("tar", ["-cf", "-", "-C", ${JSON.stringify(openclawDir)}, ...existingDirs], { + stdio: ["ignore", "pipe", "pipe"], + }); + if (r.stdout) fs.writeSync(1, r.stdout); + // Simulate OpenShell ending the stream without the final exit event/stderr marker. + process.exit(1); +} +process.exit(0); +`, + ); + + writeOpenClawRegistry("alpha"); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}:${oldPath || ""}`; + + const backup = sandboxState.backupSandboxState("alpha"); + expect(backup.success).toBe(true); + expect(backup.failedDirs).toEqual([]); + expect(backup.backedUpDirs).toEqual(existingDirs); + } finally { + if (oldOpenshell === undefined) { + delete process.env.NEMOCLAW_OPENSHELL_BIN; + } else { + process.env.NEMOCLAW_OPENSHELL_BIN = oldOpenshell; + } + process.env.PATH = oldPath; + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + it("excludes tar-failed directories from the restorable manifest", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-partial-tar-")); const oldPath = process.env.PATH; From 2c40a23e86c6473782fa8d345be5eabc381ca1cf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 15:37:06 -0700 Subject: [PATCH 14/26] Stabilize gRPC dashboard forward reuse --- .../openshell/forward-bridge-state.ts | 28 +++++++++++++++++++ src/lib/onboard.ts | 5 +++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index 4a3a4c4e45..6c2cc62fc2 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -117,6 +117,31 @@ function waitForPidExit(pid: number, timeoutMs = 2_000): boolean { return !isPidAlive(pid); } +function canBindForwardPort(bind: string, port: number): boolean { + const host = bind === "0.0.0.0" || bind === "::" ? "127.0.0.1" : bind; + const script = + "const net=require('node:net');" + + "const srv=net.createServer();" + + "let done=false;" + + "const finish=(code)=>{if(done)return;done=true;process.exit(code);};" + + "srv.once('error',()=>finish(1));" + + `srv.listen(${port}, ${JSON.stringify(host)}, () => srv.close(() => finish(0)));`; + const result = spawnSync(process.execPath, ["-e", script], { + stdio: "ignore", + timeout: 1_500, + }); + return result.status === 0; +} + +function waitForPortRelease(bind: string, port: number, timeoutMs = 5_000): boolean { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (canBindForwardPort(bind, port)) return true; + sleepMs(100); + } + return canBindForwardPort(bind, port); +} + export function stopForwardBridge(sandboxName: string, port: number | string): boolean { const state = readStateFile(statePath(sandboxName, port)); if (state && isPidAlive(state.pid) && !isTestForwardPid(state.pid)) { @@ -133,6 +158,7 @@ export function stopForwardBridge(sandboxName: string, port: number | string): b } waitForPidExit(state.pid, 1_000); } + waitForPortRelease(state.bind, state.port); } removeForwardState(sandboxName, port); return Boolean(state); @@ -282,6 +308,8 @@ export function startForwardBridgeDetached( } export const __forwardBridgeTestHooks = { + canBindForwardPort, probeForwardReady, + waitForPortRelease, waitForPidExit, }; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 761c3c2454..2534ac9f97 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -58,6 +58,9 @@ const { agentSupportsWebSearch, }: typeof import("./onboard/web-search-support") = require("./onboard/web-search-support"); const onboardDashboard: typeof import("./onboard/dashboard") = require("./onboard/dashboard"); +const { + forwardStatesAsListOutput, +}: typeof import("./adapters/openshell/forward-bridge-state") = require("./adapters/openshell/forward-bridge-state"); const { buildGatewayBootstrapSecretsScript, createGatewayBootstrapRepairHelpers, @@ -2797,7 +2800,7 @@ async function createSandbox( } const preferredPort = controlUiPort ?? envPort ?? persistedPort ?? (agent ? agent.forwardPort : DASHBOARD_PORT); - const earlyForwards = runCaptureOpenshell(["forward", "list"], { ignoreError: true }); + const earlyForwards = forwardStatesAsListOutput(); const effectivePort = findAvailableDashboardPort(sandboxName, preferredPort, earlyForwards); if (effectivePort !== preferredPort) { console.warn(` ! Port ${preferredPort} is taken. Using port ${effectivePort} instead.`); From a4803c2e28f77ad95636669621cf362e66d60e32 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 15:51:49 -0700 Subject: [PATCH 15/26] Reclaim stale gRPC dashboard listeners --- src/lib/onboard.ts | 23 ++++++++- src/lib/onboard/dashboard.ts | 49 ++++++++++++++++--- src/lib/onboard/forward-cleanup.test.ts | 48 ++++++++++++++++++ src/lib/onboard/forward-cleanup.ts | 30 ++++++++++++ src/lib/onboard/stale-gateway-cleanup.test.ts | 29 +++++++++++ src/lib/onboard/stale-gateway-cleanup.ts | 13 +++++ 6 files changed, 183 insertions(+), 9 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 2534ac9f97..ef81db3eb9 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -25,7 +25,10 @@ const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gatew const { ensureOllamaLoopbackSystemdOverride, }: typeof import("./onboard/ollama-systemd") = require("./onboard/ollama-systemd"); -const { bestEffortForwardStop } = require("./onboard/forward-cleanup"); +const { + bestEffortForwardStop, + stopTrackedGrpcForwardBridgeForPort, +} = require("./onboard/forward-cleanup"); const { CUSTOM_BUILD_CONTEXT_WARN_BYTES, isInsideIgnoredCustomBuildContextPath, @@ -2073,6 +2076,24 @@ async function preflight( continue; } } + if ( + port === DASHBOARD_PORT && + (gatewayReuseState === "stale" || + gatewayReuseState === "missing" || + gatewayReuseState === "active-unnamed") + ) { + const stoppedBridge = stopTrackedGrpcForwardBridgeForPort(port, { pid: portCheck.pid }); + if (stoppedBridge) { + console.log( + ` Cleaning up stale gRPC dashboard bridge on port ${port} for sandbox '${stoppedBridge.sandboxName}'...`, + ); + portCheck = await checkPortAvailable(port, portCheckOptions); + if (portCheck.ok) { + console.log(` ✓ Port ${port} available after stale gRPC bridge cleanup (${label})`); + continue; + } + } + } // Auto-cleanup orphaned SSH port-forward from a previous NemoClaw session // (e.g. dashboard forward left behind after destroy). Only kill the process // if its command line contains "openshell" to avoid killing unrelated SSH diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 3fcc50de5c..29841b1a41 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -22,6 +22,10 @@ import { stopAllForwardBridges, stopForwardBridge, } from "../adapters/openshell/forward-bridge-state"; +import { + getProtectedDashboardPortsForSandbox, + stopStaleDashboardListeners, +} from "./stale-gateway-cleanup"; const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; export const CONTROL_UI_PORT = DASHBOARD_PORT; @@ -230,6 +234,20 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa process.exit(1); } + function stopLegacyForwardOnPreferredPort(sandboxName: string, preferredPort: number): boolean { + const result = stopStaleDashboardListeners( + {}, + { + extraPorts: [preferredPort], + protectedPorts: getProtectedDashboardPortsForSandbox( + registry.listSandboxes().sandboxes, + sandboxName, + ), + }, + ); + return result.stopped.length > 0; + } + function ensureDashboardForward( sandboxName: string, chatUiUrl = `http://127.0.0.1:${CONTROL_UI_PORT}`, @@ -257,15 +275,30 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa if (actualPort !== preferredPort) { if (rollbackSandboxOnFailure) { - const err = new Error( - `Dashboard port ${preferredPort} became host-bound during sandbox build; ` + - `cannot reallocate to ${actualPort} after the sandbox has been created with ` + - `CHAT_UI_URL=${preferredPort}. Free the port and re-run \`${deps.cliName()} onboard\`, ` + - `or pass \`--control-ui-port \` to pick a different dashboard port.`, - ); - rollbackSandboxAndExit(sandboxName, err); + if (stopLegacyForwardOnPreferredPort(sandboxName, preferredPort)) { + existingForwards = forwardStatesAsListOutput(); + try { + actualPort = findAvailableDashboardPort(sandboxName, preferredPort, existingForwards); + } catch (err) { + rollbackSandboxAndExit(sandboxName, err); + } + if (actualPort === preferredPort) { + console.warn(` Reclaimed stale dashboard listener on port ${preferredPort}.`); + } + } + } + if (actualPort !== preferredPort) { + if (rollbackSandboxOnFailure) { + const err = new Error( + `Dashboard port ${preferredPort} became host-bound during sandbox build; ` + + `cannot reallocate to ${actualPort} after the sandbox has been created with ` + + `CHAT_UI_URL=${preferredPort}. Free the port and re-run \`${deps.cliName()} onboard\`, ` + + `or pass \`--control-ui-port \` to pick a different dashboard port.`, + ); + rollbackSandboxAndExit(sandboxName, err); + } + console.warn(` ! Port ${preferredPort} is taken. Using port ${actualPort} instead.`); } - console.warn(` ! Port ${preferredPort} is taken. Using port ${actualPort} instead.`); } const occupied = getOccupiedPorts(existingForwards); diff --git a/src/lib/onboard/forward-cleanup.test.ts b/src/lib/onboard/forward-cleanup.test.ts index f43c2bfbd4..a0d053b4da 100644 --- a/src/lib/onboard/forward-cleanup.test.ts +++ b/src/lib/onboard/forward-cleanup.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; import { bestEffortForwardStop, bestEffortForwardStopForSandbox, + stopTrackedGrpcForwardBridgeForPort, } from "../../../dist/lib/onboard/forward-cleanup"; function forwardListWith(entries: Array<{ sandbox: string; port: number; status?: string }>): string { @@ -113,3 +114,50 @@ describe("bestEffortForwardStopForSandbox", () => { expect(run).toHaveBeenCalledTimes(1); }); }); + +describe("stopTrackedGrpcForwardBridgeForPort", () => { + it("stops a tracked gRPC bridge when its port and PID match the listener", () => { + const stop = vi.fn(() => true); + const state = { + sandboxName: "my-sandbox", + bind: "127.0.0.1", + port: 18789, + targetHost: "127.0.0.1", + targetPort: 4000, + pid: 4242, + startedAt: "2026-05-27T00:00:00.000Z", + }; + + const stopped = stopTrackedGrpcForwardBridgeForPort(18789, { + pid: 4242, + listForwardStates: () => [state], + stopForwardBridge: stop, + }); + + expect(stopped).toEqual(state); + expect(stop).toHaveBeenCalledWith("my-sandbox", 18789); + }); + + it("leaves a tracked bridge alone when the port listener PID differs", () => { + const stop = vi.fn(() => true); + + const stopped = stopTrackedGrpcForwardBridgeForPort(18789, { + pid: 9999, + listForwardStates: () => [ + { + sandboxName: "my-sandbox", + bind: "127.0.0.1", + port: 18789, + targetHost: "127.0.0.1", + targetPort: 4000, + pid: 4242, + startedAt: "2026-05-27T00:00:00.000Z", + }, + ], + stopForwardBridge: stop, + }); + + expect(stopped).toBeNull(); + expect(stop).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/forward-cleanup.ts b/src/lib/onboard/forward-cleanup.ts index 5ccffe71f5..359e74320a 100644 --- a/src/lib/onboard/forward-cleanup.ts +++ b/src/lib/onboard/forward-cleanup.ts @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { + listForwardStates, + stopForwardBridge, + type SandboxForwardState, +} from "../adapters/openshell/forward-bridge-state"; import { getOccupiedPorts } from "./dashboard-port"; @@ -15,6 +20,12 @@ export type ForwardListRunner = ( opts: { ignoreError?: boolean; timeout?: number }, ) => string; +export interface TrackedGrpcForwardBridgeCleanupOptions { + pid?: number | null; + listForwardStates?: () => SandboxForwardState[]; + stopForwardBridge?: (sandboxName: string, port: number | string) => boolean; +} + /** * `openshell forward stop ` — port-scoped, kills whatever forward is * currently bound to that port. Use only when the caller has no sandbox @@ -87,3 +98,22 @@ export function bestEffortForwardStopForSandbox( }); return owner === sandboxName ? "stopped" : "no-entry"; } + +export function stopTrackedGrpcForwardBridgeForPort( + port: number | string, + options: TrackedGrpcForwardBridgeCleanupOptions = {}, +): SandboxForwardState | null { + const numericPort = Number(port); + if (!Number.isInteger(numericPort)) return null; + + const states = (options.listForwardStates ?? listForwardStates)(); + const state = states.find( + (entry) => + entry.port === numericPort && + (options.pid == null || entry.pid === options.pid), + ); + if (!state) return null; + + (options.stopForwardBridge ?? stopForwardBridge)(state.sandboxName, state.port); + return state; +} diff --git a/src/lib/onboard/stale-gateway-cleanup.test.ts b/src/lib/onboard/stale-gateway-cleanup.test.ts index d85426da0e..b85ad54aed 100644 --- a/src/lib/onboard/stale-gateway-cleanup.test.ts +++ b/src/lib/onboard/stale-gateway-cleanup.test.ts @@ -115,6 +115,35 @@ describe("stopStaleDashboardListeners", () => { expect(log).toHaveBeenCalledWith(expect.stringContaining("Stopped stale dashboard gateway listener 2522044")); }); + it("can scan an agent dashboard port outside the OpenClaw dashboard range", () => { + const kill = vi.fn<(pid: number, signal?: NodeJS.Signals | number) => boolean>(() => true); + let pidGone = false; + const responses = new Map RunResult)>([ + ["lsof -ti :8642 -sTCP:LISTEN", { status: 0, stdout: "864200\n", stderr: "" }], + ["ps -p 864200 -o user=", { status: 0, stdout: "tester\n", stderr: "" }], + [ + "ps -p 864200 -o args=", + { status: 0, stdout: "openshell-forward --port 8642\n", stderr: "" }, + ], + [ + "ps -p 864200 -o pid=", + () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "864200\n", stderr: "" }), + ], + ]); + const { run } = makeRun(responses); + const customKill: StaleGatewayDeps["kill"] = (pid, signal) => { + kill(pid, signal); + if (signal === "SIGTERM") pidGone = true; + return true; + }; + const result = stopStaleDashboardListeners( + { ...baseDeps({ run, kill: customKill }) }, + { extraPorts: [8642] }, + ); + expect(result.stopped).toEqual([864200]); + expect(kill).toHaveBeenCalledWith(864200, "SIGTERM"); + }); + it("escalates to SIGKILL when SIGTERM does not free the process", () => { const sentSignals: NodeJS.Signals[] = []; let pidGone = false; diff --git a/src/lib/onboard/stale-gateway-cleanup.ts b/src/lib/onboard/stale-gateway-cleanup.ts index a0eb2cbd3c..8e8884ec25 100644 --- a/src/lib/onboard/stale-gateway-cleanup.ts +++ b/src/lib/onboard/stale-gateway-cleanup.ts @@ -66,6 +66,11 @@ export interface StaleGatewayOptions { * disrupt the forward of an existing sandbox (#3260). */ protectedPorts?: Iterable; + /** + * Additional non-default dashboard ports to scan. Agent sandboxes such as + * Hermes can use fixed service ports outside the OpenClaw dashboard range. + */ + extraPorts?: Iterable; } const CMDLINE_MARKERS = ["openclaw-gateway", "openshell-forward", "openshell forward"]; @@ -246,7 +251,15 @@ export function stopStaleDashboardListeners( if (deps.commandExists && !deps.commandExists("lsof")) return result; const seen = new Set(); + const portsToScan = new Set(); for (let port = DASHBOARD_PORT_RANGE_START; port <= DASHBOARD_PORT_RANGE_END; port += 1) { + portsToScan.add(port); + } + for (const port of options.extraPorts ?? []) { + if (Number.isFinite(port)) portsToScan.add(port); + } + + for (const port of portsToScan) { if (protectedPorts.has(port)) { const pids = lsofPidsForPort(port, deps); if (pids.length > 0) { From e6b5d07beead51c027a8541325967fcf60783e87 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 15:55:34 -0700 Subject: [PATCH 16/26] Move preflight dashboard recovery into module --- src/lib/onboard.ts | 55 ++++---------- .../onboard/preflight-port-recovery.test.ts | 61 +++++++++++++++ src/lib/onboard/preflight-port-recovery.ts | 75 +++++++++++++++++++ 3 files changed, 149 insertions(+), 42 deletions(-) create mode 100644 src/lib/onboard/preflight-port-recovery.test.ts create mode 100644 src/lib/onboard/preflight-port-recovery.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index ef81db3eb9..59060cd2c2 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -25,10 +25,8 @@ const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gatew const { ensureOllamaLoopbackSystemdOverride, }: typeof import("./onboard/ollama-systemd") = require("./onboard/ollama-systemd"); -const { - bestEffortForwardStop, - stopTrackedGrpcForwardBridgeForPort, -} = require("./onboard/forward-cleanup"); +const { bestEffortForwardStop } = require("./onboard/forward-cleanup"); +const { recoverPreflightDashboardPort } = require("./onboard/preflight-port-recovery"); const { CUSTOM_BUILD_CONTEXT_WARN_BYTES, isInsideIgnoredCustomBuildContextPath, @@ -2076,44 +2074,17 @@ async function preflight( continue; } } - if ( - port === DASHBOARD_PORT && - (gatewayReuseState === "stale" || - gatewayReuseState === "missing" || - gatewayReuseState === "active-unnamed") - ) { - const stoppedBridge = stopTrackedGrpcForwardBridgeForPort(port, { pid: portCheck.pid }); - if (stoppedBridge) { - console.log( - ` Cleaning up stale gRPC dashboard bridge on port ${port} for sandbox '${stoppedBridge.sandboxName}'...`, - ); - portCheck = await checkPortAvailable(port, portCheckOptions); - if (portCheck.ok) { - console.log(` ✓ Port ${port} available after stale gRPC bridge cleanup (${label})`); - continue; - } - } - } - // Auto-cleanup orphaned SSH port-forward from a previous NemoClaw session - // (e.g. dashboard forward left behind after destroy). Only kill the process - // if its command line contains "openshell" to avoid killing unrelated SSH - // tunnels the user may have set up on the same port. (#1950) - if (port === DASHBOARD_PORT && portCheck.process === "ssh" && portCheck.pid) { - // Use `ps` to get the command line — works on Linux, macOS, and WSL. - const cmdline = captureProcessArgs(portCheck.pid); - if (cmdline.includes("openshell")) { - console.log( - ` Cleaning up orphaned SSH port-forward on port ${port} (PID ${portCheck.pid})...`, - ); - run(["kill", String(portCheck.pid)], { ignoreError: true }); - sleepSeconds(1); - portCheck = await checkPortAvailable(port, portCheckOptions); - if (portCheck.ok) { - console.log(` ✓ Port ${port} available after orphaned forward cleanup (${label})`); - continue; - } - } - } + portCheck = await recoverPreflightDashboardPort({ + port, + label, + gatewayReuseState, + portCheck, + portCheckOptions, + checkPortAvailable, + captureProcessArgs, + run, + }); + if (portCheck.ok) continue; console.error(""); console.error(` !! Port ${port} is not available.`); console.error(` ${label} needs this port.`); diff --git a/src/lib/onboard/preflight-port-recovery.test.ts b/src/lib/onboard/preflight-port-recovery.test.ts new file mode 100644 index 0000000000..814a54229e --- /dev/null +++ b/src/lib/onboard/preflight-port-recovery.test.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { recoverPreflightDashboardPort } from "../../../dist/lib/onboard/preflight-port-recovery"; + +describe("recoverPreflightDashboardPort", () => { + it("reclaims a tracked gRPC dashboard bridge when gateway metadata is stale", async () => { + const log = vi.fn(); + const checkPortAvailable = vi.fn(async () => ({ ok: true })); + const stopTrackedGrpcForwardBridgeForPort = vi.fn(() => ({ + sandboxName: "old-sandbox", + bind: "127.0.0.1", + port: 18789, + targetHost: "127.0.0.1", + targetPort: 4000, + pid: 4242, + startedAt: "2026-05-27T00:00:00.000Z", + })); + + const result = await recoverPreflightDashboardPort({ + port: 18789, + label: "NemoClaw dashboard", + gatewayReuseState: "stale", + portCheck: { ok: false, process: "node", pid: 4242 }, + checkPortAvailable, + captureProcessArgs: vi.fn(), + run: vi.fn(), + stopTrackedGrpcForwardBridgeForPort, + log, + }); + + expect(result.ok).toBe(true); + expect(stopTrackedGrpcForwardBridgeForPort).toHaveBeenCalledWith(18789, { pid: 4242 }); + expect(log).toHaveBeenCalledWith( + " Cleaning up stale gRPC dashboard bridge on port 18789 for sandbox 'old-sandbox'...", + ); + }); + + it("preserves the existing orphaned SSH forward cleanup behavior", async () => { + const run = vi.fn(); + const sleep = vi.fn(); + const checkPortAvailable = vi.fn(async () => ({ ok: true })); + + const result = await recoverPreflightDashboardPort({ + port: 18789, + label: "NemoClaw dashboard", + gatewayReuseState: "healthy", + portCheck: { ok: false, process: "ssh", pid: 1234 }, + checkPortAvailable, + captureProcessArgs: vi.fn(() => "ssh -L 18789:localhost:4000 openshell"), + run, + sleep, + }); + + expect(result.ok).toBe(true); + expect(run).toHaveBeenCalledWith(["kill", "1234"], { ignoreError: true }); + expect(sleep).toHaveBeenCalledWith(1); + }); +}); diff --git a/src/lib/onboard/preflight-port-recovery.ts b/src/lib/onboard/preflight-port-recovery.ts new file mode 100644 index 0000000000..b60432eced --- /dev/null +++ b/src/lib/onboard/preflight-port-recovery.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayReuseState } from "../state/gateway"; +import type { CheckPortOpts, PortProbeResult } from "./preflight"; +import { DASHBOARD_PORT } from "../core/ports"; +import { sleepSeconds } from "../core/wait"; + +import { stopTrackedGrpcForwardBridgeForPort } from "./forward-cleanup"; + +type CheckPortAvailable = (port?: number, opts?: CheckPortOpts) => Promise; +type RunCommand = (args: string[], options: { ignoreError: true }) => unknown; + +export interface RecoverPreflightDashboardPortOptions { + port: number; + label: string; + gatewayReuseState: GatewayReuseState; + portCheck: PortProbeResult; + portCheckOptions?: CheckPortOpts; + checkPortAvailable: CheckPortAvailable; + captureProcessArgs: (pid: number) => string; + run: RunCommand; + stopTrackedGrpcForwardBridgeForPort?: typeof stopTrackedGrpcForwardBridgeForPort; + log?: (message: string) => void; + sleep?: (seconds: number) => void; +} + +function canReclaimStaleGrpcBridge(gatewayReuseState: GatewayReuseState): boolean { + return ( + gatewayReuseState === "stale" || + gatewayReuseState === "missing" || + gatewayReuseState === "active-unnamed" + ); +} + +export async function recoverPreflightDashboardPort( + options: RecoverPreflightDashboardPortOptions, +): Promise { + if (options.port !== DASHBOARD_PORT) return options.portCheck; + + const log = options.log ?? console.log; + let portCheck = options.portCheck; + if (canReclaimStaleGrpcBridge(options.gatewayReuseState)) { + const stopBridge = + options.stopTrackedGrpcForwardBridgeForPort ?? stopTrackedGrpcForwardBridgeForPort; + const stoppedBridge = stopBridge(options.port, { + pid: portCheck.pid, + }); + if (stoppedBridge) { + log( + ` Cleaning up stale gRPC dashboard bridge on port ${options.port} for sandbox '${stoppedBridge.sandboxName}'...`, + ); + portCheck = await options.checkPortAvailable(options.port, options.portCheckOptions); + if (portCheck.ok) { + log(` ✓ Port ${options.port} available after stale gRPC bridge cleanup (${options.label})`); + return portCheck; + } + } + } + + if (portCheck.process === "ssh" && portCheck.pid) { + const cmdline = options.captureProcessArgs(portCheck.pid); + if (cmdline.includes("openshell")) { + log(` Cleaning up orphaned SSH port-forward on port ${options.port} (PID ${portCheck.pid})...`); + options.run(["kill", String(portCheck.pid)], { ignoreError: true }); + (options.sleep ?? sleepSeconds)(1); + portCheck = await options.checkPortAvailable(options.port, options.portCheckOptions); + if (portCheck.ok) { + log(` ✓ Port ${options.port} available after orphaned forward cleanup (${options.label})`); + } + } + } + + return portCheck; +} From 19ada2a46ba2dba6cfc5706937b118945b1e5cbc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 16:20:15 -0700 Subject: [PATCH 17/26] Treat gRPC forward bridge readiness as TCP readiness --- .../openshell/forward-bridge-state.test.ts | 16 ++++++++++++++++ .../adapters/openshell/forward-bridge-state.ts | 7 +++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/lib/adapters/openshell/forward-bridge-state.test.ts b/src/lib/adapters/openshell/forward-bridge-state.test.ts index d428e1620d..4987185178 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.test.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.test.ts @@ -57,6 +57,22 @@ describe("OpenShell gRPC forward bridge readiness", () => { expect(__forwardBridgeTestHooks.probeForwardReady("127.0.0.1", port)).toBe(false); }); + it("treats local TCP reachability as bridge readiness without requiring dashboard HTTP", async () => { + const port = await unusedLocalPort(); + const server = net.createServer((socket) => socket.end("not-http")); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve()); + }); + try { + expect(__forwardBridgeTestHooks.probeForwardReady("127.0.0.1", port)).toBe(true); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); + it("waits for a stopped bridge process to release its local port", async () => { const originalHome = process.env.HOME; const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-forward-stop-")); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index 6c2cc62fc2..4bbb084ad4 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -215,14 +215,13 @@ function probeForwardReady(bind: string, port: number): boolean { const script = "const net=require('node:net');" + `const socket=net.createConnection({host:${JSON.stringify(host)},port:${port}});` + - "let data='';let done=false;" + + "let done=false;" + "const finish=(code)=>{if(done)return;done=true;socket.destroy();process.exit(code);};" + "socket.setTimeout(1000);" + - "socket.on('connect',()=>socket.write('GET /health HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n'));" + - "socket.on('data',(chunk)=>{data+=chunk.toString('utf8');if(/^HTTP\\//.test(data))finish(0);});" + + "socket.on('connect',()=>finish(0));" + "socket.on('error',()=>finish(1));" + "socket.on('timeout',()=>finish(1));" + - "socket.on('end',()=>finish(/^HTTP\\//.test(data)?0:1));"; + "socket.on('end',()=>finish(1));"; const result = spawnSync(process.execPath, ["-e", script], { stdio: "ignore", timeout: 1500, From 6b202d59726b1b36a6e641994f7a2ef903b99369 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 16:59:11 -0700 Subject: [PATCH 18/26] Remove legacy OpenShell forward restore --- scripts/install.sh | 94 +---- .../openshell/grpc-migration-guard.test.ts | 2 + src/lib/adapters/openshell/sync-runner.ts | 11 +- src/lib/onboard/dashboard-access.test.ts | 12 - src/lib/onboard/dashboard-access.ts | 18 - src/lib/onboard/forward-start.test.ts | 385 ------------------ src/lib/onboard/forward-start.ts | 331 --------------- test/install-preflight.test.ts | 6 +- 8 files changed, 17 insertions(+), 842 deletions(-) delete mode 100644 src/lib/onboard/forward-start.test.ts delete mode 100644 src/lib/onboard/forward-start.ts diff --git a/scripts/install.sh b/scripts/install.sh index 193c1399b8..16f591e245 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -274,24 +274,15 @@ resolve_onboarded_agent() { } restore_onboard_forward_after_post_checks() { - local sandbox_name agent_name agent_display port openshell_bin attempt state_dir pid_file watcher_script watcher_pid + local sandbox_name agent_name port state_dir pid_file sandbox_name="$(resolve_default_sandbox_name)" agent_name="$(resolve_onboarded_agent)" - agent_display="$(agent_display_name "$agent_name")" case "$agent_name" in hermes) port=8642 ;; *) return 0 ;; esac - if [[ -n "${NEMOCLAW_OPENSHELL_BIN:-}" && -x "$NEMOCLAW_OPENSHELL_BIN" ]]; then - openshell_bin="$NEMOCLAW_OPENSHELL_BIN" - elif command_exists openshell; then - openshell_bin="$(command -v openshell)" - else - return 0 - fi - state_dir="${HOME}/.nemoclaw/state" mkdir -p "$state_dir" 2>/dev/null || true pid_file="${state_dir}/${agent_name}-${sandbox_name}-${port}.forward.pid" @@ -310,86 +301,9 @@ restore_onboard_forward_after_post_checks() { rm -f "$pid_file" fi - stop_agent_forward_if_owned() { - local forward_list owner status - "$openshell_bin" forward stop "$port" "$sandbox_name" >/dev/null 2>&1 && return 0 - forward_list="$("$openshell_bin" forward list 2>/dev/null || true)" - owner="$(awk -v sandbox="$sandbox_name" -v port="$port" ' - $1 == sandbox && $3 == port { - print $1 - exit - } - ' <<<"$forward_list")" - status="$(awk -v sandbox="$sandbox_name" -v port="$port" ' - $1 == sandbox && $3 == port { - print tolower($5) - exit - } - ' <<<"$forward_list")" - if [[ "$owner" == "$sandbox_name" && ("$status" == "running" || "$status" == "active") ]]; then - "$openshell_bin" forward stop "$port" "$sandbox_name" >/dev/null 2>&1 || true - fi - } - - for attempt in 1 2 3; do - stop_agent_forward_if_owned - if [ "$attempt" -gt 1 ]; then - sleep 2 - fi - "$openshell_bin" forward start --background "$port" "$sandbox_name" >/dev/null 2>&1 || true - watcher_pid="" - if [[ "${NEMOCLAW_SKIP_FORWARD_WATCHER:-}" != "1" ]] && command_exists node; then - watcher_script="${pid_file}.js" - cat >"$watcher_script" <<'NODE' -const { spawnSync } = require("child_process"); -const [openshellBin, port, sandboxName] = process.argv.slice(2); -function run(args) { - spawnSync(openshellBin, args, { stdio: "ignore" }); -} -function healthy() { - return spawnSync("curl", ["-sf", "--max-time", "3", `http://127.0.0.1:${port}/health`], { - stdio: "ignore", - }).status === 0; -} -function tick() { - if (healthy()) return; - run(["forward", "stop", port, sandboxName]); - run(["forward", "start", "--background", port, sandboxName]); -} -tick(); -setInterval(tick, 10_000); -NODE - node -e ' - const { spawn } = require("child_process"); - const fs = require("fs"); - const [script, openshellBin, port, sandboxName, pidFile] = process.argv.slice(1); - const child = spawn(process.execPath, [script, openshellBin, port, sandboxName], { - detached: true, - stdio: "ignore", - }); - fs.writeFileSync(pidFile, String(child.pid) + "\n"); - child.unref(); - ' "$watcher_script" "$openshell_bin" "$port" "$sandbox_name" "$pid_file" \ - >/dev/null 2>&1 || true - fi - sleep 4 - if command_exists curl \ - && curl -sf --max-time 3 "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then - return 0 - fi - watcher_pid="$(cat "$pid_file" 2>/dev/null || true)" - if ! command_exists curl && [[ -n "$watcher_pid" ]] && kill -0 "$watcher_pid" >/dev/null 2>&1; then - return 0 - fi - if [[ -n "$watcher_pid" ]]; then - kill "$watcher_pid" >/dev/null 2>&1 || true - fi - rm -f "$pid_file" - done - - warn "Could not restore ${agent_display} host forward on port ${port}." - warn "Run: openshell forward start --background ${port} ${sandbox_name}" - return 1 + # gRPC-only NemoClaw owns dashboard forwarding from the CLI itself. The + # installer must not resurrect OpenShell's SSH-backed forward helper here. + return 0 } # step N "Description" — numbered section header diff --git a/src/lib/adapters/openshell/grpc-migration-guard.test.ts b/src/lib/adapters/openshell/grpc-migration-guard.test.ts index 9a1abbeda4..7c8d9663b2 100644 --- a/src/lib/adapters/openshell/grpc-migration-guard.test.ts +++ b/src/lib/adapters/openshell/grpc-migration-guard.test.ts @@ -13,6 +13,7 @@ const PRODUCTION_TARGETS = [ "src/lib/state/sandbox.ts", "src/lib/skill-install.ts", "src/lib/onboard/dashboard.ts", + "src/lib/onboard/dashboard-access.ts", "src/lib/onboard/sandbox-verification-exec.ts", "src/lib/sandbox/version.ts", "src/lib/share-command.ts", @@ -20,6 +21,7 @@ const PRODUCTION_TARGETS = [ "src/lib/status-command-deps.ts", "src/lib/tunnel/services.ts", "src/lib/verify-deployment.ts", + "scripts/install.sh", ]; const BANNED = [ diff --git a/src/lib/adapters/openshell/sync-runner.ts b/src/lib/adapters/openshell/sync-runner.ts index c6b082dca4..b58cb37c7f 100644 --- a/src/lib/adapters/openshell/sync-runner.ts +++ b/src/lib/adapters/openshell/sync-runner.ts @@ -24,7 +24,14 @@ function shellQuoteArg(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } -function legacyFakeRemoteCommand(argv: readonly string[]): string { +function legacyFakeRemoteCommand(argv: readonly string[], input: Buffer = Buffer.alloc(0)): string { + if ( + input.length > 0 && + (argv[0] === "sh" || argv[0] === "bash") && + (argv[1] === "-s" || argv[1] === "-") + ) { + return input.toString("utf-8"); + } if ((argv[0] === "sh" || argv[0] === "bash") && (argv[1] === "-c" || argv[1] === "-lc")) { return argv[2] ?? ""; } @@ -43,7 +50,7 @@ function runLegacyFakeSshTransport( } const sshBin = process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN || "ssh"; - const remoteCommand = legacyFakeRemoteCommand(request.argv); + const remoteCommand = legacyFakeRemoteCommand(request.argv, input); const helperArgs = [`openshell-${request.sandboxName}`, remoteCommand]; const helperCommand = /\.(?:c|m)?js$/i.test(sshBin) ? process.execPath : sshBin; const helperArgv = helperCommand === process.execPath ? [sshBin, ...helperArgs] : helperArgs; diff --git a/src/lib/onboard/dashboard-access.test.ts b/src/lib/onboard/dashboard-access.test.ts index e8557f85a1..5f8f09d82f 100644 --- a/src/lib/onboard/dashboard-access.test.ts +++ b/src/lib/onboard/dashboard-access.test.ts @@ -8,7 +8,6 @@ import { dashboardUrlForDisplay, getDashboardAccessInfo, getDashboardForwardPort, - getDashboardForwardStartCommand, getDashboardForwardTarget, getDashboardGuidanceLines, getWslHostAddress, @@ -21,17 +20,6 @@ describe("dashboard access helpers", () => { expect(getDashboardForwardTarget("http://10.0.0.25:18789", { isWsl: false })).toBe("0.0.0.0:18789"); }); - it("builds the OpenShell forward start command with the resolved target", () => { - const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); - - expect( - getDashboardForwardStartCommand("alpha", { - chatUiUrl: "http://10.0.0.25:18789", - openshellShellCommand, - }), - ).toBe("openshell forward start --background 0.0.0.0:18789 alpha"); - }); - it("redacts token fragments for display", () => { expect(buildAuthenticatedDashboardUrl("http://127.0.0.1:18789/", "secret token")).toBe( "http://127.0.0.1:18789/#token=secret%20token", diff --git a/src/lib/onboard/dashboard-access.ts b/src/lib/onboard/dashboard-access.ts index b417af6975..ebdf3788df 100644 --- a/src/lib/onboard/dashboard-access.ts +++ b/src/lib/onboard/dashboard-access.ts @@ -7,15 +7,12 @@ import { DASHBOARD_PORT } from "../core/ports"; import { buildChain, buildControlUiUrls } from "../dashboard/contract"; type RunCapture = (args: string[], options: { ignoreError: true }) => string; -type OpenshellShellCommand = (args: string[], options?: { openshellBinary?: string }) => string; export type DashboardAccessOptions = WslDetectionOptions & { chatUiUrl?: string; token?: string | null; wslHostAddress?: string | null; runCapture?: RunCapture; - openshellBinary?: string; - openshellShellCommand?: OpenshellShellCommand; fetchGatewayAuthToken?: (sandboxName: string) => string | null; env?: NodeJS.ProcessEnv; }; @@ -93,21 +90,6 @@ export function getDashboardForwardTarget( return buildDashboardChain(chatUiUrl, options).forwardTarget; } -export function getDashboardForwardStartCommand( - sandboxName: string, - options: DashboardAccessOptions = {}, -): string { - if (!options.openshellShellCommand) { - throw new Error("getDashboardForwardStartCommand requires openshellShellCommand"); - } - const chatUiUrl = defaultChatUiUrl(options); - const forwardTarget = getDashboardForwardTarget(chatUiUrl, options); - return `${options.openshellShellCommand( - ["forward", "start", "--background", forwardTarget, sandboxName], - options, - )}`; -} - export function buildAuthenticatedDashboardUrl(baseUrl: string, token: string | null = null): string { if (!token) return baseUrl; return `${baseUrl}#token=${encodeURIComponent(token)}`; diff --git a/src/lib/onboard/forward-start.test.ts b/src/lib/onboard/forward-start.test.ts deleted file mode 100644 index 7a4cfefc35..0000000000 --- a/src/lib/onboard/forward-start.test.ts +++ /dev/null @@ -1,385 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; - -import { describe, expect, it, vi } from "vitest"; - -import { - buildDetachedForwardStartSpawn, - looksLikeForwardPortConflict, - runDetachedForwardStartWithDiagnostics, - runDetachedForwardStartWithPortReleaseRetries, -} from "../../../dist/lib/onboard/forward-start"; - -// Build an `openshell forward list`-shaped output for the given live entries. -// Mirrors the column layout (SANDBOX BIND PORT PID STATUS) that -// `getOccupiedPorts` parses, so the helper recognises the forward as live. -function forwardListWith(entries: Array<{ sandbox: string; port: number; status?: string }>): string { - const header = "SANDBOX BIND PORT PID STATUS"; - const rows = entries.map( - (e) => `${e.sandbox} 127.0.0.1 ${e.port} 1234 ${e.status ?? "running"}`, - ); - return [header, ...rows].join("\n"); -} - -describe("runDetachedForwardStartWithDiagnostics", () => { - it("returns ok as soon as the forward appears in the list", () => { - const fetchList = vi - .fn() - .mockReturnValueOnce(forwardListWith([])) // first poll: nothing yet - .mockReturnValue(forwardListWith([{ sandbox: "my-sandbox", port: 18789 }])); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 10_000, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(true); - expect(result.reason).toBe("ok"); - expect(result.pid).toBe(42); - expect(spawn).toHaveBeenCalledTimes(1); - // First poll missed → one sleep before the second poll observed the entry. - expect(sleep).toHaveBeenCalledTimes(1); - }); - - it("ignores entries that belong to a different sandbox", () => { - const fetchList = vi - .fn() - .mockReturnValue(forwardListWith([{ sandbox: "other-sandbox", port: 18789 }])); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 50, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("timeout"); - }); - - it("reports timeout when the forward never appears", () => { - const fetchList = vi.fn().mockReturnValue(forwardListWith([])); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("timeout"); - expect(result.diagnostic).toMatch(/forward did not appear in list within 30ms/); - }); - - it("surfaces spawn errors immediately without polling", () => { - const fetchList = vi.fn(); - const spawn = vi.fn().mockReturnValue({ error: new Error("ENOENT: openshell not found") }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 10_000, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("spawn-error"); - expect(result.diagnostic).toMatch(/ENOENT/); - expect(fetchList).not.toHaveBeenCalled(); - expect(sleep).not.toHaveBeenCalled(); - }); - - it("preflights argv[0] and short-circuits on a missing openshell binary", () => { - const fetchList = vi.fn().mockReturnValue(""); - const sleep = vi.fn(); - // Real `buildDetachedForwardStartSpawn` checks `fs.accessSync(argv[0], - // X_OK)` before spawning, so a missing binary surfaces as a synchronous - // spawn-error instead of relying on Node's async `error` event (which - // cannot fire while the helper is sleeping inside spawnSync). - const spawn = buildDetachedForwardStartSpawn(["/nonexistent/openshell-binary-for-test"]); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 5_000, pollIntervalMs: 5, sleepMs: sleep }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("spawn-error"); - expect(result.diagnostic).toMatch(/ENOENT|EACCES|no such file|permission denied/i); - // No polling should have happened; the helper returned at the spawn - // preflight step. - expect(fetchList).not.toHaveBeenCalled(); - expect(sleep).not.toHaveBeenCalled(); - }); - - it("invokes onProgress while waiting for the forward to appear", () => { - let now = 0; - const realNow = Date.now; - Date.now = () => now; - try { - const fetchList = vi.fn().mockReturnValue(""); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn().mockImplementation((ms) => { - now += ms; - }); - const onProgress = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { - overallTimeoutMs: 120_000, - pollIntervalMs: 1_000, - sleepMs: sleep, - onProgress, - progressIntervalMs: 30_000, - }, - ); - - expect(result.ok).toBe(false); - expect(onProgress).toHaveBeenCalled(); - const calls = onProgress.mock.calls; - expect(calls.length).toBeGreaterThanOrEqual(3); - expect(calls[0][0].elapsedMs).toBeGreaterThanOrEqual(30_000); - expect(result.diagnostic).toMatch(/forward did not appear in list within 120000ms/); - expect(result.diagnostic).toMatch(/last forward list: /); - } finally { - Date.now = realNow; - } - }); - - it("surfaces persistent fetchForwardList failures in the timeout diagnostic", () => { - const fetchList = vi.fn().mockImplementation(() => { - throw new Error("gateway transport: connection refused"); - }); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("timeout"); - expect(result.diagnostic).toMatch(/openshell forward list failed/); - expect(result.diagnostic).toMatch(/connection refused/); - }); - - it("treats fetchForwardList exceptions as transient and keeps polling", () => { - const fetchList = vi - .fn() - .mockImplementationOnce(() => { - throw new Error("gateway not reachable yet"); - }) - .mockReturnValue(forwardListWith([{ sandbox: "my-sandbox", port: 18789 }])); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 10_000, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(true); - expect(fetchList).toHaveBeenCalledTimes(2); - }); - - it("clears a transient fetch error from the diagnostic when a later poll succeeds", () => { - const fetchList = vi - .fn() - .mockImplementationOnce(() => { - throw new Error("transient gateway: connection refused"); - }) - .mockReturnValue(""); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("timeout"); - expect(result.diagnostic).not.toMatch(/openshell forward list failed/); - }); - - it("SIGTERMs the detached child on timeout", () => { - const fetchList = vi.fn().mockReturnValue(""); - const spawn = vi.fn().mockReturnValue({ pid: 4242 }); - const sleep = vi.fn(); - const realKill = process.kill; - const killSpy = vi.fn(); - // Replace process.kill so the test does not actually try to signal pid 4242. - (process as { kill: typeof process.kill }).kill = killSpy as unknown as typeof process.kill; - try { - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 20, pollIntervalMs: 10, sleepMs: sleep }, - ); - expect(result.ok).toBe(false); - expect(result.reason).toBe("timeout"); - expect(killSpy).toHaveBeenCalledWith(4242, "SIGTERM"); - } finally { - (process as { kill: typeof process.kill }).kill = realKill; - } - }); - - it("SIGTERMs the detached child on a port-conflict diagnostic", () => { - // Spawn writes an EADDRINUSE line to the stderr file descriptor so the - // first poll iteration reads it back and trips the conflict branch. - const fetchList = vi.fn().mockReturnValue(""); - const spawn = vi.fn().mockImplementation(({ stderr }: { stderr: number }) => { - fs.writeSync(stderr, "listen tcp 0.0.0.0:18789: bind: address already in use\n"); - return { pid: 8888 }; - }); - const sleep = vi.fn(); - const realKill = process.kill; - const killSpy = vi.fn(); - (process as { kill: typeof process.kill }).kill = killSpy as unknown as typeof process.kill; - try { - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 1_000, pollIntervalMs: 10, sleepMs: sleep }, - ); - expect(result.ok).toBe(false); - expect(result.reason).toBe("spawn-conflict"); - expect(killSpy).toHaveBeenCalledWith(8888, "SIGTERM"); - } finally { - (process as { kill: typeof process.kill }).kill = realKill; - } - }); - - it("does not SIGTERM when spawn never produced a pid", () => { - const fetchList = vi.fn(); - const spawn = vi.fn().mockReturnValue({ error: new Error("ENOENT") }); - const sleep = vi.fn(); - const realKill = process.kill; - const killSpy = vi.fn(); - (process as { kill: typeof process.kill }).kill = killSpy as unknown as typeof process.kill; - try { - const result = runDetachedForwardStartWithDiagnostics( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - { overallTimeoutMs: 1_000, pollIntervalMs: 10, sleepMs: sleep }, - ); - expect(result.reason).toBe("spawn-error"); - expect(killSpy).not.toHaveBeenCalled(); - } finally { - (process as { kill: typeof process.kill }).kill = realKill; - } - }); -}); - -describe("runDetachedForwardStartWithPortReleaseRetries", () => { - it("retries after a port-conflict diagnostic, then succeeds", () => { - const fetchList = vi - .fn() - .mockReturnValueOnce(forwardListWith([])) // first attempt: never appears - .mockReturnValueOnce(forwardListWith([])) // (timeout settles) - .mockReturnValue(forwardListWith([{ sandbox: "my-sandbox", port: 18789 }])); - const beforeRetry = vi.fn(); - // First spawn surfaces a port-conflict in its diagnostic synthesised via - // an Error message; the second spawn succeeds and the forward appears. - const spawn = vi - .fn() - .mockReturnValueOnce({ error: new Error("EADDRINUSE: address already in use") }) - .mockReturnValueOnce({ pid: 99 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithPortReleaseRetries( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - beforeRetry, - { overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep, maxRetries: 3 }, - ); - - expect(result.ok).toBe(true); - expect(beforeRetry).toHaveBeenCalledTimes(1); - expect(spawn).toHaveBeenCalledTimes(2); - }); - - it("does not retry when the failure does not look like a port conflict", () => { - const fetchList = vi.fn().mockReturnValue(forwardListWith([])); - const beforeRetry = vi.fn(); - const spawn = vi.fn().mockReturnValue({ pid: 42 }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithPortReleaseRetries( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - beforeRetry, - { overallTimeoutMs: 20, pollIntervalMs: 10, sleepMs: sleep, maxRetries: 3 }, - ); - - expect(result.ok).toBe(false); - expect(result.reason).toBe("timeout"); - expect(beforeRetry).not.toHaveBeenCalled(); - expect(spawn).toHaveBeenCalledTimes(1); - }); - - it("stops retrying after maxRetries even if conflict diagnostics persist", () => { - const fetchList = vi.fn().mockReturnValue(forwardListWith([])); - const beforeRetry = vi.fn(); - const spawn = vi - .fn() - .mockReturnValue({ error: new Error("EADDRINUSE: address already in use") }); - const sleep = vi.fn(); - - const result = runDetachedForwardStartWithPortReleaseRetries( - spawn, - fetchList, - { port: 18789, sandboxName: "my-sandbox" }, - beforeRetry, - { overallTimeoutMs: 20, pollIntervalMs: 10, sleepMs: sleep, maxRetries: 2 }, - ); - - expect(result.ok).toBe(false); - expect(beforeRetry).toHaveBeenCalledTimes(2); - expect(spawn).toHaveBeenCalledTimes(3); // initial + 2 retries - }); -}); - -describe("looksLikeForwardPortConflict", () => { - it("matches the common port-in-use signals", () => { - expect(looksLikeForwardPortConflict("listen tcp 0.0.0.0:18789: bind: address already in use")).toBe( - true, - ); - expect(looksLikeForwardPortConflict("EADDRINUSE")).toBe(true); - expect(looksLikeForwardPortConflict("port 18789 in use")).toBe(true); - }); - - it("returns false for unrelated errors", () => { - expect(looksLikeForwardPortConflict("transport: connection refused")).toBe(false); - expect(looksLikeForwardPortConflict("")).toBe(false); - }); -}); diff --git a/src/lib/onboard/forward-start.ts b/src/lib/onboard/forward-start.ts deleted file mode 100644 index a9274f1716..0000000000 --- a/src/lib/onboard/forward-start.ts +++ /dev/null @@ -1,331 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawn as spawnChild } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; - -import { compactText } from "../core/url-utils"; -import { redact } from "../security/redact"; -import { getOccupiedPorts } from "./dashboard-port"; -import { cleanupTempDir, secureTempFile } from "./temp-files"; - -// `openshell forward start --background` daemonises the actual forward -// process, but the parent CLI's stdio is inherited by the daemon child on -// some platforms (notably the Docker compatibility gateway used when the -// host glibc is older than the openshell-gateway requirement). spawnSync -// then waits on those fds until the daemon exits — minutes later — and -// reports ETIMEDOUT even though the forward is established. -// -// The detached path below spawns the CLI with `detached: true`, hands it -// independent diagnostic file descriptors, and confirms success by polling -// `openshell forward list` for an entry matching `(port, sandboxName)`. -// The CLI's exit code is no longer the success signal — the appearance of -// the live forward in the list is. - -export type ForwardListFetcher = () => string; - -export type DetachedForwardSpawnRunner = ( - stdio: { stdout: number; stderr: number }, -) => { pid?: number; error?: Error }; - -export interface DetachedForwardStartOutcome { - ok: boolean; - diagnostic: string; - pid?: number; - reason: "ok" | "spawn-error" | "timeout" | "spawn-conflict"; -} - -export interface DetachedForwardStartOptions { - overallTimeoutMs?: number; - pollIntervalMs?: number; - sleepMs?: (ms: number) => void; - // Called once per `progressIntervalMs` while the helper is still waiting - // for the forward to appear in `openshell forward list`. The default is a - // no-op so the helper stays terminal-quiet in non-interactive contexts. - onProgress?: (info: { elapsedMs: number; listSnapshot: string }) => void; - progressIntervalMs?: number; - // Number of EADDRINUSE-style retries after the initial attempt. Honoured - // only by `runDetachedForwardStartWithPortReleaseRetries`. Defaults to 3. - maxRetries?: number; -} - -function readDiagnosticFile(filePath: string): string { - try { - return fs.readFileSync(filePath, "utf-8"); - } catch (error) { - if ( - typeof error === "object" && - error !== null && - "code" in error && - error.code === "ENOENT" - ) { - return ""; - } - throw error; - } -} - -export function looksLikeForwardPortConflict(diagnostic: string): boolean { - return /eaddrinuse|address already in use|port .* in use|bind: .*in use/i.test(diagnostic); -} - -function blockingSleepMs(ms: number): void { - if (ms <= 0) return; - // Synchronous sleep — onboard's forward-start sits in a sync code path, - // so we cannot await. spawnSync of `node -e setTimeout` is the same - // primitive `sleepMs` in core/wait uses, but we keep the call site - // injectable so tests can stub it without spawning subprocesses. We - // intentionally do NOT `.unref()` the timer in the child: an unref'd - // timer lets the child's event loop drain immediately, so spawnSync - // returns instantly and the caller spins through the poll loop without - // actually waiting. - const { spawnSync } = require("node:child_process"); - spawnSync(process.execPath, ["-e", `setTimeout(() => {}, ${ms});`], { - stdio: "ignore", - timeout: ms + 5_000, - }); -} - -/** - * Build a `DetachedForwardSpawnRunner` that spawns the given argv as a - * detached child, writing stdio to the file descriptors supplied by - * `runDetachedForwardStartWithDiagnostics`. Kept in this module so the - * onboard call site stays a thin wire-up and the spawn-on-Node detail - * (`detached: true` + `unref()`) lives next to the consumer that relies - * on it. - */ -export function buildDetachedForwardStartSpawn( - argv: readonly string[], -): DetachedForwardSpawnRunner { - return ({ stdout, stderr }) => { - // Preflight: the helper polls synchronously, so a Node `error` event - // dispatched after `spawn` returns cannot reach the poll loop while it - // is sleeping on a `spawnSync` child. Catch the obvious ENOENT/EACCES - // cases up front via `fs.accessSync` so the helper returns the real - // failure immediately instead of timing out 180s later. - try { - fs.accessSync(argv[0], fs.constants.X_OK); - } catch (e) { - return { error: e instanceof Error ? e : new Error(String(e)) }; - } - try { - const child = spawnChild(argv[0], argv.slice(1), { - stdio: ["ignore", stdout, stderr], - detached: true, - }); - // Swallow any belated `error` event so a race between accessSync and - // execve does not crash the process via an unhandled emitter. - child.on("error", () => {}); - // A null/undefined pid means execve failed even though the preflight - // succeeded (race against permission changes, ulimit, etc.). The async - // `error` event would otherwise be swallowed by the listener above and - // the caller would wait the full deadline for a child that never ran. - if (child.pid == null) { - return { error: new Error(`spawn ${argv[0]} returned no pid`) }; - } - child.unref(); - return { pid: child.pid }; - } catch (e) { - return { error: e instanceof Error ? e : new Error(String(e)) }; - } - }; -} - -function isForwardConfirmed( - forwardListOutput: string, - expect: { port: number; sandboxName: string }, -): boolean { - return getOccupiedPorts(forwardListOutput).get(String(expect.port)) === expect.sandboxName; -} - -/** - * Best-effort SIGTERM of the detached `openshell forward start --background` - * process when the helper gives up. Without this, a slow gateway handshake - * can still register a forward minutes after onboard already rolled the - * sandbox back, causing the next onboard attempt on the same port to race - * an orphan CLI for the dashboard. `kill` swallows ESRCH so a child that - * already exited is a no-op. - */ -function terminateDetachedForwardChild(pid: number | undefined): void { - if (!pid || pid <= 0) return; - try { - process.kill(pid, "SIGTERM"); - } catch { - /* already exited or out of our reach */ - } -} - -/** - * Default progress logger for the detached forward-start helper. Emits a - * single line to stdout every `progressIntervalMs` while the helper is - * still polling. Kept here so the onboard call site does not need to - * recreate the same closure inline. - */ -export function buildForwardStartProgressLogger(port: number): (info: { elapsedMs: number }) => void { - return ({ elapsedMs }) => { - console.log( - ` Still waiting for forward on port ${port} to register (${Math.round(elapsedMs / 1000)}s elapsed)...`, - ); - }; -} - -/** - * Spawn `openshell forward start --background` as a detached child and wait - * for the resulting forward to appear in `openshell forward list`. Returns - * `ok: true` as soon as the live entry is observed, regardless of whether - * the original spawn process has exited yet. Returns `ok: false` with a - * captured diagnostic when: - * - the spawn itself failed (ENOENT, permission denied, …); - * - the parent process wrote an EADDRINUSE-style error to stderr before - * the deadline (port conflict — retry path); - * - the deadline expired without the forward appearing. - * - * The diagnostic file pair is removed before return, so the temp dir does - * not leak across retries. - */ -export function runDetachedForwardStartWithDiagnostics( - runDetachedSpawn: DetachedForwardSpawnRunner, - fetchForwardList: ForwardListFetcher, - expect: { port: number; sandboxName: string }, - options: DetachedForwardStartOptions = {}, -): DetachedForwardStartOutcome { - // 180s deadline accommodates Docker compatibility gateways (host glibc - // older than openshell-gateway's requirement runs the gateway in an extra - // Docker container, adding per-call gRPC latency that can push the - // forward-registration handshake past a tighter timeout). - const overallTimeoutMs = options.overallTimeoutMs ?? 180_000; - const pollIntervalMs = options.pollIntervalMs ?? 500; - const sleepImpl = options.sleepMs ?? blockingSleepMs; - const onProgress = options.onProgress; - const progressIntervalMs = options.progressIntervalMs ?? 30_000; - let nextProgressAt = Date.now() + progressIntervalMs; - - const forwardDiagPath = secureTempFile("nemoclaw-forward-start", ".out"); - const forwardDiagDir = path.dirname(forwardDiagPath); - const forwardErrPath = path.join(forwardDiagDir, "nemoclaw-forward-start.err"); - // `fs.openSync` with `"w"` truncates / creates the diagnostic files; the - // child inherits the fds via posix_spawn semantics. We close the host's - // copies immediately so only the child's reference keeps them alive, - // which lets the kernel reclaim them when the (detached) child exits. - const outFd = fs.openSync(forwardDiagPath, "w", 0o600); - const errFd = fs.openSync(forwardErrPath, "w", 0o600); - - let pid: number | undefined; - let spawnError: Error | undefined; - try { - const spawnResult = runDetachedSpawn({ stdout: outFd, stderr: errFd }); - pid = spawnResult.pid; - spawnError = spawnResult.error; - } finally { - try { - fs.closeSync(outFd); - } catch { - /* best effort */ - } - try { - fs.closeSync(errFd); - } catch { - /* best effort */ - } - } - - let lastFetchError: string | null = null; - const readDiag = (): string => { - const stderr = readDiagnosticFile(forwardErrPath); - const stdout = readDiagnosticFile(forwardDiagPath); - const message = spawnError instanceof Error ? spawnError.message : ""; - const fetchSuffix = lastFetchError ? ` openshell forward list failed: ${lastFetchError}` : ""; - return compactText(redact(`${stderr} ${stdout} ${message}${fetchSuffix}`)); - }; - - try { - if (spawnError) { - return { ok: false, diagnostic: readDiag(), pid, reason: "spawn-error" }; - } - - const start = Date.now(); - const deadline = start + overallTimeoutMs; - let lastListSnapshot = ""; - while (Date.now() < deadline) { - let list = ""; - try { - list = fetchForwardList() || ""; - // Clear the cached transient error so a recovered gateway does not - // leave a stale "openshell forward list failed: …" suffix on the - // eventual timeout diagnostic. - lastFetchError = null; - } catch (err) { - lastFetchError = err instanceof Error ? err.message : String(err); - } - lastListSnapshot = list; - if (isForwardConfirmed(list, expect)) { - return { ok: true, diagnostic: readDiag(), pid, reason: "ok" }; - } - const diagSoFar = readDiag(); - if (looksLikeForwardPortConflict(diagSoFar)) { - terminateDetachedForwardChild(pid); - return { ok: false, diagnostic: diagSoFar, pid, reason: "spawn-conflict" }; - } - if (onProgress && Date.now() >= nextProgressAt) { - onProgress({ elapsedMs: Date.now() - start, listSnapshot: list }); - nextProgressAt = Date.now() + progressIntervalMs; - } - sleepImpl(pollIntervalMs); - } - const finalDiag = readDiag(); - const listTail = lastListSnapshot - ? ` last forward list: ${compactText(redact(lastListSnapshot)).slice(0, 240)}` - : " last forward list: "; - const timeoutSummary = `forward did not appear in list within ${overallTimeoutMs}ms;${listTail}`; - // The detached `openshell forward start --background` process may still - // be running (e.g. blocked on a slow gateway handshake). If the caller - // is about to roll back the sandbox, leaving an orphan CLI that may yet - // succeed would race with the next onboard attempt for the same port. - terminateDetachedForwardChild(pid); - return { - ok: false, - diagnostic: finalDiag ? `${timeoutSummary} ${finalDiag}` : timeoutSummary, - pid, - reason: "timeout", - }; - } finally { - cleanupTempDir(forwardDiagPath, "nemoclaw-forward-start"); - } -} - -/** - * Retry the detached forward-start when the diagnostic looks like an - * EADDRINUSE-style port conflict. `beforeRetry` runs between attempts so - * the caller can drop any stale forward bound to the same port before - * trying again. - */ -export function runDetachedForwardStartWithPortReleaseRetries( - runDetachedSpawn: DetachedForwardSpawnRunner, - fetchForwardList: ForwardListFetcher, - expect: { port: number; sandboxName: string }, - beforeRetry: () => void, - options: DetachedForwardStartOptions = {}, -): DetachedForwardStartOutcome { - const maxRetries = options.maxRetries ?? 3; - let attempt = runDetachedForwardStartWithDiagnostics( - runDetachedSpawn, - fetchForwardList, - expect, - options, - ); - for ( - let retries = 0; - !attempt.ok && looksLikeForwardPortConflict(attempt.diagnostic) && retries < maxRetries; - retries++ - ) { - beforeRetry(); - attempt = runDetachedForwardStartWithDiagnostics( - runDetachedSpawn, - fetchForwardList, - expect, - options, - ); - } - return attempt; -} diff --git a/test/install-preflight.test.ts b/test/install-preflight.test.ts index 5336fa0cf8..8144d564a1 100644 --- a/test/install-preflight.test.ts +++ b/test/install-preflight.test.ts @@ -2722,7 +2722,7 @@ exit 1 expect(pathValue.startsWith(`${localBin}:`)).toBe(true); }); - it("restore_onboard_forward_after_post_checks: restores Hermes forward from session", () => { + it("restore_onboard_forward_after_post_checks: does not start legacy OpenShell forwards", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemohermes-forward-restore-")); const fakeBin = path.join(tmp, "bin"); const stateDir = path.join(tmp, ".nemoclaw"); @@ -2765,9 +2765,7 @@ exit 0 }); expect(r.status).toBe(0); - const openshellCalls = fs.readFileSync(openshellLog, "utf-8"); - expect(openshellCalls).toContain("forward stop 8642 created-by-onboard"); - expect(openshellCalls).toContain("forward start --background 8642 created-by-onboard"); + expect(fs.existsSync(openshellLog)).toBe(false); }); // -- resolve_default_sandbox_name -- From 10f8341f4a192333396d12b67445777a666bc934 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 17:34:42 -0700 Subject: [PATCH 19/26] Check gRPC forward state in double onboard E2E --- test/e2e/test-double-onboard.sh | 129 +++++++++++++++++++++++++++----- 1 file changed, 112 insertions(+), 17 deletions(-) diff --git a/test/e2e/test-double-onboard.sh b/test/e2e/test-double-onboard.sh index 3d585f4f8b..1a81e18962 100755 --- a/test/e2e/test-double-onboard.sh +++ b/test/e2e/test-double-onboard.sh @@ -69,6 +69,8 @@ dump_diagnostics() { openshell sandbox list 2>&1 | sed 's/^/ /' || true info "openshell forward list:" openshell forward list 2>&1 | sed 's/^/ /' || true + info "NemoClaw gRPC forward state:" + dump_grpc_forward_state | sed 's/^/ /' for sandbox_name in "${SANDBOX_A:-}" "${SANDBOX_B:-}"; do [ -n "$sandbox_name" ] || continue info "${sandbox_name} /etc/resolv.conf:" @@ -306,6 +308,7 @@ run_nemoclaw() { stop_forward_if_set() { local port="${1:-}" if [ -n "$port" ]; then + stop_grpc_forward_for_port "$port" "" 2>/dev/null || true openshell forward stop "$port" 2>/dev/null || true fi } @@ -357,27 +360,119 @@ PY forward_owner_for_port() { local port="$1" - FORWARD_OUTPUT="$forward_output" python3 - "$port" <<'PY' + python3 - "$port" <<'PY' +import glob +import json import os -import re import sys -target = sys.argv[1] -clean = re.sub(r"\x1b\[[0-9;]*m", "", os.environ.get("FORWARD_OUTPUT", "")) - -for line in clean.splitlines(): - parts = line.strip().split() - if len(parts) < 5 or parts[0].lower() == "sandbox": +target = int(sys.argv[1]) +state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") + +for path in glob.glob(os.path.join(state_dir, "*.json")): + try: + with open(path, encoding="utf-8") as fh: + state = json.load(fh) + if int(state.get("port", -1)) != target: + continue + pid = int(state.get("pid", 0) or 0) + if pid <= 0: + continue + os.kill(pid, 0) + except OSError: + continue + except Exception: continue - status = " ".join(parts[4:]).lower() - if parts[2] == target and "running" in status: - print(parts[0]) - sys.exit(0) + print(state.get("sandboxName", "")) + sys.exit(0) sys.exit(1) PY } +dump_grpc_forward_state() { + python3 <<'PY' +import glob +import json +import os + +state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") +paths = sorted(glob.glob(os.path.join(state_dir, "*.json"))) +if not paths: + print("No NemoClaw gRPC forwards.") +for path in paths: + try: + with open(path, encoding="utf-8") as fh: + state = json.load(fh) + print( + f"{state.get('sandboxName', '')} " + f"{state.get('bind', '')} {state.get('port', '')} " + f"{state.get('pid', '')} -> " + f"{state.get('targetHost', '')}:{state.get('targetPort', '')}" + ) + except Exception as exc: + print(f"{path}: unreadable ({exc})") +PY +} + +stop_grpc_forward_for_port() { + local port="$1" + local expected_owner="${2:-}" + + python3 - "$port" "$expected_owner" <<'PY' +import glob +import json +import os +import signal +import sys +import time + +target = int(sys.argv[1]) +expected_owner = sys.argv[2] +state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") +stopped = False + +for path in glob.glob(os.path.join(state_dir, "*.json")): + try: + with open(path, encoding="utf-8") as fh: + state = json.load(fh) + if int(state.get("port", -1)) != target: + continue + if expected_owner and state.get("sandboxName") != expected_owner: + continue + pid = int(state.get("pid", 0) or 0) + except Exception: + continue + + if pid > 0: + for sig, wait_seconds in ((signal.SIGTERM, 2.0), (signal.SIGKILL, 1.0)): + try: + os.kill(pid, sig) + except ProcessLookupError: + break + except PermissionError: + break + deadline = time.time() + wait_seconds + while time.time() < deadline: + try: + os.kill(pid, 0) + except OSError: + break + time.sleep(0.1) + else: + continue + break + + try: + os.unlink(path) + except OSError: + pass + stopped = True + +sys.exit(0 if stopped else 1) +PY +} + # ══════════════════════════════════════════════════════════════════ # Phase 0: Pre-cleanup # ══════════════════════════════════════════════════════════════════ @@ -650,6 +745,7 @@ fi if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then info "Stopping '$SANDBOX_B' dashboard forward to verify stored-port recovery..." + stop_grpc_forward_for_port "$port_b" "$SANDBOX_B" 2>/dev/null || true openshell forward stop "$port_b" 2>/dev/null || true PROBE_LOG="$(mktemp)" @@ -677,7 +773,6 @@ if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then dump_diagnostics "probe-only dashboard forward recovery" fi - forward_output="$(openshell forward list 2>&1 || true)" owner_a="$(forward_owner_for_port "$port_a" 2>/dev/null || true)" owner_b="$(forward_owner_for_port "$port_b" 2>/dev/null || true)" @@ -685,16 +780,16 @@ if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then pass "Second sandbox dashboard forward restored on its recorded port" else fail "Second sandbox dashboard forward owner mismatch on port $port_b (owner=${owner_b:-missing})" - info "Observed forward list:" - printf '%s\n' "$forward_output" | sed 's/^/ /' + info "Observed NemoClaw gRPC forward state:" + dump_grpc_forward_state | sed 's/^/ /' fi if [ "$owner_a" = "$SANDBOX_A" ]; then pass "First sandbox dashboard forward kept its recorded port" else fail "First sandbox dashboard forward owner mismatch on port $port_a (owner=${owner_a:-missing})" - info "Observed forward list:" - printf '%s\n' "$forward_output" | sed 's/^/ /' + info "Observed NemoClaw gRPC forward state:" + dump_grpc_forward_state | sed 's/^/ /' fi fi From 19a0519109e458e9059d867fa866c228c72c2acf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 27 May 2026 17:38:15 -0700 Subject: [PATCH 20/26] Drop legacy forward registry check from double onboard E2E --- test/e2e/test-double-onboard.sh | 139 -------------------------------- 1 file changed, 139 deletions(-) diff --git a/test/e2e/test-double-onboard.sh b/test/e2e/test-double-onboard.sh index 1a81e18962..91ef8358e6 100755 --- a/test/e2e/test-double-onboard.sh +++ b/test/e2e/test-double-onboard.sh @@ -69,8 +69,6 @@ dump_diagnostics() { openshell sandbox list 2>&1 | sed 's/^/ /' || true info "openshell forward list:" openshell forward list 2>&1 | sed 's/^/ /' || true - info "NemoClaw gRPC forward state:" - dump_grpc_forward_state | sed 's/^/ /' for sandbox_name in "${SANDBOX_A:-}" "${SANDBOX_B:-}"; do [ -n "$sandbox_name" ] || continue info "${sandbox_name} /etc/resolv.conf:" @@ -308,7 +306,6 @@ run_nemoclaw() { stop_forward_if_set() { local port="${1:-}" if [ -n "$port" ]; then - stop_grpc_forward_for_port "$port" "" 2>/dev/null || true openshell forward stop "$port" 2>/dev/null || true fi } @@ -357,122 +354,6 @@ sys.exit(1) PY } -forward_owner_for_port() { - local port="$1" - - python3 - "$port" <<'PY' -import glob -import json -import os -import sys - -target = int(sys.argv[1]) -state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") - -for path in glob.glob(os.path.join(state_dir, "*.json")): - try: - with open(path, encoding="utf-8") as fh: - state = json.load(fh) - if int(state.get("port", -1)) != target: - continue - pid = int(state.get("pid", 0) or 0) - if pid <= 0: - continue - os.kill(pid, 0) - except OSError: - continue - except Exception: - continue - print(state.get("sandboxName", "")) - sys.exit(0) - -sys.exit(1) -PY -} - -dump_grpc_forward_state() { - python3 <<'PY' -import glob -import json -import os - -state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") -paths = sorted(glob.glob(os.path.join(state_dir, "*.json"))) -if not paths: - print("No NemoClaw gRPC forwards.") -for path in paths: - try: - with open(path, encoding="utf-8") as fh: - state = json.load(fh) - print( - f"{state.get('sandboxName', '')} " - f"{state.get('bind', '')} {state.get('port', '')} " - f"{state.get('pid', '')} -> " - f"{state.get('targetHost', '')}:{state.get('targetPort', '')}" - ) - except Exception as exc: - print(f"{path}: unreadable ({exc})") -PY -} - -stop_grpc_forward_for_port() { - local port="$1" - local expected_owner="${2:-}" - - python3 - "$port" "$expected_owner" <<'PY' -import glob -import json -import os -import signal -import sys -import time - -target = int(sys.argv[1]) -expected_owner = sys.argv[2] -state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") -stopped = False - -for path in glob.glob(os.path.join(state_dir, "*.json")): - try: - with open(path, encoding="utf-8") as fh: - state = json.load(fh) - if int(state.get("port", -1)) != target: - continue - if expected_owner and state.get("sandboxName") != expected_owner: - continue - pid = int(state.get("pid", 0) or 0) - except Exception: - continue - - if pid > 0: - for sig, wait_seconds in ((signal.SIGTERM, 2.0), (signal.SIGKILL, 1.0)): - try: - os.kill(pid, sig) - except ProcessLookupError: - break - except PermissionError: - break - deadline = time.time() + wait_seconds - while time.time() < deadline: - try: - os.kill(pid, 0) - except OSError: - break - time.sleep(0.1) - else: - continue - break - - try: - os.unlink(path) - except OSError: - pass - stopped = True - -sys.exit(0 if stopped else 1) -PY -} - # ══════════════════════════════════════════════════════════════════ # Phase 0: Pre-cleanup # ══════════════════════════════════════════════════════════════════ @@ -745,7 +626,6 @@ fi if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then info "Stopping '$SANDBOX_B' dashboard forward to verify stored-port recovery..." - stop_grpc_forward_for_port "$port_b" "$SANDBOX_B" 2>/dev/null || true openshell forward stop "$port_b" 2>/dev/null || true PROBE_LOG="$(mktemp)" @@ -772,25 +652,6 @@ if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then printf '%s\n' "$probe_output" | sed 's/^/ /' dump_diagnostics "probe-only dashboard forward recovery" fi - - owner_a="$(forward_owner_for_port "$port_a" 2>/dev/null || true)" - owner_b="$(forward_owner_for_port "$port_b" 2>/dev/null || true)" - - if [ "$owner_b" = "$SANDBOX_B" ]; then - pass "Second sandbox dashboard forward restored on its recorded port" - else - fail "Second sandbox dashboard forward owner mismatch on port $port_b (owner=${owner_b:-missing})" - info "Observed NemoClaw gRPC forward state:" - dump_grpc_forward_state | sed 's/^/ /' - fi - - if [ "$owner_a" = "$SANDBOX_A" ]; then - pass "First sandbox dashboard forward kept its recorded port" - else - fail "First sandbox dashboard forward owner mismatch on port $port_a (owner=${owner_a:-missing})" - info "Observed NemoClaw gRPC forward state:" - dump_grpc_forward_state | sed 's/^/ /' - fi fi # ══════════════════════════════════════════════════════════════════ From f4eca817e7734b4d532fcd9325ed758e13a95f43 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 15:18:56 -0700 Subject: [PATCH 21/26] feat: consume OpenShell SDK for sandbox transport --- ci/env-var-doc-allowlist.json | 10 +- package-lock.json | 81 ++- package.json | 5 +- scripts/copy-openshell-protos.mjs | 15 - src/lib/actions/sandbox/connect.ts | 2 +- src/lib/actions/sandbox/process-recovery.ts | 8 +- .../openshell/forward-bridge-runner.ts | 53 -- .../openshell/forward-bridge-state.test.ts | 2 +- .../openshell/forward-bridge-state.ts | 77 +-- .../openshell/gateway-metadata.test.ts | 114 +++- .../adapters/openshell/gateway-metadata.ts | 208 ++++-- .../openshell/grpc-migration-guard.test.ts | 12 +- src/lib/adapters/openshell/grpc.test.ts | 107 ++- src/lib/adapters/openshell/grpc.ts | 618 +++++------------- .../adapters/openshell/proto/datamodel.proto | 15 - .../adapters/openshell/proto/openshell.proto | 214 ------ src/lib/adapters/openshell/sync-runner.ts | 204 +++--- src/lib/onboard/dashboard.ts | 4 +- src/lib/sandbox/version.ts | 2 +- src/lib/share-command.test.ts | 2 +- src/lib/share-command.ts | 2 +- src/lib/skill-install.ts | 2 +- src/lib/status-command-deps.test.ts | 34 +- src/lib/tunnel/services-sandbox.test.ts | 4 +- src/lib/tunnel/services.ts | 2 +- src/lib/verify-deployment.ts | 2 +- test/cli.test.ts | 9 +- test/destroy-cleanup-sandbox-services.test.ts | 2 +- test/helpers/grpc-fake-ssh.cjs | 64 -- test/helpers/sdk-fake-exec.cjs | 85 +++ test/nemoclaw-cli-recovery.test.ts | 7 +- test/onboard.test.ts | 7 +- test/rebuild-credential-preflight.test.ts | 13 +- test/rebuild-shields-auto-unlock.test.ts | 9 +- test/recover-port-forward.test.ts | 7 +- test/repro-2201.test.ts | 13 +- test/sandbox-connect-inference.test.ts | 7 +- test/shellquote-sandbox.test.ts | 12 +- test/snapshot-gateway-guard.test.ts | 7 +- test/snapshot.test.ts | 10 +- 40 files changed, 785 insertions(+), 1266 deletions(-) delete mode 100644 scripts/copy-openshell-protos.mjs delete mode 100644 src/lib/adapters/openshell/forward-bridge-runner.ts delete mode 100644 src/lib/adapters/openshell/proto/datamodel.proto delete mode 100644 src/lib/adapters/openshell/proto/openshell.proto delete mode 100755 test/helpers/grpc-fake-ssh.cjs create mode 100644 test/helpers/sdk-fake-exec.cjs diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index 307d915280..569aed4f43 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -32,15 +32,11 @@ "reason": "Internal installer sentinel exported only during OpenShell gateway replacement so onboard restores the pre-upgrade sandbox backup. Not user-facing." }, { - "name": "NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN", - "reason": "Vitest-only compatibility shim used by the OpenShell gRPC sync runner to drive existing hermetic fake OpenShell fixtures. Never user-set in production." + "name": "NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN", + "reason": "Vitest-only helper used by the OpenShell SDK sync runner to drive hermetic fake OpenShell fixtures without loading the native SDK binding. Never user-set in production." }, { - "name": "NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH", - "reason": "Vitest-only sentinel that enables the OpenShell gRPC sync runner's legacy fake transport while CLI tests migrate away from SSH-shaped fixtures. Never user-set in production." - }, - { - "name": "NEMOCLAW_GRPC_TEST_TRANSPORT", + "name": "NEMOCLAW_SDK_TEST_TRANSPORT", "reason": "Vitest-only sentinel that prevents hermetic CLI tests from loading real OpenShell gateway metadata and routes sandbox exec calls through test fixtures. Never user-set in production." }, { diff --git a/package-lock.json b/package-lock.json index bb258c5b61..ece8a31b5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,9 +14,8 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", - "@grpc/grpc-js": "^1.14.4", - "@grpc/proto-loader": "^0.8.1", "@oclif/core": "^4.10.5", + "@openshell/sdk": "file:../OpenShell/crates/openshell-sdk-node", "js-yaml": "^4.1.1", "p-retry": "^4.6.2", "qrcode-terminal": "^0.12.0", @@ -43,6 +42,17 @@ "node": ">=22.16.0" } }, + "../OpenShell/crates/openshell-sdk-node": { + "name": "@openshell/sdk", + "version": "0.0.0-alpha.0", + "license": "Apache-2.0", + "devDependencies": { + "@napi-rs/cli": "^3" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/@anthropic-ai/sdk": { "version": "0.91.1", "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", @@ -1768,37 +1778,6 @@ } } }, - "node_modules/@grpc/grpc-js": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", - "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/proto-loader": "^0.8.0", - "@js-sdsl/ordered-map": "^4.4.2" - }, - "engines": { - "node": ">=12.10.0" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", - "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.5.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/@j178/prek": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/@j178/prek/-/prek-0.3.6.tgz", @@ -1869,16 +1848,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@js-sdsl/ordered-map": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", - "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/js-sdsl" - } - }, "node_modules/@mariozechner/clipboard": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.5.tgz", @@ -2156,6 +2125,10 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/@openshell/sdk": { + "resolved": "../OpenShell/crates/openshell-sdk-node", + "link": true + }, "node_modules/@oxc-project/types": { "version": "0.124.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz", @@ -2170,30 +2143,35 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/codegen": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.1", @@ -2204,30 +2182,35 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/inquire": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/utf8": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@rolldown/binding-android-arm64": { @@ -2736,6 +2719,7 @@ "version": "25.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.2.tgz", "integrity": "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -3325,6 +3309,7 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3783,6 +3768,7 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4204,6 +4190,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -5154,6 +5141,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "dev": true, "license": "MIT" }, "node_modules/lodash.kebabcase": { @@ -5195,6 +5183,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, "license": "Apache-2.0" }, "node_modules/lru-cache": { @@ -5704,6 +5693,7 @@ "version": "7.5.8", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.8.tgz", "integrity": "sha512-dvpCIeLPbXZS/Ete7yLaO7RenOdken2NHKykBXbsaGxZT0UTltcarBciw+A78SRQs9iMAAVpsYA+l8b1hTePIA==", + "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -5794,6 +5784,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6245,6 +6236,7 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -6549,6 +6541,7 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -6573,6 +6566,7 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -6591,6 +6585,7 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, "license": "ISC", "engines": { "node": ">=12" diff --git a/package.json b/package.json index dd69804382..eb005bbaf1 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "format:ts": "cd nemoclaw && npm run lint:fix && npm run format", "check:installer-hash": "bash scripts/check-installer-hash.sh", "typecheck": "tsc -p jsconfig.json", - "build:cli": "tsc -p tsconfig.src.json && node scripts/copy-openshell-protos.mjs && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", + "build:cli": "tsc -p tsconfig.src.json && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", "typecheck:cli": "tsc -p tsconfig.cli.json", "validate:configs": "tsx scripts/validate-configs.ts", "type-safety:hotspots": "tsx scripts/type-safety-hotspots.ts", @@ -47,9 +47,8 @@ }, "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", - "@grpc/grpc-js": "^1.14.4", - "@grpc/proto-loader": "^0.8.1", "@oclif/core": "^4.10.5", + "@openshell/sdk": "file:../OpenShell/crates/openshell-sdk-node", "js-yaml": "^4.1.1", "p-retry": "^4.6.2", "qrcode-terminal": "^0.12.0", diff --git a/scripts/copy-openshell-protos.mjs b/scripts/copy-openshell-protos.mjs deleted file mode 100644 index 3c9c544723..0000000000 --- a/scripts/copy-openshell-protos.mjs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import path from "node:path"; - -const root = process.cwd(); -const source = path.join(root, "src", "lib", "adapters", "openshell", "proto"); -const dest = path.join(root, "dist", "lib", "adapters", "openshell", "proto"); - -fs.mkdirSync(dest, { recursive: true }); -for (const entry of fs.readdirSync(source, { withFileTypes: true })) { - if (!entry.isFile() || !entry.name.endsWith(".proto")) continue; - fs.copyFileSync(path.join(source, entry.name), path.join(dest, entry.name)); -} diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 3bca6dd379..430ff3e766 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -244,7 +244,7 @@ function probeSandboxInferenceRoute( lastProbe = { healthy: probe.status === 0 && /^OK\s+[0-9]{3}\b/.test(detail), broken: /^BROKEN\s+[0-9]{3}\b/.test(detail), - detail: detail || `OpenShell gRPC exec exited with status ${String(probe.status)}`, + detail: detail || `OpenShell SDK exec exited with status ${String(probe.status)}`, }; if (lastProbe.healthy || attempt === boundedAttempts) return lastProbe; sleepSync(delayMs); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index a49d4e849b..11e9761c6b 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -75,7 +75,7 @@ function getSandboxHealthProbeUrl(sandboxName: string): string { return `http://127.0.0.1:${resolveSandboxDashboardPort(sandboxName)}/health`; } -/** Run a shell command inside the sandbox over OpenShell gRPC. */ +/** Run a shell command inside the sandbox over the OpenShell SDK. */ function executeSandboxShellSync( sandboxName: string, command: string, @@ -132,7 +132,7 @@ async function executeSandboxExecCommandForStatus( sandboxName: string, command: string, ): Promise { - if (process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1") { + if (process.env.NEMOCLAW_SDK_TEST_TRANSPORT === "1") { return executeSandboxExecCommand(sandboxName, command, OPENSHELL_PROBE_TIMEOUT_MS); } const markedCommand = `printf '%s\\n' '${SANDBOX_EXEC_STARTED_MARKER}'; ${command}`; @@ -203,7 +203,7 @@ export async function isSandboxGatewayRunningForStatus( /** * Probe the full inference chain by curling `https://inference.local/v1/models` - * from inside the sandbox via OpenShell gRPC exec. This is the path agent + * from inside the sandbox via OpenShell SDK exec. This is the path agent * traffic actually takes (openclaw gateway -> auth proxy -> backend). Any HTTP * response (including 401) means routing works; 000 / no response means DNS, * proxy, or gateway is broken. The optional 3rd line in #3265. @@ -330,7 +330,7 @@ function ensureSandboxPortForward(sandboxName: string): boolean { } /** - * Probe NemoClaw's gRPC forward state for the sandbox's dashboard forward. + * Probe NemoClaw's SDK forward state for the sandbox's dashboard forward. * Returns true when an entry exists for the expected sandbox+port pair * with STATUS=running, false when the entry is missing or non-running, * "occupied" when another sandbox already owns the expected port, and diff --git a/src/lib/adapters/openshell/forward-bridge-runner.ts b/src/lib/adapters/openshell/forward-bridge-runner.ts deleted file mode 100644 index 6b248d3cde..0000000000 --- a/src/lib/adapters/openshell/forward-bridge-runner.ts +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createSandboxGrpcClient } from "./grpc"; -import { writeForwardState } from "./forward-bridge-state"; - -type RunnerArgs = { - sandboxName: string; - bind: string; - port: number; - targetHost: string; - targetPort: number; -}; - -async function main(): Promise { - const raw = process.argv[2]; - if (!raw) throw new Error("missing forward bridge args"); - const args = JSON.parse(raw) as RunnerArgs; - const client = createSandboxGrpcClient(); - const handle = await client.startForward(args.sandboxName, { - localHost: args.bind, - localPort: args.port, - targetHost: args.targetHost, - targetPort: args.targetPort, - serviceId: `nemoclaw-dashboard-${args.port}`, - }); - writeForwardState({ - sandboxName: args.sandboxName, - bind: args.bind, - port: args.port, - targetHost: args.targetHost, - targetPort: args.targetPort, - pid: process.pid, - startedAt: new Date().toISOString(), - }); - - const close = async () => { - try { - await handle.close(); - } catch { - /* ignore */ - } - client.close(); - process.exit(0); - }; - process.on("SIGTERM", () => void close()); - process.on("SIGINT", () => void close()); -} - -main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); -}); diff --git a/src/lib/adapters/openshell/forward-bridge-state.test.ts b/src/lib/adapters/openshell/forward-bridge-state.test.ts index 4987185178..32b891c074 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.test.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.test.ts @@ -51,7 +51,7 @@ async function canListen(port: number): Promise { }); } -describe("OpenShell gRPC forward bridge readiness", () => { +describe("OpenShell SDK forward bridge readiness", () => { it("does not treat state-file presence as readiness when the local port is unreachable", async () => { const port = await unusedLocalPort(); expect(__forwardBridgeTestHooks.probeForwardReady("127.0.0.1", port)).toBe(false); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index 4bbb084ad4..047e01a738 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn, spawnSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -178,20 +178,8 @@ export function forwardStatesAsListOutput(states = listForwardStates()): string return `${lines.join("\n")}\n`; } -function runnerCommand(): { command: string; args: string[] } { - const built = path.join(__dirname, "forward-bridge-runner.js"); - if (fs.existsSync(built)) return { command: process.execPath, args: [built] }; - - const source = path.join(__dirname, "forward-bridge-runner.ts"); - const root = path.resolve(__dirname, "..", "..", "..", ".."); - const tsxBin = path.join(root, "node_modules", ".bin", process.platform === "win32" ? "tsx.cmd" : "tsx"); - if (fs.existsSync(source) && fs.existsSync(tsxBin)) return { command: tsxBin, args: [source] }; - - throw new Error("OpenShell gRPC forward bridge runner is not available. Run `npm run build:cli` first."); -} - function useTestForwardBridge(): boolean { - return process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1" || process.env.VITEST_WORKER_ID !== undefined; + return process.env.NEMOCLAW_SDK_TEST_TRANSPORT === "1" || process.env.VITEST_WORKER_ID !== undefined; } function testForwardPid(): number { @@ -202,14 +190,6 @@ function isTestForwardPid(pid: number): boolean { return useTestForwardBridge() && pid === testForwardPid(); } -function readOpenFileDescriptor(fd: number): string { - const stat = fs.fstatSync(fd); - if (stat.size <= 0) return ""; - const buffer = Buffer.alloc(Math.min(stat.size, 64 * 1024)); - const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0); - return buffer.subarray(0, bytesRead).toString("utf-8"); -} - function probeForwardReady(bind: string, port: number): boolean { const host = bind === "0.0.0.0" || bind === "::" ? "127.0.0.1" : bind; const script = @@ -250,59 +230,12 @@ export function startForwardBridgeDetached( writeForwardState(state); return { ok: true, state, diagnostic: "" }; } - const { command, args } = runnerCommand(); - ensureStateDir(); - const safeSandbox = sandboxName.replace(/[^A-Za-z0-9._-]/g, "_"); - const diagnosticDir = fs.mkdtempSync(path.join(stateDir(), `${safeSandbox}-${options.port}-`)); - const diagnosticPath = path.join( - diagnosticDir, - "bridge.log", - ); - const out = fs.openSync(diagnosticPath, "w+", 0o600); - const child = spawn( - command, - [ - ...args, - JSON.stringify({ - sandboxName, - bind, - port: options.port, - targetHost, - targetPort: options.targetPort, - }), - ], - { detached: true, stdio: ["ignore", out, out], env: process.env }, - ); - child.unref(); - - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const state = getForwardState(sandboxName, options.port); - if (state?.pid === child.pid) { - if (probeForwardReady(bind, options.port)) { - fs.closeSync(out); - return { ok: true, state: state ?? undefined, diagnostic: "" }; - } - } - sleepMs(250); - } - - let diagnostic = ""; - try { - diagnostic = readOpenFileDescriptor(out).trim(); - } catch { - /* ignore */ - } - fs.closeSync(out); - try { - if (child.pid) process.kill(child.pid, "SIGTERM"); - } catch { - /* ignore */ - } + void targetHost; + void timeoutMs; return { ok: false, diagnostic: - diagnostic || `forward bridge did not become ready within ${String(timeoutMs)}ms`, + "OpenShell SDK forwarding is not available yet. Land an upstream @openshell/sdk forwarding API before enabling dashboard forwarding.", }; } diff --git a/src/lib/adapters/openshell/gateway-metadata.test.ts b/src/lib/adapters/openshell/gateway-metadata.test.ts index 10d88f60f1..b3cabf8b89 100644 --- a/src/lib/adapters/openshell/gateway-metadata.test.ts +++ b/src/lib/adapters/openshell/gateway-metadata.test.ts @@ -9,15 +9,16 @@ import { describe, expect, it } from "vitest"; import { resolveGatewayMetadata } from "./gateway-metadata"; -function writeGateway(home: string, name: string, metadata: Record): void { +function writeGateway(home: string, name: string, metadata: Record): string { const dir = path.join(home, ".config", "openshell", "gateways", name); fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, "metadata.json"), JSON.stringify(metadata)); fs.writeFileSync(path.join(home, ".config", "openshell", "active_gateway"), name); + return dir; } describe("resolveGatewayMetadata", () => { - it("resolves the active plaintext local gateway", () => { + it("resolves the active plaintext local gateway into SDK ConnectOptions", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); try { writeGateway(home, "nemoclaw", { @@ -31,80 +32,123 @@ describe("resolveGatewayMetadata", () => { expect(gateway.name).toBe("nemoclaw"); expect(gateway.target).toBe("127.0.0.1:8080"); expect(gateway.authMode).toBe("plaintext"); - expect(gateway.mtlsDir).toContain(path.join("gateways", "nemoclaw", "mtls")); + expect(gateway.sdkCompatible).toBe(true); + expect(gateway.connectOptions).toMatchObject({ + gateway: "http://127.0.0.1:8080/", + }); + expect(gateway.connectOptions).not.toHaveProperty("edgeToken"); + expect(gateway.connectOptions).not.toHaveProperty("oidcToken"); } finally { fs.rmSync(home, { recursive: true, force: true }); } }); - it("rejects bearer-auth remote gateways with an actionable error", () => { + it("loads Cloudflare edge_token for SDK edge auth", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); try { - writeGateway(home, "remote", { - name: "remote", + const dir = writeGateway(home, "edge", { + name: "edge", gateway_endpoint: "https://gateway.example.test", - auth_mode: "oidc", + auth_mode: "cloudflare_jwt", is_remote: true, }); + fs.writeFileSync(path.join(dir, "edge_token"), "edge-token\n", { mode: 0o600 }); - expect(() => - resolveGatewayMetadata({ env: { HOME: home, OPENSHELL_GATEWAY: "remote" } as NodeJS.ProcessEnv }), - ).toThrow(/supports local plaintext and mTLS gateways only/); + const gateway = resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv }); + + expect(gateway.authMode).toBe("cloudflare_jwt"); + expect(gateway.sdkCompatible).toBe(true); + expect(gateway.connectOptions).toMatchObject({ + gateway: "https://gateway.example.test/", + edgeToken: "edge-token", + }); } finally { fs.rmSync(home, { recursive: true, force: true }); } }); - it("resolves mTLS gateways to the local certificate bundle", () => { + it("loads oidc_token.json access_token for SDK OIDC auth", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); try { - writeGateway(home, "secure", { - name: "secure", - gateway_endpoint: "https://127.0.0.1:17670", - auth_mode: "mtls", + const dir = writeGateway(home, "oidc", { + name: "oidc", + gateway_endpoint: "https://gateway.example.test", + auth_mode: "oidc", + oidc_issuer: "https://issuer.example.test", + oidc_client_id: "openshell", }); + fs.writeFileSync( + path.join(dir, "oidc_token.json"), + JSON.stringify({ + access_token: "oidc-token", + refresh_token: "refresh", + issuer: "https://issuer.example.test", + client_id: "openshell", + }), + { mode: 0o600 }, + ); const gateway = resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv }); - expect(gateway.target).toBe("127.0.0.1:17670"); - expect(gateway.authMode).toBe("mtls"); - expect(gateway.mtlsDir).toBe( - path.join(home, ".config", "openshell", "gateways", "secure", "mtls"), - ); + expect(gateway.authMode).toBe("oidc"); + expect(gateway.sdkCompatible).toBe(true); + expect(gateway.connectOptions).toMatchObject({ + gateway: "https://gateway.example.test/", + oidcToken: "oidc-token", + }); } finally { fs.rmSync(home, { recursive: true, force: true }); } }); - it("rejects mTLS metadata with a plaintext endpoint", () => { + it("passes CA and insecure settings through when the SDK supports the auth mode", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); try { - writeGateway(home, "bad", { - name: "bad", - gateway_endpoint: "http://127.0.0.1:8080", - auth_mode: "mtls", + const dir = writeGateway(home, "oidc", { + name: "oidc", + gateway_endpoint: "https://127.0.0.1:17670", + auth_mode: "oidc", }); + fs.writeFileSync(path.join(dir, "oidc_token.json"), JSON.stringify({ access_token: "tok" })); + fs.mkdirSync(path.join(dir, "mtls"), { recursive: true }); + fs.writeFileSync(path.join(dir, "mtls", "ca.crt"), "CA PEM"); - expect(() => resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv })).toThrow( - /requires an https:\/\/ endpoint/, - ); + const gateway = resolveGatewayMetadata({ + env: { HOME: home, OPENSHELL_GATEWAY_INSECURE: "1" } as NodeJS.ProcessEnv, + }); + + expect(gateway.connectOptions.caCert?.toString("utf-8")).toBe("CA PEM"); + expect(gateway.connectOptions.insecureSkipVerify).toBe(true); } finally { fs.rmSync(home, { recursive: true, force: true }); } }); - it("rejects plaintext metadata with a TLS endpoint", () => { + it("marks mTLS gateways as an upstream SDK prerequisite", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); try { - writeGateway(home, "bad", { - name: "bad", + writeGateway(home, "secure", { + name: "secure", gateway_endpoint: "https://127.0.0.1:17670", - auth_mode: "plaintext", + auth_mode: "mtls", }); - expect(() => resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv })).toThrow( - /requires an http:\/\/ endpoint/, - ); + const gateway = resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv }); + + expect(gateway.authMode).toBe("mtls"); + expect(gateway.sdkCompatible).toBe(false); + expect(gateway.connectOptions.gateway).toBe("https://127.0.0.1:17670/"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects missing active gateway metadata", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "openshell-gw-")); + try { + expect(() => + resolveGatewayMetadata({ env: { HOME: home } as NodeJS.ProcessEnv }), + ).toThrow(/No active OpenShell gateway/); } finally { fs.rmSync(home, { recursive: true, force: true }); } diff --git a/src/lib/adapters/openshell/gateway-metadata.ts b/src/lib/adapters/openshell/gateway-metadata.ts index 09838c1364..dbdc6fbbb0 100644 --- a/src/lib/adapters/openshell/gateway-metadata.ts +++ b/src/lib/adapters/openshell/gateway-metadata.ts @@ -5,6 +5,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { ConnectOptions } from "@openshell/sdk"; + export interface GatewayMetadata { name: string; gateway_endpoint: string; @@ -21,15 +23,20 @@ export interface GatewayMetadata { oidc_scopes?: string; } +export type GatewayAuthMode = "plaintext" | "tls" | "mtls" | "cloudflare_jwt" | "oidc"; + export interface ResolvedGatewayMetadata { name: string; endpoint: URL; target: string; - authMode: "plaintext" | "mtls"; + authMode: GatewayAuthMode; metadataPath: string | null; gatewayDir: string | null; mtlsDir: string | null; + caCertPath: string | null; insecureTls: boolean; + sdkCompatible: boolean; + connectOptions: ConnectOptions; } export interface GatewayMetadataOptions { @@ -39,6 +46,12 @@ export interface GatewayMetadataOptions { gatewayInsecure?: boolean; } +type LoadedGatewayMetadata = { + metadata: GatewayMetadata; + metadataPath: string; + gatewayDir: string; +}; + function configRoot(env: NodeJS.ProcessEnv): string { return path.join(env.XDG_CONFIG_HOME || path.join(env.HOME || os.homedir(), ".config"), "openshell"); } @@ -67,11 +80,7 @@ function readActiveGateway(env: NodeJS.ProcessEnv): string | null { } } -function readGatewayMetadata(name: string, env: NodeJS.ProcessEnv): { - metadata: GatewayMetadata; - metadataPath: string; - gatewayDir: string; -} { +function readGatewayMetadata(name: string, env: NodeJS.ProcessEnv): LoadedGatewayMetadata { const safeName = sanitizeGatewayName(name); const gatewayDir = path.join(gatewaysDir(env), safeName); const metadataPath = path.join(gatewayDir, "metadata.json"); @@ -93,7 +102,7 @@ function parseEndpoint(endpoint: string): URL { } if (url.protocol !== "http:" && url.protocol !== "https:") { throw new Error( - `OpenShell gateway endpoint '${endpoint}' must use http:// or https:// for gRPC.`, + `OpenShell gateway endpoint '${endpoint}' must use http:// or https:// for the OpenShell SDK.`, ); } return url; @@ -104,9 +113,20 @@ function endpointTarget(url: URL): string { return `${url.hostname}:${url.protocol === "https:" ? 443 : 80}`; } -function normalizeAuthMode(mode: string | null | undefined, url: URL): "plaintext" | "mtls" { +function normalizeAuthMode(mode: string | null | undefined, url: URL, fromMetadata: boolean): GatewayAuthMode { const normalized = (mode || "").trim().toLowerCase(); - if (!normalized) return url.protocol === "http:" ? "plaintext" : "mtls"; + if (!normalized) { + if (url.protocol === "http:") return "plaintext"; + return fromMetadata ? "mtls" : "tls"; + } + if (normalized === "plaintext") { + if (url.protocol !== "http:") { + throw new Error( + `OpenShell gateway auth mode 'plaintext' requires an http:// endpoint, got '${url.toString()}'.`, + ); + } + return "plaintext"; + } if (normalized === "mtls") { if (url.protocol !== "https:") { throw new Error( @@ -115,25 +135,100 @@ function normalizeAuthMode(mode: string | null | undefined, url: URL): "plaintex } return "mtls"; } - if (normalized === "plaintext") { - if (url.protocol !== "http:") { + if (normalized === "cloudflare_jwt" || normalized === "oidc") return normalized; + if (normalized === "tls") return "tls"; + throw new Error( + `OpenShell gateway auth mode '${normalized}' is not supported by NemoClaw's OpenShell SDK adapter.`, + ); +} + +function readOptionalTrimmedFile(filePath: string): string | null { + try { + const value = fs.readFileSync(filePath, "utf-8").trim(); + return value || null; + } catch { + return null; + } +} + +function readEdgeToken(gatewayDir: string, env: NodeJS.ProcessEnv): string | null { + const fromEnv = env.OPENSHELL_EDGE_TOKEN || env.OPENSHELL_GATEWAY_EDGE_TOKEN; + if (fromEnv && fromEnv.trim()) return fromEnv.trim(); + return ( + readOptionalTrimmedFile(path.join(gatewayDir, "edge_token")) || + readOptionalTrimmedFile(path.join(gatewayDir, "cf_token")) + ); +} + +function readOidcToken(gatewayDir: string, env: NodeJS.ProcessEnv): string | null { + const fromEnv = env.OPENSHELL_OIDC_TOKEN || env.OPENSHELL_GATEWAY_OIDC_TOKEN; + if (fromEnv && fromEnv.trim()) return fromEnv.trim(); + const tokenPath = path.join(gatewayDir, "oidc_token.json"); + try { + const parsed = JSON.parse(fs.readFileSync(tokenPath, "utf-8")) as { access_token?: unknown }; + return typeof parsed.access_token === "string" && parsed.access_token.trim() + ? parsed.access_token.trim() + : null; + } catch { + return null; + } +} + +function readCaCert(gatewayDir: string | null): { caCert?: Buffer; caCertPath: string | null } { + if (!gatewayDir) return { caCertPath: null }; + const caCertPath = path.join(gatewayDir, "mtls", "ca.crt"); + try { + return { caCert: fs.readFileSync(caCertPath), caCertPath }; + } catch { + return { caCertPath }; + } +} + +function buildConnectOptions(params: { + endpoint: URL; + authMode: GatewayAuthMode; + gatewayDir: string | null; + insecureTls: boolean; + env: NodeJS.ProcessEnv; +}): { connectOptions: ConnectOptions; caCertPath: string | null; sdkCompatible: boolean } { + const { caCert, caCertPath } = readCaCert(params.gatewayDir); + const connectOptions: ConnectOptions = { + gateway: params.endpoint.toString(), + ...(caCert ? { caCert } : {}), + ...(params.insecureTls ? { insecureSkipVerify: true } : {}), + }; + + if (params.authMode === "cloudflare_jwt") { + if (!params.gatewayDir) { + throw new Error("OpenShell Cloudflare gateway metadata is required to load edge_token."); + } + const edgeToken = readEdgeToken(params.gatewayDir, params.env); + if (!edgeToken) { throw new Error( - `OpenShell gateway auth mode 'plaintext' requires an http:// endpoint, got '${url.toString()}'.`, + "OpenShell Cloudflare gateway is selected, but no edge_token is available. Run `openshell gateway login`.", ); } - return "plaintext"; + connectOptions.edgeToken = edgeToken; } - if (normalized === "cloudflare_jwt" || normalized === "oidc") { - throw new Error( - `OpenShell gateway auth mode '${normalized}' is bearer-token based. ` + - "NemoClaw's direct gRPC transport currently supports local plaintext and mTLS gateways only. " + - "Select a local NemoClaw gateway or configure mTLS for this gateway.", - ); + + if (params.authMode === "oidc") { + if (!params.gatewayDir) { + throw new Error("OpenShell OIDC gateway metadata is required to load oidc_token.json."); + } + const oidcToken = readOidcToken(params.gatewayDir, params.env); + if (!oidcToken) { + throw new Error( + "OpenShell OIDC gateway is selected, but no oidc_token.json access_token is available. Run `openshell gateway login`.", + ); + } + connectOptions.oidcToken = oidcToken; } - throw new Error( - `OpenShell gateway auth mode '${normalized}' is not supported by NemoClaw's direct gRPC transport. ` + - "Supported modes: plaintext, mtls.", - ); + + return { + connectOptions, + caCertPath, + sdkCompatible: params.authMode !== "mtls", + }; } export function resolveGatewayMetadata( @@ -148,49 +243,54 @@ export function resolveGatewayMetadata( env.OPENSHELL_GATEWAY_INSECURE === "1" || env.OPENSHELL_GATEWAY_INSECURE === "true"; - if (endpointOverride) { - const endpoint = parseEndpoint(endpointOverride); - return { - name: gatewayName || "endpoint", - endpoint, - target: endpointTarget(endpoint), - authMode: normalizeAuthMode(endpoint.protocol === "http:" ? "plaintext" : "mtls", endpoint), - metadataPath: null, - gatewayDir: gatewayName ? path.join(gatewaysDir(env), sanitizeGatewayName(gatewayName)) : null, - mtlsDir: gatewayName - ? path.join(gatewaysDir(env), sanitizeGatewayName(gatewayName), "mtls") - : null, - insecureTls, - }; + let loaded: LoadedGatewayMetadata | null = null; + if (gatewayName) { + try { + loaded = readGatewayMetadata(gatewayName, env); + } catch (error) { + if (!endpointOverride) { + throw new Error( + `Failed to load OpenShell gateway metadata for '${gatewayName}': ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + } } - if (!gatewayName) { + if (!endpointOverride && !loaded) { throw new Error( "No active OpenShell gateway is configured. Run `openshell gateway select ` or set OPENSHELL_GATEWAY.", ); } - let loaded: ReturnType; - try { - loaded = readGatewayMetadata(gatewayName, env); - } catch (error) { - throw new Error( - `Failed to load OpenShell gateway metadata for '${gatewayName}': ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } + const endpoint = parseEndpoint(endpointOverride || loaded?.metadata.gateway_endpoint || ""); + const name = loaded?.metadata.name || gatewayName || "endpoint"; + const gatewayDir = loaded?.gatewayDir ?? (gatewayName ? path.join(gatewaysDir(env), sanitizeGatewayName(gatewayName)) : null); + const authMode = normalizeAuthMode( + loaded?.metadata.auth_mode ?? env.OPENSHELL_GATEWAY_AUTH_MODE, + endpoint, + loaded !== null, + ); + const { connectOptions, caCertPath, sdkCompatible } = buildConnectOptions({ + endpoint, + authMode, + gatewayDir, + insecureTls, + env, + }); - const endpoint = parseEndpoint(loaded.metadata.gateway_endpoint); - const authMode = normalizeAuthMode(loaded.metadata.auth_mode, endpoint); return { - name: loaded.metadata.name || gatewayName, + name, endpoint, target: endpointTarget(endpoint), authMode, - metadataPath: loaded.metadataPath, - gatewayDir: loaded.gatewayDir, - mtlsDir: path.join(loaded.gatewayDir, "mtls"), + metadataPath: loaded?.metadataPath ?? null, + gatewayDir, + mtlsDir: gatewayDir ? path.join(gatewayDir, "mtls") : null, + caCertPath, insecureTls, + sdkCompatible, + connectOptions, }; } diff --git a/src/lib/adapters/openshell/grpc-migration-guard.test.ts b/src/lib/adapters/openshell/grpc-migration-guard.test.ts index 7c8d9663b2..e46d60284a 100644 --- a/src/lib/adapters/openshell/grpc-migration-guard.test.ts +++ b/src/lib/adapters/openshell/grpc-migration-guard.test.ts @@ -21,10 +21,19 @@ const PRODUCTION_TARGETS = [ "src/lib/status-command-deps.ts", "src/lib/tunnel/services.ts", "src/lib/verify-deployment.ts", + "src/lib/adapters/openshell/grpc.ts", + "src/lib/adapters/openshell/sync-runner.ts", + "package.json", "scripts/install.sh", ]; const BANNED = [ + /@grpc\/grpc-js/, + /@grpc\/proto-loader/, + /\bprotoLoader\b/, + /\bloadPackageDefinition\b/, + /\bCreateSshSession\b/, + /\bForwardTcp\b/, /captureSandboxSshConfig/, /sandbox ssh-config/, /spawnSync\(\s*["']ssh["']/, @@ -33,6 +42,7 @@ const BANNED = [ /sandbox download/, /forward start --background/, /openshell forward start/, + /copy-openshell-protos/, ]; function filesUnder(target: string): string[] { @@ -52,7 +62,7 @@ function filesUnder(target: string): string[] { return out; } -describe("OpenShell gRPC migration guard", () => { +describe("OpenShell SDK migration guard", () => { it("keeps sandbox lifecycle production code off SSH and SSH-backed forwards", () => { const violations: string[] = []; for (const target of PRODUCTION_TARGETS) { diff --git a/src/lib/adapters/openshell/grpc.test.ts b/src/lib/adapters/openshell/grpc.test.ts index 2ac7d491b4..1206447af4 100644 --- a/src/lib/adapters/openshell/grpc.test.ts +++ b/src/lib/adapters/openshell/grpc.test.ts @@ -1,95 +1,84 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { EventEmitter } from "node:events"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { __grpcTestHooks, execBinaryStreamSync } from "./grpc"; +vi.mock("@openshell/sdk", () => ({ + OpenShellClient: { connect: vi.fn() }, +})); -function collect(events: unknown[]) { - const stream = new EventEmitter() as EventEmitter & { cancel: () => void }; - stream.cancel = () => undefined; - const result = __grpcTestHooks.collectExecStream(stream as any, "test exec", 0); - queueMicrotask(() => { - for (const event of events) stream.emit("data", event); - stream.emit("end"); - }); - return result; -} +import { __grpcTestHooks, execBinaryStreamSync } from "./grpc"; -describe("OpenShell gRPC exec stream parsing", () => { - it("parses keepCase exit_code events", async () => { - await expect( - collect([ - { stdout: { data: Buffer.from("ok\n") } }, - { exit: { exit_code: 0 } }, - ]), - ).resolves.toMatchObject({ - status: 0, - stdout: Buffer.from("ok\n"), +describe("OpenShell SDK adapter", () => { + it("maps NemoClaw exec options to SDK exec options", () => { + const opts = __grpcTestHooks.toSdkExecOptions({ + workdir: "/sandbox/workspace", + env: { A: "1" }, + timeoutMs: 1500, + stdin: "hello", }); + + expect(opts.workdir).toBe("/sandbox/workspace"); + expect(opts.environment).toEqual({ A: "1" }); + expect(opts.timeoutSecs).toBe(2); + expect(opts.stdin?.toString("utf-8")).toBe("hello"); }); - it("parses camelCase exitCode events", async () => { - await expect( - collect([ - { stderr: { data: Buffer.from("boom\n") } }, - { exit: { exitCode: 42 } }, - ]), - ).resolves.toMatchObject({ - status: 42, - stderr: Buffer.from("boom\n"), + it("normalizes SDK exec results into NemoClaw's binary result shape", () => { + const result = __grpcTestHooks.sdkResultToStream({ + exitCode: 42, + stdout: Buffer.from("out"), + stderr: Buffer.from("err"), }); - }); - it("keeps missing exit events non-successful so call sites must prove completion", async () => { - await expect(collect([{ stdout: { data: Buffer.from("partial") } }])).resolves.toMatchObject({ - status: 1, - stdout: Buffer.from("partial"), + expect(result).toEqual({ + status: 42, + stdout: Buffer.from("out"), + stderr: Buffer.from("err"), }); }); - it("uses unary stdin for modest non-tty input so script uploads do not hang waiting for interactive EOF", () => { - expect(__grpcTestHooks.shouldInlineExecInput(Buffer.alloc(1024), {})).toBe(true); - expect(__grpcTestHooks.shouldInlineExecInput(Buffer.alloc(1024), { tty: true })).toBe(false); - expect(__grpcTestHooks.shouldInlineExecInput(Buffer.alloc(16 * 1024 * 1024 + 1), {})).toBe(false); + it("formats SDK error-code prefixes without exposing raw binding noise", () => { + expect(__grpcTestHooks.formatSdkError(new Error("[not_found] sandbox missing"))).toBe( + "not_found: sandbox missing", + ); + expect(__grpcTestHooks.formatSdkError(new Error("plain failure"))).toBe("plain failure"); }); - it("preserves large sync-runner binary stdout without hitting spawnSync ENOBUFS", () => { - const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-grpc-large-stdout-")); + it("preserves large sync-runner binary stdout through the fake SDK runner", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sdk-large-stdout-")); const oldEnv = { - transport: process.env.NEMOCLAW_GRPC_TEST_TRANSPORT, - legacy: process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH, - fakeSsh: process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN, + transport: process.env.NEMOCLAW_SDK_TEST_TRANSPORT, + fakeExec: process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN, }; try { - const fakeSsh = path.join(fixture, "ssh"); + const fakeExec = path.join(fixture, "sdk-fake-exec.cjs"); fs.writeFileSync( - fakeSsh, + fakeExec, `#!/usr/bin/env node -require("node:fs").writeSync(1, Buffer.alloc(2 * 1024 * 1024, 0x61)); -process.exit(0); +process.stdout.write(JSON.stringify({ + status: 0, + stdoutBase64: Buffer.alloc(2 * 1024 * 1024, 0x61).toString("base64"), + stderrBase64: "" +})); `, { mode: 0o755 }, ); - process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; - process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; - process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = fakeSsh; + process.env.NEMOCLAW_SDK_TEST_TRANSPORT = "1"; + process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN = fakeExec; const result = execBinaryStreamSync("alpha", ["cat", "/tmp/large"], { timeoutMs: 15_000 }); expect(result.status).toBe(0); expect(result.stdout.length).toBe(2 * 1024 * 1024); } finally { - if (oldEnv.transport === undefined) delete process.env.NEMOCLAW_GRPC_TEST_TRANSPORT; - else process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = oldEnv.transport; - if (oldEnv.legacy === undefined) delete process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH; - else process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = oldEnv.legacy; - if (oldEnv.fakeSsh === undefined) delete process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN; - else process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = oldEnv.fakeSsh; + if (oldEnv.transport === undefined) delete process.env.NEMOCLAW_SDK_TEST_TRANSPORT; + else process.env.NEMOCLAW_SDK_TEST_TRANSPORT = oldEnv.transport; + if (oldEnv.fakeExec === undefined) delete process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN; + else process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN = oldEnv.fakeExec; fs.rmSync(fixture, { recursive: true, force: true }); } }); diff --git a/src/lib/adapters/openshell/grpc.ts b/src/lib/adapters/openshell/grpc.ts index 4e75c8d535..b530a70816 100644 --- a/src/lib/adapters/openshell/grpc.ts +++ b/src/lib/adapters/openshell/grpc.ts @@ -3,12 +3,16 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; -import net from "node:net"; import path from "node:path"; -import { fileURLToPath } from "node:url"; -import * as grpc from "@grpc/grpc-js"; -import * as protoLoader from "@grpc/proto-loader"; +import type { + ExecOptions as SdkExecOptions, + ExecResult as SdkExecResult, + Health, + ListOptions, + SandboxRef, + SandboxSpec, +} from "@openshell/sdk"; import { type GatewayMetadataOptions, @@ -39,30 +43,8 @@ export interface SandboxStreamResult { stderr: Buffer; } -export interface SandboxLogLine { - sandbox_id?: string; - timestamp_ms?: number | string; - level?: string; - target?: string; - message?: string; - source?: string; - fields?: Record; -} - -export interface SandboxLogsResult { - logs: SandboxLogLine[]; - bufferTotal: number; -} - -export type SandboxWatchEvent = - | { type: "sandbox"; sandbox: unknown } - | { type: "log"; log: SandboxLogLine } - | { type: "event"; event: unknown } - | { type: "warning"; message: string }; - export interface SandboxForwardHandle { sandboxName: string; - sandboxId: string; localPort: number; localHost: string; targetHost: string; @@ -74,243 +56,187 @@ export interface SandboxGrpcClientOptions extends GatewayMetadataOptions { gateway?: ResolvedGatewayMetadata; } -type GrpcClient = grpc.Client & Record any>; - -const DEFAULT_GRPC_TIMEOUT_MS = 30_000; -const STREAM_CHUNK_SIZE = 64 * 1024; -const INLINE_STDIN_LIMIT = 16 * 1024 * 1024; +const DEFAULT_SDK_TIMEOUT_MS = 30_000; +const MAX_SYNC_STDIO_BUFFER = 512 * 1024 * 1024; -let packageDefinition: grpc.GrpcObject | null = null; +type OpenShellClientConstructor = typeof import("@openshell/sdk").OpenShellClient; +type OpenShellClientInstance = InstanceType; -function protoRoot(): string { - return path.join(__dirname, "proto"); -} +const clientCache = new Map>(); +let sdkClientConstructor: OpenShellClientConstructor | null = null; -function loadOpenShellPackage(): grpc.GrpcObject { - if (packageDefinition) return packageDefinition; - const definition = protoLoader.loadSync(path.join(protoRoot(), "openshell.proto"), { - defaults: false, - enums: String, - includeDirs: [protoRoot()], - keepCase: true, - longs: String, - oneofs: true, - bytes: Buffer, - }); - packageDefinition = grpc.loadPackageDefinition(definition) as grpc.GrpcObject; - return packageDefinition; -} - -function getOpenShellConstructor(): typeof grpc.Client { - const loaded = loadOpenShellPackage() as any; - const ctor = loaded.openshell?.v1?.OpenShell; - if (!ctor) { - throw new Error("Failed to load openshell.v1.OpenShell from vendored proto files."); +function openShellClientConstructor(): OpenShellClientConstructor { + if (!sdkClientConstructor) { + sdkClientConstructor = require("@openshell/sdk").OpenShellClient as OpenShellClientConstructor; } - return ctor; + return sdkClientConstructor; } -function readRequiredFile(filePath: string, label: string): Buffer { - try { - return fs.readFileSync(filePath); - } catch (error) { - throw new Error( - `Failed to read OpenShell ${label} at ${filePath}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } +function cacheKey(gateway: ResolvedGatewayMetadata): string { + return [ + gateway.name, + gateway.connectOptions.gateway, + gateway.authMode, + gateway.insecureTls ? "insecure" : "verify", + gateway.caCertPath || "", + ].join("\n"); } -function createCredentials(gateway: ResolvedGatewayMetadata): grpc.ChannelCredentials { - if (gateway.authMode === "plaintext" || gateway.endpoint.protocol === "http:") { - return grpc.credentials.createInsecure(); - } - if (!gateway.mtlsDir) { +function getSharedSdkClient(gateway: ResolvedGatewayMetadata): Promise { + if (!gateway.sdkCompatible) { throw new Error( - "OpenShell mTLS gateway metadata did not include a certificate directory. " + - "Run `openshell gateway info` and verify the gateway is registered locally.", + "The selected OpenShell gateway uses mTLS client-certificate authentication, " + + "but @openshell/sdk does not expose client-certificate auth yet. " + + "Select a plaintext, OIDC, or Cloudflare JWT gateway, or land SDK mTLS support upstream first.", ); } - const ca = readRequiredFile(path.join(gateway.mtlsDir, "ca.crt"), "mTLS CA"); - const cert = readRequiredFile(path.join(gateway.mtlsDir, "tls.crt"), "mTLS client certificate"); - const key = readRequiredFile(path.join(gateway.mtlsDir, "tls.key"), "mTLS client key"); - return grpc.credentials.createSsl(ca, key, cert); -} - -function method any>(client: GrpcClient, name: string): T { - const lower = name.charAt(0).toLowerCase() + name.slice(1); - const fn = client[lower] || client[name]; - if (typeof fn !== "function") { - throw new Error(`OpenShell gRPC method ${name} is unavailable.`); + const key = cacheKey(gateway); + let existing = clientCache.get(key); + if (!existing) { + existing = openShellClientConstructor().connect(gateway.connectOptions); + clientCache.set(key, existing); } - return fn.bind(client) as T; + return existing; } -function callOptions(timeoutMs?: number): grpc.CallOptions { - const deadlineMs = timeoutMs ?? DEFAULT_GRPC_TIMEOUT_MS; - return deadlineMs > 0 ? { deadline: new Date(Date.now() + deadlineMs) } : {}; -} - -function grpcStatusName(code: number | undefined): string { - if (typeof code !== "number") return "UNKNOWN"; - return grpc.status[code] || String(code); -} - -export function formatGrpcError(error: unknown): string { - if (!error || typeof error !== "object") return String(error); - const maybe = error as Partial; - const status = grpcStatusName(maybe.code); - const details = maybe.details || maybe.message || String(error); - return `${status}: ${details}`; +function stdinBuffer(input: Buffer | string | undefined): Buffer | undefined { + if (input === undefined) return undefined; + return Buffer.isBuffer(input) ? input : Buffer.from(input); } -function makeTimeoutError(label: string, timeoutMs: number): Error { - const error = new Error(`${label} timed out after ${timeoutMs} ms`) as NodeJS.ErrnoException; - error.code = "ETIMEDOUT"; - return error; +function timeoutSecs(opts: SandboxExecOptions): number | undefined { + if (opts.timeoutSeconds !== undefined) return opts.timeoutSeconds; + if (opts.timeoutMs === undefined || opts.timeoutMs <= 0) return undefined; + return Math.max(1, Math.ceil(opts.timeoutMs / 1000)); } -function bufferFromData(data: unknown): Buffer { - if (Buffer.isBuffer(data)) return data; - if (data instanceof Uint8Array) return Buffer.from(data); - if (typeof data === "string") return Buffer.from(data, "base64"); - return Buffer.alloc(0); +function toSdkExecOptions(opts: SandboxExecOptions = {}): SdkExecOptions { + return { + ...(opts.workdir ? { workdir: opts.workdir } : {}), + ...(opts.env ? { environment: opts.env } : {}), + ...(timeoutSecs(opts) !== undefined ? { timeoutSecs: timeoutSecs(opts) } : {}), + ...(stdinBuffer(opts.stdin) ? { stdin: stdinBuffer(opts.stdin) } : {}), + }; } -function exitCodeFromEvent(event: any): number | null { - const raw = event?.exit?.exit_code ?? event?.exit?.exitCode; - if (raw === undefined || raw === null) return null; +function sdkExitCode(result: SdkExecResult): number { + const raw = (result as SdkExecResult & { exit_code?: unknown }).exitCode ?? (result as any).exit_code; const parsed = Number(raw); return Number.isFinite(parsed) ? parsed : 1; } -function stdinBuffer(input: Buffer | string | undefined): Buffer | undefined { - if (input === undefined) return undefined; - return Buffer.isBuffer(input) ? input : Buffer.from(input); -} - -function shouldInlineExecInput(input: Buffer, opts: SandboxExecOptions = {}): boolean { - return opts.tty !== true && input.length <= INLINE_STDIN_LIMIT; -} - -function sandboxIdFromResponse(response: any, sandboxName: string): string { - const sandbox = response?.sandbox; - const id = sandbox?.metadata?.id || sandbox?.id; - if (typeof id === "string" && id.trim()) return id; - throw new Error(`OpenShell gateway returned no sandbox id for '${sandboxName}'.`); +function sdkBuffer(value: Buffer | Uint8Array | string | undefined): Buffer { + if (Buffer.isBuffer(value)) return value; + if (value instanceof Uint8Array) return Buffer.from(value); + if (typeof value === "string") return Buffer.from(value); + return Buffer.alloc(0); } -function execRequest(sandboxId: string, argv: string[], opts: SandboxExecOptions = {}) { +function sdkResultToStream(result: SdkExecResult): SandboxStreamResult { return { - sandbox_id: sandboxId, - command: argv, - workdir: opts.workdir || "", - environment: opts.env || {}, - timeout_seconds: opts.timeoutSeconds ?? Math.max(0, Math.ceil((opts.timeoutMs ?? 0) / 1000)), - stdin: stdinBuffer(opts.stdin), - tty: opts.tty === true, - cols: opts.cols ?? 0, - rows: opts.rows ?? 0, + status: sdkExitCode(result), + stdout: sdkBuffer(result.stdout), + stderr: sdkBuffer(result.stderr), }; } -function collectExecStream( - stream: grpc.ClientReadableStream, - label: string, - timeoutMs?: number, -): Promise { - return new Promise((resolve, reject) => { - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - let exitCode: number | null = null; - let settled = false; - let timer: NodeJS.Timeout | undefined; - - const settle = (fn: () => void) => { - if (settled) return; - settled = true; - if (timer) clearTimeout(timer); - fn(); - }; - - stream.on("data", (event: any) => { - if (event?.stdout?.data !== undefined) stdout.push(bufferFromData(event.stdout.data)); - if (event?.stderr?.data !== undefined) stderr.push(bufferFromData(event.stderr.data)); - const parsedExitCode = exitCodeFromEvent(event); - if (parsedExitCode !== null) exitCode = parsedExitCode; - }); - stream.on("error", (error) => { - settle(() => reject(new Error(`${label} failed: ${formatGrpcError(error)}`))); - }); - stream.on("end", () => { - settle(() => - resolve({ - status: exitCode ?? 1, - stdout: Buffer.concat(stdout), - stderr: Buffer.concat(stderr), - }), - ); - }); - - if (timeoutMs && timeoutMs > 0) { - timer = setTimeout(() => { - try { - stream.cancel(); - } catch { - /* ignore */ - } - settle(() => reject(makeTimeoutError(label, timeoutMs))); - }, timeoutMs); - } - }); +function sdkErrorCode(error: unknown): string | null { + if (!(error instanceof Error)) return null; + const match = /^\[([^\]]+)\]/.exec(error.message); + return match ? match[1] : null; } -function collectDuplexExecStream( - stream: grpc.ClientDuplexStream, - label: string, - timeoutMs?: number, -): Promise { - return collectExecStream(stream as unknown as grpc.ClientReadableStream, label, timeoutMs); +export function formatSdkError(error: unknown): string { + const code = sdkErrorCode(error); + const message = error instanceof Error ? error.message : String(error); + const clean = message.replace(/^\[[^\]]+\]\s*/, ""); + return code ? `${code}: ${clean}` : clean; } -function writeInputChunks(stream: grpc.ClientDuplexStream, input: Buffer): void { - for (let offset = 0; offset < input.length; offset += STREAM_CHUNK_SIZE) { - stream.write({ stdin: input.subarray(offset, offset + STREAM_CHUNK_SIZE) }); - } -} +export const formatGrpcError = formatSdkError; export class SandboxGrpcClient { readonly gateway: ResolvedGatewayMetadata; - private readonly client: GrpcClient; constructor(options: SandboxGrpcClientOptions = {}) { this.gateway = options.gateway ?? resolveGatewayMetadata(options); - const OpenShell = getOpenShellConstructor(); - this.client = new OpenShell(this.gateway.target, createCredentials(this.gateway), { - "grpc.max_receive_message_length": 256 * 1024 * 1024, - "grpc.max_send_message_length": 256 * 1024 * 1024, - }) as GrpcClient; + } + + private sdk(): Promise { + return getSharedSdkClient(this.gateway); } close(): void { - this.client.close(); + // @openshell/sdk clients are shared and currently expose no close method. + } + + async health(): Promise { + try { + return await (await this.sdk()).health(); + } catch (error) { + throw new Error(`OpenShell SDK health failed: ${formatSdkError(error)}`); + } + } + + async createSandbox(spec: SandboxSpec): Promise { + try { + return await (await this.sdk()).createSandbox(spec); + } catch (error) { + throw new Error(`CreateSandbox failed: ${formatSdkError(error)}`); + } + } + + async getSandbox(sandboxName: string): Promise { + try { + return await (await this.sdk()).getSandbox(sandboxName); + } catch (error) { + throw new Error(`GetSandbox '${sandboxName}' failed: ${formatSdkError(error)}`); + } + } + + async listSandboxes(options?: ListOptions | null): Promise { + try { + return await (await this.sdk()).listSandboxes(options); + } catch (error) { + throw new Error(`ListSandboxes failed: ${formatSdkError(error)}`); + } + } + + async deleteSandbox(sandboxName: string): Promise { + try { + return await (await this.sdk()).deleteSandbox(sandboxName); + } catch (error) { + throw new Error(`DeleteSandbox '${sandboxName}' failed: ${formatSdkError(error)}`); + } + } + + async waitReady(sandboxName: string, timeoutSeconds: number): Promise { + try { + return await (await this.sdk()).waitReady(sandboxName, timeoutSeconds); + } catch (error) { + throw new Error(`WaitReady '${sandboxName}' failed: ${formatSdkError(error)}`); + } } - async getSandboxId(sandboxName: string, timeoutMs?: number): Promise { - const getSandbox = method< - (request: { name: string }, options: grpc.CallOptions, callback: grpc.requestCallback) => void - >(this.client, "GetSandbox"); - const response = await new Promise((resolve, reject) => { - getSandbox({ name: sandboxName }, callOptions(timeoutMs), (error, value) => { - if (error) { - reject(new Error(`GetSandbox '${sandboxName}' failed: ${formatGrpcError(error)}`)); - } else { - resolve(value); - } - }); - }); - return sandboxIdFromResponse(response, sandboxName); + async waitDeleted(sandboxName: string, timeoutSeconds: number): Promise { + try { + await (await this.sdk()).waitDeleted(sandboxName, timeoutSeconds); + } catch (error) { + throw new Error(`WaitDeleted '${sandboxName}' failed: ${formatSdkError(error)}`); + } + } + + async execBinary( + sandboxName: string, + argv: string[], + opts: SandboxExecOptions = {}, + ): Promise { + try { + const result = await (await this.sdk()).exec(sandboxName, argv, toSdkExecOptions(opts)); + return sdkResultToStream(result); + } catch (error) { + throw new Error(`OpenShell SDK exec '${sandboxName}' failed: ${formatSdkError(error)}`); + } } async execBinaryStream( @@ -318,16 +244,7 @@ export class SandboxGrpcClient { argv: string[], opts: SandboxExecOptions = {}, ): Promise { - const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); - const execSandbox = method< - (request: unknown, options?: grpc.CallOptions) => grpc.ClientReadableStream - >(this.client, "ExecSandbox"); - const request = execRequest(sandboxId, argv, opts); - return collectExecStream( - execSandbox(request, callOptions(opts.timeoutMs)), - `ExecSandbox '${sandboxName}'`, - opts.timeoutMs, - ); + return this.execBinary(sandboxName, argv, opts); } async execText( @@ -335,7 +252,7 @@ export class SandboxGrpcClient { argv: string[], opts: SandboxExecOptions = {}, ): Promise { - const result = await this.execBinaryStream(sandboxName, argv, opts); + const result = await this.execBinary(sandboxName, argv, opts); return { status: result.status, stdout: result.stdout.toString("utf-8"), @@ -349,191 +266,12 @@ export class SandboxGrpcClient { input: Buffer | string, opts: SandboxExecOptions = {}, ): Promise { - const inputBuffer = stdinBuffer(input) ?? Buffer.alloc(0); - if (shouldInlineExecInput(inputBuffer, opts)) { - return this.execBinaryStream(sandboxName, argv, { ...opts, stdin: inputBuffer }); - } - - const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); - const execInteractive = method<() => grpc.ClientDuplexStream>( - this.client, - "ExecSandboxInteractive", - ); - const stream = execInteractive(); - const result = collectDuplexExecStream( - stream, - `ExecSandboxInteractive '${sandboxName}'`, - opts.timeoutMs, - ); - stream.write({ start: execRequest(sandboxId, argv, opts) }); - writeInputChunks(stream, inputBuffer); - stream.end(); - return result; - } - - async getSandboxLogs( - sandboxName: string, - opts: { - lines?: number; - sinceMs?: number; - sources?: string[]; - minLevel?: string; - timeoutMs?: number; - } = {}, - ): Promise { - const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); - const getLogs = method< - (request: unknown, options: grpc.CallOptions, callback: grpc.requestCallback) => void - >(this.client, "GetSandboxLogs"); - const response = await new Promise((resolve, reject) => { - getLogs( - { - sandbox_id: sandboxId, - lines: opts.lines ?? 0, - since_ms: opts.sinceMs ?? 0, - sources: opts.sources ?? [], - min_level: opts.minLevel ?? "", - }, - callOptions(opts.timeoutMs), - (error, value) => { - if (error) { - reject(new Error(`GetSandboxLogs '${sandboxName}' failed: ${formatGrpcError(error)}`)); - } else { - resolve(value); - } - }, - ); - }); - return { - logs: Array.isArray(response?.logs) ? response.logs : [], - bufferTotal: Number(response?.buffer_total ?? 0), - }; - } - - async watchSandbox( - sandboxName: string, - opts: { - followStatus?: boolean; - followLogs?: boolean; - followEvents?: boolean; - logTailLines?: number; - eventTail?: number; - stopOnTerminal?: boolean; - logSinceMs?: number; - logSources?: string[]; - logMinLevel?: string; - timeoutMs?: number; - onEvent?: (event: SandboxWatchEvent) => void; - } = {}, - ): Promise { - const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); - const watchSandbox = method< - (request: unknown, options?: grpc.CallOptions) => grpc.ClientReadableStream - >(this.client, "WatchSandbox"); - const stream = watchSandbox( - { - id: sandboxId, - follow_status: opts.followStatus === true, - follow_logs: opts.followLogs === true, - follow_events: opts.followEvents === true, - log_tail_lines: opts.logTailLines ?? 0, - event_tail: opts.eventTail ?? 0, - stop_on_terminal: opts.stopOnTerminal === true, - log_since_ms: opts.logSinceMs ?? 0, - log_sources: opts.logSources ?? [], - log_min_level: opts.logMinLevel ?? "", - }, - callOptions(opts.timeoutMs), - ); - await new Promise((resolve, reject) => { - stream.on("data", (event: any) => { - if (!opts.onEvent) return; - if (event?.sandbox !== undefined) opts.onEvent({ type: "sandbox", sandbox: event.sandbox }); - if (event?.log !== undefined) opts.onEvent({ type: "log", log: event.log }); - if (event?.event !== undefined) opts.onEvent({ type: "event", event: event.event }); - if (event?.warning !== undefined) { - opts.onEvent({ - type: "warning", - message: String(event.warning.message || ""), - }); - } - }); - stream.on("error", (error) => { - reject(new Error(`WatchSandbox '${sandboxName}' failed: ${formatGrpcError(error)}`)); - }); - stream.on("end", () => resolve()); - }); - } - - async createSshSession( - sandboxName: string, - timeoutMs = DEFAULT_GRPC_TIMEOUT_MS, - ): Promise<{ sandboxId: string; token: string }> { - const sandboxId = await this.getSandboxId(sandboxName, timeoutMs); - const createSession = method< - ( - request: { sandbox_id: string }, - options: grpc.CallOptions, - callback: grpc.requestCallback, - ) => void - >(this.client, "CreateSshSession"); - const response = await new Promise((resolve, reject) => { - createSession({ sandbox_id: sandboxId }, callOptions(timeoutMs), (error, value) => { - if (error) { - reject(new Error(`CreateSshSession '${sandboxName}' failed: ${formatGrpcError(error)}`)); - } else { - resolve(value); - } - }); - }); - const token = response?.token; - if (typeof token !== "string" || token.length === 0) { - throw new Error(`OpenShell gateway returned no relay token for '${sandboxName}'.`); - } - return { sandboxId, token }; - } - - async forwardTcpConnection( - socket: net.Socket, - params: { - sandboxName: string; - targetHost: string; - targetPort: number; - serviceId?: string; - timeoutMs?: number; - }, - ): Promise { - const { sandboxId, token } = await this.createSshSession(params.sandboxName, params.timeoutMs); - const forwardTcp = method<() => grpc.ClientDuplexStream>(this.client, "ForwardTcp"); - const stream = forwardTcp(); - stream.write({ - init: { - sandbox_id: sandboxId, - service_id: params.serviceId || "", - tcp: { host: params.targetHost, port: params.targetPort }, - authorization_token: token, - }, - }); - socket.on("data", (chunk) => stream.write({ data: chunk })); - socket.on("end", () => stream.end()); - socket.on("error", () => { - try { - stream.cancel(); - } catch { - /* ignore */ - } - }); - stream.on("data", (frame: any) => { - const data = frame?.data; - if (data !== undefined) socket.write(bufferFromData(data)); - }); - stream.on("end", () => socket.end()); - stream.on("error", () => socket.destroy()); + return this.execBinary(sandboxName, argv, { ...opts, stdin: stdinBuffer(input) }); } async startForward( sandboxName: string, - options: { + _options: { localPort: number; localHost?: string; targetHost?: string; @@ -541,36 +279,10 @@ export class SandboxGrpcClient { serviceId?: string; }, ): Promise { - const localHost = options.localHost || "127.0.0.1"; - const targetHost = options.targetHost || "127.0.0.1"; - const sandboxId = await this.getSandboxId(sandboxName); - const server = net.createServer((socket) => { - this.forwardTcpConnection(socket, { - sandboxName, - targetHost, - targetPort: options.targetPort, - serviceId: options.serviceId, - }).catch(() => socket.destroy()); - }); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(options.localPort, localHost, () => { - server.off("error", reject); - resolve(); - }); - }); - return { - sandboxName, - sandboxId, - localPort: options.localPort, - localHost, - targetHost, - targetPort: options.targetPort, - close: () => - new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }), - }; + throw new Error( + `OpenShell SDK forwarding is not available for '${sandboxName}' yet. ` + + "Land an upstream @openshell/sdk forwarding API before enabling dashboard forwarding through the SDK.", + ); } } @@ -599,7 +311,7 @@ function syncRunnerCommand(): { command: string; args: string[] } { const tsxBin = path.join(root, "node_modules", ".bin", process.platform === "win32" ? "tsx.cmd" : "tsx"); if (fs.existsSync(source) && fs.existsSync(tsxBin)) return { command: tsxBin, args: [source] }; - throw new Error("OpenShell gRPC sync runner is not available. Run `npm run build:cli` first."); + throw new Error("OpenShell SDK sync runner is not available. Run `npm run build:cli` first."); } function runSyncRunner( @@ -612,7 +324,7 @@ function runSyncRunner( encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], timeout: timeoutMs && timeoutMs > 0 ? timeoutMs + 5_000 : undefined, - maxBuffer: 512 * 1024 * 1024, + maxBuffer: MAX_SYNC_STDIO_BUFFER, env: process.env, }); if (result.error) { @@ -625,7 +337,7 @@ function runSyncRunner( return JSON.parse(result.stdout) as SyncRunnerResponse; } catch (error) { throw new Error( - `OpenShell gRPC sync runner returned invalid JSON: ${ + `OpenShell SDK sync runner returned invalid JSON: ${ error instanceof Error ? error.message : String(error) }`, ); @@ -638,7 +350,7 @@ export function execTextSync( opts: SandboxExecOptions = {}, ): SandboxExecResult { const response = runSyncRunner({ op: "execText", sandboxName, argv, opts }, opts.timeoutMs); - if (!response.ok || !response.result) throw new Error(response.error || "OpenShell gRPC exec failed"); + if (!response.ok || !response.result) throw new Error(response.error || "OpenShell SDK exec failed"); return { status: response.result.status, stdout: response.result.stdout || "", @@ -652,7 +364,7 @@ export function execBinaryStreamSync( opts: SandboxExecOptions = {}, ): SandboxStreamResult { const response = runSyncRunner({ op: "execBinary", sandboxName, argv, opts }, opts.timeoutMs); - if (!response.ok || !response.result) throw new Error(response.error || "OpenShell gRPC exec failed"); + if (!response.ok || !response.result) throw new Error(response.error || "OpenShell SDK exec failed"); return { status: response.result.status, stdout: Buffer.from(response.result.stdoutBase64 || "", "base64"), @@ -671,7 +383,7 @@ export function execInputStreamSync( { op: "execInput", sandboxName, argv, inputBase64, opts }, opts.timeoutMs, ); - if (!response.ok || !response.result) throw new Error(response.error || "OpenShell gRPC exec failed"); + if (!response.ok || !response.result) throw new Error(response.error || "OpenShell SDK exec failed"); return { status: response.result.status, stdout: Buffer.from(response.result.stdoutBase64 || "", "base64"), @@ -679,11 +391,13 @@ export function execInputStreamSync( }; } +export function __clearSandboxSdkClientCacheForTests(): void { + clientCache.clear(); +} + export const __grpcTestHooks = { - collectExecStream, - exitCodeFromEvent, - shouldInlineExecInput, + formatSdkError, + toSdkExecOptions, + sdkResultToStream, + cacheKey, }; - -// Keep CommonJS transpilation happy if this file is ever executed through ESM-aware tooling. -void fileURLToPath; diff --git a/src/lib/adapters/openshell/proto/datamodel.proto b/src/lib/adapters/openshell/proto/datamodel.proto deleted file mode 100644 index ee4434d4fc..0000000000 --- a/src/lib/adapters/openshell/proto/datamodel.proto +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; - -package openshell.datamodel.v1; - -// Minimal public metadata subset used by NemoClaw's OpenShell gRPC adapter. -message ObjectMeta { - string id = 1; - string name = 2; - int64 created_at_ms = 3; - map labels = 4; - uint64 resource_version = 5; -} diff --git a/src/lib/adapters/openshell/proto/openshell.proto b/src/lib/adapters/openshell/proto/openshell.proto deleted file mode 100644 index 7519ef7ab8..0000000000 --- a/src/lib/adapters/openshell/proto/openshell.proto +++ /dev/null @@ -1,214 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; - -package openshell.v1; - -import "datamodel.proto"; - -service OpenShell { - rpc Health(HealthRequest) returns (HealthResponse); - rpc GetSandbox(GetSandboxRequest) returns (SandboxResponse); - rpc ExecSandbox(ExecSandboxRequest) returns (stream ExecSandboxEvent); - rpc ExecSandboxInteractive(stream ExecSandboxInput) returns (stream ExecSandboxEvent); - rpc GetSandboxLogs(GetSandboxLogsRequest) returns (GetSandboxLogsResponse); - rpc WatchSandbox(WatchSandboxRequest) returns (stream SandboxStreamEvent); - rpc CreateSshSession(CreateSshSessionRequest) returns (CreateSshSessionResponse); - rpc ForwardTcp(stream TcpForwardFrame) returns (stream TcpForwardFrame); -} - -message HealthRequest {} - -message HealthResponse { - ServiceStatus status = 1; - string version = 2; -} - -enum ServiceStatus { - SERVICE_STATUS_UNSPECIFIED = 0; - SERVICE_STATUS_HEALTHY = 1; - SERVICE_STATUS_DEGRADED = 2; - SERVICE_STATUS_UNHEALTHY = 3; -} - -message GetSandboxRequest { - string name = 1; -} - -message SandboxResponse { - Sandbox sandbox = 1; -} - -message Sandbox { - openshell.datamodel.v1.ObjectMeta metadata = 1; - SandboxStatus status = 3; - SandboxPhase phase = 4; - uint32 current_policy_version = 5; -} - -message SandboxStatus { - string sandbox_name = 1; - string agent_pod = 2; - string agent_fd = 3; - string sandbox_fd = 4; - repeated SandboxCondition conditions = 5; -} - -message SandboxCondition { - string type = 1; - string status = 2; - string reason = 3; - string message = 4; - string last_transition_time = 5; -} - -enum SandboxPhase { - SANDBOX_PHASE_UNSPECIFIED = 0; - SANDBOX_PHASE_PROVISIONING = 1; - SANDBOX_PHASE_READY = 2; - SANDBOX_PHASE_ERROR = 3; - SANDBOX_PHASE_DELETING = 4; - SANDBOX_PHASE_UNKNOWN = 5; -} - -message ExecSandboxRequest { - string sandbox_id = 1; - repeated string command = 2; - string workdir = 3; - map environment = 4; - uint32 timeout_seconds = 5; - bytes stdin = 6; - bool tty = 7; - uint32 cols = 8; - uint32 rows = 9; -} - -message ExecSandboxStdout { - bytes data = 1; -} - -message ExecSandboxStderr { - bytes data = 1; -} - -message ExecSandboxExit { - int32 exit_code = 1; -} - -message ExecSandboxEvent { - oneof payload { - ExecSandboxStdout stdout = 1; - ExecSandboxStderr stderr = 2; - ExecSandboxExit exit = 3; - } -} - -message ExecSandboxInput { - oneof payload { - ExecSandboxRequest start = 1; - bytes stdin = 2; - ExecSandboxWindowResize resize = 3; - } -} - -message ExecSandboxWindowResize { - uint32 cols = 1; - uint32 rows = 2; -} - -message GetSandboxLogsRequest { - string sandbox_id = 1; - uint32 lines = 2; - int64 since_ms = 3; - repeated string sources = 4; - string min_level = 5; -} - -message GetSandboxLogsResponse { - repeated SandboxLogLine logs = 1; - uint32 buffer_total = 2; -} - -message WatchSandboxRequest { - string id = 1; - bool follow_status = 2; - bool follow_logs = 3; - bool follow_events = 4; - uint32 log_tail_lines = 5; - uint32 event_tail = 6; - bool stop_on_terminal = 7; - int64 log_since_ms = 8; - repeated string log_sources = 9; - string log_min_level = 10; -} - -message SandboxStreamEvent { - oneof payload { - Sandbox sandbox = 1; - SandboxLogLine log = 2; - PlatformEvent event = 3; - SandboxStreamWarning warning = 4; - } -} - -message SandboxLogLine { - string sandbox_id = 1; - int64 timestamp_ms = 2; - string level = 3; - string target = 4; - string message = 5; - string source = 6; - map fields = 7; -} - -message PlatformEvent { - int64 timestamp_ms = 1; - string source = 2; - string type = 3; - string reason = 4; - string message = 5; - map metadata = 6; -} - -message SandboxStreamWarning { - string message = 1; -} - -message CreateSshSessionRequest { - string sandbox_id = 1; -} - -message CreateSshSessionResponse { - string sandbox_id = 1; - string token = 2; - string gateway_host = 3; - uint32 gateway_port = 4; - string gateway_scheme = 5; - string host_key_fingerprint = 7; - int64 expires_at_ms = 8; -} - -message TcpForwardInit { - string sandbox_id = 1; - string service_id = 4; - oneof target { - SshRelayTarget ssh = 5; - TcpRelayTarget tcp = 6; - } - string authorization_token = 7; -} - -message SshRelayTarget {} - -message TcpRelayTarget { - string host = 1; - uint32 port = 2; -} - -message TcpForwardFrame { - oneof payload { - TcpForwardInit init = 1; - bytes data = 2; - } -} diff --git a/src/lib/adapters/openshell/sync-runner.ts b/src/lib/adapters/openshell/sync-runner.ts index b58cb37c7f..a5c252d153 100644 --- a/src/lib/adapters/openshell/sync-runner.ts +++ b/src/lib/adapters/openshell/sync-runner.ts @@ -1,8 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; import { createSandboxGrpcClient, type SandboxExecOptions } from "./grpc"; @@ -14,17 +15,34 @@ type SyncRequest = { opts?: SandboxExecOptions; }; +type FakeExecResponse = { + status: number; + stdoutBase64?: string; + stderrBase64?: string; + stdout?: string; + stderr?: string; +}; + function readRequest(): SyncRequest { const raw = fs.readFileSync(0, "utf-8"); return JSON.parse(raw) as SyncRequest; } +function fakeSdkExecCommand(): { command: string; args: string[] } { + const helper = process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN; + if (!helper) { + throw new Error("NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN is required when NEMOCLAW_SDK_TEST_TRANSPORT=1"); + } + if (/\.(?:c|m)?js$/i.test(helper)) return { command: process.execPath, args: [helper] }; + return { command: helper, args: [] }; +} + function shellQuoteArg(value: string): string { if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(value)) return value; return `'${value.replace(/'/g, `'\\''`)}'`; } -function legacyFakeRemoteCommand(argv: readonly string[], input: Buffer = Buffer.alloc(0)): string { +function legacyRemoteCommand(argv: readonly string[], input: Buffer): string { if ( input.length > 0 && (argv[0] === "sh" || argv[0] === "bash") && @@ -38,110 +56,128 @@ function legacyFakeRemoteCommand(argv: readonly string[], input: Buffer = Buffer return argv.map(shellQuoteArg).join(" "); } -function runLegacyFakeSshTransport( +function runLegacyRemoteFakeExec( + command: string, + args: string[], request: SyncRequest, - input: Buffer = Buffer.alloc(0), -): { status: number; stdout: Buffer; stderr: Buffer } { - if ( - process.env.NEMOCLAW_GRPC_TEST_TRANSPORT !== "1" || - process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH !== "1" - ) { - throw new Error("legacy fake transport is not enabled"); - } - - const sshBin = process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN || "ssh"; - const remoteCommand = legacyFakeRemoteCommand(request.argv, input); - const helperArgs = [`openshell-${request.sandboxName}`, remoteCommand]; - const helperCommand = /\.(?:c|m)?js$/i.test(sshBin) ? process.execPath : sshBin; - const helperArgv = helperCommand === process.execPath ? [sshBin, ...helperArgs] : helperArgs; - const result = spawnSync(helperCommand, helperArgv, { + input: Buffer, +): FakeExecResponse { + const result = spawnSync(command, [...args, `openshell-${request.sandboxName}`, legacyRemoteCommand(request.argv, input)], { input, + stdio: ["pipe", "pipe", "pipe"], timeout: request.opts?.timeoutMs && request.opts.timeoutMs > 0 ? request.opts.timeoutMs + 5_000 : undefined, - stdio: ["pipe", "pipe", "pipe"], maxBuffer: 256 * 1024 * 1024, env: process.env, }); if (result.error) throw result.error; return { status: result.status ?? 1, - stdout: Buffer.isBuffer(result.stdout) ? result.stdout : Buffer.from(result.stdout || ""), - stderr: Buffer.isBuffer(result.stderr) ? result.stderr : Buffer.from(result.stderr || ""), + stdoutBase64: Buffer.from(result.stdout || "").toString("base64"), + stderrBase64: Buffer.from(result.stderr || "").toString("base64"), }; } +function runFakeSdkExec(request: SyncRequest, input: Buffer): FakeExecResponse { + const { command, args } = fakeSdkExecCommand(); + if (path.basename(args[0] ?? command) === "ssh") { + return runLegacyRemoteFakeExec(command, args, request, input); + } + const result = spawnSync(command, args, { + input: JSON.stringify({ + sandboxName: request.sandboxName, + argv: request.argv, + inputBase64: input.toString("base64"), + opts: request.opts ?? {}, + }), + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + timeout: + request.opts?.timeoutMs && request.opts.timeoutMs > 0 + ? request.opts.timeoutMs + 5_000 + : undefined, + maxBuffer: 256 * 1024 * 1024, + env: process.env, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error((result.stderr || result.stdout || `fake SDK exec exited ${result.status}`).trim()); + } + return JSON.parse(result.stdout) as FakeExecResponse; +} + +function writeTextResult(result: FakeExecResponse): void { + const stdout = + result.stdout ?? + Buffer.from(result.stdoutBase64 || "", "base64").toString("utf-8"); + const stderr = + result.stderr ?? + Buffer.from(result.stderrBase64 || "", "base64").toString("utf-8"); + process.stdout.write( + JSON.stringify({ + ok: true, + result: { status: result.status, stdout, stderr }, + }), + ); +} + +function writeBinaryResult(result: FakeExecResponse): void { + process.stdout.write( + JSON.stringify({ + ok: true, + result: { + status: result.status, + stdoutBase64: + result.stdoutBase64 ?? Buffer.from(result.stdout || "", "utf-8").toString("base64"), + stderrBase64: + result.stderrBase64 ?? Buffer.from(result.stderr || "", "utf-8").toString("base64"), + }, + }), + ); +} + async function main(): Promise { const request = readRequest(); - if (process.env.NEMOCLAW_GRPC_TEST_TRANSPORT === "1") { - const input = - request.op === "execInput" - ? Buffer.from(request.inputBase64 || "", "base64") - : Buffer.alloc(0); - const result = runLegacyFakeSshTransport(request, input); - if (request.op === "execText") { - process.stdout.write( - JSON.stringify({ - ok: true, - result: { - status: result.status, - stdout: result.stdout.toString("utf-8"), - stderr: result.stderr.toString("utf-8"), - }, - }), - ); - return; - } - process.stdout.write( - JSON.stringify({ - ok: true, - result: { - status: result.status, - stdoutBase64: result.stdout.toString("base64"), - stderrBase64: result.stderr.toString("base64"), - }, - }), - ); + const input = + request.op === "execInput" + ? Buffer.from(request.inputBase64 || "", "base64") + : Buffer.alloc(0); + + if (process.env.NEMOCLAW_SDK_TEST_TRANSPORT === "1") { + const result = runFakeSdkExec(request, input); + if (request.op === "execText") writeTextResult(result); + else writeBinaryResult(result); return; } const client = createSandboxGrpcClient(); - try { - if (request.op === "execText") { - const result = await client.execText(request.sandboxName, request.argv, request.opts ?? {}); - process.stdout.write(JSON.stringify({ ok: true, result })); - return; - } - if (request.op === "execBinary") { - const result = await client.execBinaryStream(request.sandboxName, request.argv, request.opts ?? {}); - process.stdout.write( - JSON.stringify({ - ok: true, - result: { - status: result.status, - stdoutBase64: result.stdout.toString("base64"), - stderrBase64: result.stderr.toString("base64"), - }, - }), - ); - return; - } - const input = Buffer.from(request.inputBase64 || "", "base64"); - const result = await client.execInputStream(request.sandboxName, request.argv, input, request.opts ?? {}); - process.stdout.write( - JSON.stringify({ - ok: true, - result: { - status: result.status, - stdoutBase64: result.stdout.toString("base64"), - stderrBase64: result.stderr.toString("base64"), - }, - }), - ); - } finally { - client.close(); + if (request.op === "execText") { + const result = await client.execText(request.sandboxName, request.argv, request.opts ?? {}); + process.stdout.write(JSON.stringify({ ok: true, result })); + return; } + if (request.op === "execBinary") { + const result = await client.execBinary(request.sandboxName, request.argv, request.opts ?? {}); + writeBinaryResult({ + status: result.status, + stdoutBase64: result.stdout.toString("base64"), + stderrBase64: result.stderr.toString("base64"), + }); + return; + } + const result = await client.execInputStream( + request.sandboxName, + request.argv, + input, + request.opts ?? {}, + ); + writeBinaryResult({ + status: result.status, + stdoutBase64: result.stdout.toString("base64"), + stderrBase64: result.stderr.toString("base64"), + }); } main().catch((error) => { diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 29841b1a41..faee4bf000 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -329,7 +329,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ? `Failed to start dashboard forward on port ${actualPort} — the host port ` + `is held by another process. Free it and run \`${deps.cliName()} onboard\` again, ` + `or pass \`--control-ui-port \` to pick a different dashboard port.` - : `Failed to start dashboard gRPC forward on port ${actualPort}: ${fwdDiagnostic.slice(0, 240)}`, + : `Failed to start dashboard SDK forward on port ${actualPort}: ${fwdDiagnostic.slice(0, 240)}`, ); rollbackSandboxAndExit(sandboxName, err); } @@ -342,7 +342,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ); console.warn(` Free the port, then reconnect: ${deps.cliName()} ${sandboxName} connect`); } else { - console.warn(`! Port ${actualPort} gRPC forward did not start: ${fwdDiagnostic.slice(0, 240)}`); + console.warn(`! Port ${actualPort} SDK forward did not start: ${fwdDiagnostic.slice(0, 240)}`); console.warn(` Reconnect after resolving the issue: ${deps.cliName()} ${sandboxName} connect`); } } diff --git a/src/lib/sandbox/version.ts b/src/lib/sandbox/version.ts index b36e9cc712..da0ccfdd13 100644 --- a/src/lib/sandbox/version.ts +++ b/src/lib/sandbox/version.ts @@ -32,7 +32,7 @@ function resolveAgentForSandbox(sandboxName: string): ReturnType { ShareCommandError, ); await expect(runShareMount({ sandboxName: "alpha" }, deps)).rejects.toThrow( - /OpenShell gRPC/, + /OpenShell SDK/, ); expect(deps.ensureLive).toHaveBeenCalledWith("alpha"); }); diff --git a/src/lib/share-command.ts b/src/lib/share-command.ts index a87b073f8b..616ff2c789 100644 --- a/src/lib/share-command.ts +++ b/src/lib/share-command.ts @@ -175,7 +175,7 @@ export async function runShareMount( await deps.ensureLive(sandboxName); shareFail([ " Live sandbox filesystem mounts are no longer supported.", - " NemoClaw now uses OpenShell gRPC for sandbox lifecycle operations, and OpenShell does not provide a live filesystem mount API on that transport.", + " NemoClaw now uses OpenShell SDK for sandbox lifecycle operations, and OpenShell does not provide a live filesystem mount API on that transport.", ` Existing legacy mounts can still be inspected or removed with '${deps.cliName} ${sandboxName} share status' and '${deps.cliName} ${sandboxName} share unmount'.`, ]); } diff --git a/src/lib/skill-install.ts b/src/lib/skill-install.ts index ef504928dc..8651416ef0 100644 --- a/src/lib/skill-install.ts +++ b/src/lib/skill-install.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // // Skill install logic for `nemoclaw skill install `. -// Validates a local SKILL.md, uploads it to the sandbox via OpenShell gRPC, and +// Validates a local SKILL.md, uploads it to the sandbox via OpenShell SDK, and // performs agent-specific post-install steps (session refresh for // OpenClaw). Non-OpenClaw agents get a "restart gateway" hint until a // generic refresh contract is defined in the manifest schema. diff --git a/src/lib/status-command-deps.test.ts b/src/lib/status-command-deps.test.ts index 34aa80751c..67c392cffd 100644 --- a/src/lib/status-command-deps.test.ts +++ b/src/lib/status-command-deps.test.ts @@ -10,7 +10,7 @@ import path from "node:path"; const require = createRequire(import.meta.url); const { buildStatusCommandDeps } = require("../../dist/lib/status-command-deps.js") as typeof import("../../dist/lib/status-command-deps"); -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "..", "..", "test", "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "..", "..", "test", "helpers", "sdk-fake-exec.cjs"); function writeExecutable(target: string, body: string): void { fs.writeFileSync(target, body, { mode: 0o755 }); @@ -20,9 +20,8 @@ describe("buildStatusCommandDeps", () => { let previousOverride: string | undefined; let previousHome: string | undefined; let previousPath: string | undefined; - let previousGrpcTransport: string | undefined; - let previousGrpcLegacy: string | undefined; - let previousGrpcFakeSsh: string | undefined; + let previousSdkTransport: string | undefined; + let previousSdkFakeExec: string | undefined; let tmp: string; let callsFile: string; let openshell: string; @@ -31,18 +30,16 @@ describe("buildStatusCommandDeps", () => { previousOverride = process.env.NEMOCLAW_OPENSHELL_BIN; previousHome = process.env.HOME; previousPath = process.env.PATH; - previousGrpcTransport = process.env.NEMOCLAW_GRPC_TEST_TRANSPORT; - previousGrpcLegacy = process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH; - previousGrpcFakeSsh = process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN; + previousSdkTransport = process.env.NEMOCLAW_SDK_TEST_TRANSPORT; + previousSdkFakeExec = process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN; tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-status-deps-")); callsFile = path.join(tmp, "openshell.calls"); openshell = path.join(tmp, "openshell"); process.env.NEMOCLAW_OPENSHELL_BIN = openshell; process.env.HOME = tmp; process.env.PATH = `${tmp}${path.delimiter}${previousPath || ""}`; - process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; - process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; - process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = GRPC_FAKE_SSH; + process.env.NEMOCLAW_SDK_TEST_TRANSPORT = "1"; + process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN = SDK_FAKE_EXEC; }); afterEach(() => { @@ -61,20 +58,15 @@ describe("buildStatusCommandDeps", () => { } else { process.env.PATH = previousPath; } - if (previousGrpcTransport === undefined) { - delete process.env.NEMOCLAW_GRPC_TEST_TRANSPORT; + if (previousSdkTransport === undefined) { + delete process.env.NEMOCLAW_SDK_TEST_TRANSPORT; } else { - process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = previousGrpcTransport; + process.env.NEMOCLAW_SDK_TEST_TRANSPORT = previousSdkTransport; } - if (previousGrpcLegacy === undefined) { - delete process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH; + if (previousSdkFakeExec === undefined) { + delete process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN; } else { - process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = previousGrpcLegacy; - } - if (previousGrpcFakeSsh === undefined) { - delete process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN; - } else { - process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = previousGrpcFakeSsh; + process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN = previousSdkFakeExec; } fs.rmSync(tmp, { recursive: true, force: true }); }); diff --git a/src/lib/tunnel/services-sandbox.test.ts b/src/lib/tunnel/services-sandbox.test.ts index ecde9dc534..c3a61d88f4 100644 --- a/src/lib/tunnel/services-sandbox.test.ts +++ b/src/lib/tunnel/services-sandbox.test.ts @@ -135,7 +135,7 @@ describe("stopSandboxChannels", () => { logSpy.mockRestore(); }); - it("falls back to OpenShell gRPC exec when the gateway container is unavailable", () => { + it("falls back to OpenShell SDK exec when the gateway container is unavailable", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); spawnSyncSpy .mockReturnValueOnce({ status: 1, stdout: "" }) @@ -203,7 +203,7 @@ describe("stopSandboxChannels", () => { expect(spawnSyncSpy).toHaveBeenCalledTimes(2); const output = logSpy.mock.calls.map((c) => c[0]).join("\n"); - expect(output).toContain("OpenShell gRPC exec could not stop"); + expect(output).toContain("OpenShell SDK exec could not stop"); expect(output).toContain("sync runner missing"); logSpy.mockRestore(); }); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 53d2a278d4..66e66ac9ed 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -471,7 +471,7 @@ export function stopSandboxChannels(sandboxName: string): void { } as StopAttemptResult); } catch (error) { warn( - `OpenShell gRPC exec could not stop in-sandbox messaging channels: ${ + `OpenShell SDK exec could not stop in-sandbox messaging channels: ${ error instanceof Error ? error.message : String(error) }`, ); diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index 64f42f56ae..159f5f984d 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -290,7 +290,7 @@ export async function verifyDeployment( detail: dashboard.detail, hint: dashboard.reachable ? "" - : `Dashboard gRPC forward on ${chain.port} is not working. Retry: nemoclaw ${sandboxName} connect --probe-only`, + : `Dashboard SDK forward on ${chain.port} is not working. Retry: nemoclaw ${sandboxName} connect --probe-only`, }); // 4. Inference route diff --git a/test/cli.test.ts b/test/cli.test.ts index 4985e03e28..2395cbe741 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -12,7 +12,7 @@ import { execTimeout, testTimeout, testTimeoutOptions } from "./helpers/timeouts const CLI = path.join(import.meta.dirname, "..", "bin", "nemoclaw.js"); const HERMES_CLI = path.join(import.meta.dirname, "..", "bin", "nemohermes.js"); -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); const PARSER_EXIT_CODE = 2; type CliRunResult = { @@ -94,9 +94,8 @@ function runWithEnv( HOME: "/tmp/nemoclaw-cli-test-" + Date.now(), NEMOCLAW_HEALTH_POLL_COUNT: "1", NEMOCLAW_HEALTH_POLL_INTERVAL: "0", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: SDK_FAKE_EXEC, ...env, }, }); @@ -3725,7 +3724,7 @@ describe("CLI dispatch", () => { expect(fs.existsSync(sshCalls)).toBe(false); }); - it("recovers non-OpenClaw agents over gRPC sandbox exec when the recovery marker is present", () => { + it("recovers non-OpenClaw agents over SDK sandbox exec when the recovery marker is present", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-agent-")); const localBin = path.join(home, "bin"); const openshellCalls = path.join(home, "openshell-calls"); diff --git a/test/destroy-cleanup-sandbox-services.test.ts b/test/destroy-cleanup-sandbox-services.test.ts index b4eda98a20..a8fe44c96b 100644 --- a/test/destroy-cleanup-sandbox-services.test.ts +++ b/test/destroy-cleanup-sandbox-services.test.ts @@ -118,7 +118,7 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { }); describe("removeSandboxRegistryEntry forward cleanup", () => { - it("stops NemoClaw-managed gRPC forward bridges for the removed sandbox", () => { + it("stops NemoClaw-managed SDK forward bridges for the removed sandbox", () => { const stopForwardBridge = vi.fn(); const removeSandbox = vi.fn(() => true); diff --git a/test/helpers/grpc-fake-ssh.cjs b/test/helpers/grpc-fake-ssh.cjs deleted file mode 100755 index 8cf03c5167..0000000000 --- a/test/helpers/grpc-fake-ssh.cjs +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env node -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -const { spawnSync } = require("node:child_process"); -const fs = require("node:fs"); -const path = require("node:path"); - -function findFakeOpenshell() { - const homeEntries = process.env.HOME - ? [path.join(process.env.HOME, "bin"), path.join(process.env.HOME, ".local", "bin")] - : []; - const pathEntries = [ - ...homeEntries, - ...(process.env.PATH || "").split(path.delimiter).filter(Boolean), - ]; - for (const dir of pathEntries) { - const candidate = path.join(dir, "openshell"); - if (fs.existsSync(candidate)) { - return candidate; - } - } - return null; -} - -const host = process.argv[2] || ""; -const remoteCommand = process.argv.slice(3).join(" "); -const match = /^openshell-(.+)$/.exec(host); - -if (!match || !remoteCommand) { - process.stderr.write("usage: grpc-fake-ssh openshell- \n"); - process.exit(64); -} - -const openshell = findFakeOpenshell(); -const home = process.env.HOME ? path.resolve(process.env.HOME) : ""; -if (!openshell || !home || !path.resolve(openshell).startsWith(`${home}${path.sep}`)) { - process.stderr.write("grpc fake transport could not find the hermetic fake openshell under HOME\n"); - process.exit(127); -} - -const result = spawnSync( - openshell, - ["sandbox", "exec", "--name", match[1], "--", "sh", "-c", remoteCommand], - { - input: fs.readFileSync(0), - stdio: ["pipe", "pipe", "pipe"], - env: process.env, - maxBuffer: 256 * 1024 * 1024, - }, -); - -if (result.error) { - process.stderr.write(`${result.error.message}\n`); - process.exit(1); -} - -if (result.stdout && result.stdout.length > 0) { - fs.writeSync(1, result.stdout); -} -if (result.stderr && result.stderr.length > 0) { - fs.writeSync(2, result.stderr); -} -process.exit(result.status ?? 1); diff --git a/test/helpers/sdk-fake-exec.cjs b/test/helpers/sdk-fake-exec.cjs new file mode 100644 index 0000000000..ac2c8dd6a6 --- /dev/null +++ b/test/helpers/sdk-fake-exec.cjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const path = require("node:path"); + +function findFakeOpenshell() { + const homeEntries = process.env.HOME + ? [path.join(process.env.HOME, "bin"), path.join(process.env.HOME, ".local", "bin")] + : []; + const pathEntries = [ + ...homeEntries, + ...(process.env.PATH || "").split(path.delimiter).filter(Boolean), + ]; + for (const dir of pathEntries) { + const candidate = path.join(dir, "openshell"); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return null; +} + +function fail(message, code = 1) { + process.stderr.write(`${message}\n`); + process.exit(code); +} + +let request; +try { + request = JSON.parse(fs.readFileSync(0, "utf-8")); +} catch (error) { + fail(`sdk fake exec could not parse request: ${error.message}`, 64); +} + +const sandboxName = request && typeof request.sandboxName === "string" ? request.sandboxName : ""; +const argv = Array.isArray(request?.argv) ? request.argv.filter((entry) => typeof entry === "string") : []; +const input = Buffer.from( + typeof request?.inputBase64 === "string" ? request.inputBase64 : "", + "base64", +); + +if (!sandboxName || argv.length === 0) { + fail("usage: sdk-fake-exec requires { sandboxName, argv } JSON on stdin", 64); +} + +const openshell = findFakeOpenshell(); +const home = process.env.HOME ? path.resolve(process.env.HOME) : ""; +if (!openshell || !home || !path.resolve(openshell).startsWith(`${home}${path.sep}`)) { + fail("sdk fake transport could not find the hermetic fake openshell under HOME", 127); +} + +function normalizeExecArgvForFake(argv, input) { + if ( + input.length > 0 && + (argv[0] === "sh" || argv[0] === "bash") && + (argv[1] === "-s" || argv[1] === "-") + ) { + return [argv[0], "-c", input.toString("utf-8")]; + } + return argv; +} + +const result = spawnSync( + openshell, + ["sandbox", "exec", "--name", sandboxName, "--", ...normalizeExecArgvForFake(argv, input)], + { + input, + stdio: ["pipe", "pipe", "pipe"], + env: process.env, + maxBuffer: 256 * 1024 * 1024, + }, +); + +if (result.error) fail(result.error.message); + +process.stdout.write( + JSON.stringify({ + status: result.status ?? 1, + stdoutBase64: Buffer.from(result.stdout || "").toString("base64"), + stderrBase64: Buffer.from(result.stderr || "").toString("base64"), + }), +); diff --git a/test/nemoclaw-cli-recovery.test.ts b/test/nemoclaw-cli-recovery.test.ts index eb498e7f2d..57758c3032 100644 --- a/test/nemoclaw-cli-recovery.test.ts +++ b/test/nemoclaw-cli-recovery.test.ts @@ -15,7 +15,7 @@ describe("nemoclaw CLI runtime recovery", () => { testTimeoutOptions(), () => { const repoRoot = path.join(import.meta.dirname, ".."); - const grpcFakeSsh = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); + const sdkFakeExec = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-recovery-")); const homeLocalBin = path.join(tmpDir, ".local", "bin"); const stateDir = path.join(tmpDir, "state"); @@ -107,9 +107,8 @@ process.exit(0); env: { ...process.env, HOME: tmpDir, - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: grpcFakeSsh, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: sdkFakeExec, PATH: "/usr/bin:/bin", }, }, diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 834a840265..7fe2073652 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -10,7 +10,7 @@ import { describe, expect, it } from "vitest"; import { stageOptimizedSandboxBuildContext } from "../dist/lib/sandbox/build-context.js"; import { testTimeoutOptions } from "./helpers/timeouts"; -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); type ShimScalar = string | number | boolean | null | undefined; type ShimCallable = (...args: readonly string[]) => ShimValue; @@ -63,9 +63,8 @@ function stripMessagingEnv(source: NodeJS.ProcessEnv): Record { return { - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: SDK_FAKE_EXEC, }; } diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index ef2c0d77af..79e5813782 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -38,7 +38,7 @@ afterEach(() => { * optionally a saved credential in credentials.json. * * The fake openshell binary responds to sandbox list, ssh-config, and - * delete commands. The fake ssh supports backup tar operations. + * delete commands. The fake SDK exec supports backup tar operations. */ function createFixture(opts: { sandboxName?: string; @@ -185,7 +185,7 @@ function createFixture(opts: { } // ── Fake workspace dir for the backup tar call ──────────────── - const fakeRoot = path.join(tmpDir, "fake-sandbox-root"); + const fakeRoot = path.join(tmpDir, "sandbox-root"); const workspaceDir = path.join(fakeRoot, "workspace"); fs.mkdirSync(workspaceDir, { recursive: true }); fs.writeFileSync(path.join(workspaceDir, "marker.txt"), "test-workspace"); @@ -237,7 +237,7 @@ process.exit(1); { mode: 0o755 }, ); - // ── Fake ssh ────────────────────────────────────────────────── + // ── Fake SDK exec ────────────────────────────────────────────────── fs.writeFileSync( path.join(tmpDir, "ssh"), `#!/usr/bin/env node @@ -261,7 +261,7 @@ process.exit(0); { mode: 0o755 }, ); - // Patch the PLACEHOLDER in the fake ssh to point at the real fakeRoot + // Patch the PLACEHOLDER in the fake SDK exec to point at the real fakeRoot const sshScript = fs.readFileSync(path.join(tmpDir, "ssh"), "utf-8"); fs.writeFileSync( path.join(tmpDir, "ssh"), @@ -290,9 +290,8 @@ function runRebuild( env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: path.join(fixture.tmpDir, "ssh"), + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: path.join(fixture.tmpDir, "ssh"), NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", diff --git a/test/rebuild-shields-auto-unlock.test.ts b/test/rebuild-shields-auto-unlock.test.ts index 656dc05988..04c8b428bc 100644 --- a/test/rebuild-shields-auto-unlock.test.ts +++ b/test/rebuild-shields-auto-unlock.test.ts @@ -143,7 +143,7 @@ function createFixture(opts: { shieldsLocked: boolean }) { ); // Workspace dir for the backup tar - const fakeRoot = path.join(tmpDir, "fake-sandbox-root"); + const fakeRoot = path.join(tmpDir, "sandbox-root"); fs.mkdirSync(path.join(fakeRoot, "workspace"), { recursive: true }); fs.writeFileSync(path.join(fakeRoot, "workspace", "marker.txt"), "test"); const lockStatePath = path.join(tmpDir, "config-lock-state.txt"); @@ -260,7 +260,7 @@ process.exit(0); { mode: 0o755 }, ); - // Fake ssh — backup tars from the real fakeRoot + // Fake SDK exec — backup tars from the real fakeRoot fs.writeFileSync( path.join(tmpDir, "ssh"), `#!/usr/bin/env node @@ -297,9 +297,8 @@ function runRebuild(fixture: ReturnType) { env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: path.join(fixture.tmpDir, "ssh"), + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: path.join(fixture.tmpDir, "ssh"), NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", diff --git a/test/recover-port-forward.test.ts b/test/recover-port-forward.test.ts index 82eefc002c..2bc905a6e2 100644 --- a/test/recover-port-forward.test.ts +++ b/test/recover-port-forward.test.ts @@ -10,7 +10,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { execTimeout, testTimeoutOptions } from "./helpers/timeouts"; const tmpFixtures: string[] = []; -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); // Each fixture grabs a unique high port. Sharing port 18789 across tests // collides with real nemoclaw installs on the developer's machine: the @@ -170,9 +170,8 @@ function runRecover(fixture: Fixture) { HOME: fixture.tmpDir, PATH: "/usr/bin:/bin", NEMOCLAW_NO_CONNECT_HINT: "1", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: SDK_FAKE_EXEC, }, timeout: execTimeout(15_000), }, diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index ed56abc6c7..e2cf2b5124 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -154,8 +154,8 @@ function createFixture({ const sandboxName = rebuildTarget.name; - // ── Dummy workspace dir for the fake ssh tar call ───────────── - const workspaceDir = path.join(tmpDir, "fake-sandbox-root", "workspace"); + // ── Dummy workspace dir for the fake SDK exec tar call ───────────── + const workspaceDir = path.join(tmpDir, "sandbox-root", "workspace"); fs.mkdirSync(workspaceDir, { recursive: true }); fs.writeFileSync(path.join(workspaceDir, ".keep"), ""); @@ -196,11 +196,11 @@ process.exit(0); { mode: 0o755 }, ); - // ── Fake ssh ────────────────────────────────────────────────── + // ── Fake SDK exec ────────────────────────────────────────────────── // backupSandboxState makes two ssh calls: // 1. dir-existence check (command has "[ -d") → print "workspace" // 2. tar download (command has "tar") → produce a real tar archive - const fakeRoot = path.join(tmpDir, "fake-sandbox-root"); + const fakeRoot = path.join(tmpDir, "sandbox-root"); fs.writeFileSync( path.join(tmpDir, "ssh"), `#!/usr/bin/env node @@ -240,9 +240,8 @@ function runRebuild(fixture: ReturnType) { PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: path.join(fixture.tmpDir, "ssh"), + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: path.join(fixture.tmpDir, "ssh"), NO_COLOR: "1", }, timeout: 30_000, diff --git a/test/sandbox-connect-inference.test.ts b/test/sandbox-connect-inference.test.ts index 2a2485fb8c..9486e5ae02 100644 --- a/test/sandbox-connect-inference.test.ts +++ b/test/sandbox-connect-inference.test.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { execTimeout, testTimeoutOptions } from "./helpers/timeouts"; -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); /** * Tests for #1248 — inference route swap on sandbox connect. @@ -388,9 +388,8 @@ function runConnect( NEMOCLAW_NO_CONNECT_HINT: "1", NEMOCLAW_OLLAMA_PORT: "11434", NEMOCLAW_OLLAMA_PROXY_PORT: "11435", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: SDK_FAKE_EXEC, ...extraEnv, }, timeout: execTimeout(15_000), diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index 8350e6ec50..aae29f6141 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -9,7 +9,7 @@ import { pathToFileURL } from "url"; import { spawnSync } from "child_process"; import { describe, it, expect } from "vitest"; -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); describe("sandboxName command hardening in onboard.js", () => { it("re-validates sandboxName at the createSandbox boundary", async () => { @@ -82,9 +82,8 @@ for (const key of Object.keys(process.env)) { delete process.env[key]; } } -process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; -process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; -process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = ${JSON.stringify(GRPC_FAKE_SSH)}; +process.env.NEMOCLAW_SDK_TEST_TRANSPORT = "1"; +process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN = ${JSON.stringify(SDK_FAKE_EXEC)}; const commands = []; const asText = (command) => Array.isArray(command) ? command.join(" ") : String(command); runner.run = (command, opts = {}) => { @@ -139,9 +138,8 @@ try { env: { HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: SDK_FAKE_EXEC, }, timeout: 30_000, }); diff --git a/test/snapshot-gateway-guard.test.ts b/test/snapshot-gateway-guard.test.ts index f528509b79..76bf3c6b80 100644 --- a/test/snapshot-gateway-guard.test.ts +++ b/test/snapshot-gateway-guard.test.ts @@ -14,7 +14,7 @@ import path from "node:path"; import { execTimeout } from "./helpers/timeouts"; const CLI = path.join(import.meta.dirname, "..", "bin", "nemoclaw.js"); -const GRPC_FAKE_SSH = path.join(import.meta.dirname, "helpers", "grpc-fake-ssh.cjs"); +const SDK_FAKE_EXEC = path.join(import.meta.dirname, "helpers", "sdk-fake-exec.cjs"); type CliRunResult = { code: number; out: string }; @@ -27,9 +27,8 @@ function runCli(args: string, env: Record = {}): Cli ...process.env, NEMOCLAW_HEALTH_POLL_COUNT: "1", NEMOCLAW_HEALTH_POLL_INTERVAL: "0", - NEMOCLAW_GRPC_TEST_TRANSPORT: "1", - NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH: "1", - NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN: GRPC_FAKE_SSH, + NEMOCLAW_SDK_TEST_TRANSPORT: "1", + NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN: SDK_FAKE_EXEC, ...env, }, }); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index c3f1ccf6d3..cc10a95ac0 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -22,9 +22,8 @@ process.env.HOME = TMP_HOME; const REPO_ROOT = path.join(import.meta.dirname, ".."); const GRPC_TEST_ENV_KEYS = [ - "NEMOCLAW_GRPC_TEST_TRANSPORT", - "NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH", - "NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN", + "NEMOCLAW_SDK_TEST_TRANSPORT", + "NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN", ] as const; type BackupScalar = string | number | boolean | null | undefined; @@ -96,9 +95,8 @@ beforeEach(() => { }); function enableFakeGrpcTransportViaLegacySsh(binDir: string): void { - process.env.NEMOCLAW_GRPC_TEST_TRANSPORT = "1"; - process.env.NEMOCLAW_GRPC_TEST_LEGACY_FAKE_SSH = "1"; - process.env.NEMOCLAW_GRPC_TEST_FAKE_SSH_BIN = path.join(binDir, "ssh"); + process.env.NEMOCLAW_SDK_TEST_TRANSPORT = "1"; + process.env.NEMOCLAW_SDK_TEST_FAKE_EXEC_BIN = path.join(binDir, "ssh"); } function clearFakeGrpcTransport(): void { From 9e7275d7ad01e7fc59a845380e7da2fc27d6e439 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 15:21:57 -0700 Subject: [PATCH 22/26] chore: add temporary OpenShell SDK placeholder --- package-lock.json | 13 ++- package.json | 2 +- vendor/openshell-sdk-placeholder/index.d.ts | 83 +++++++++++++++++++ vendor/openshell-sdk-placeholder/index.js | 37 +++++++++ vendor/openshell-sdk-placeholder/lib.d.ts | 1 + vendor/openshell-sdk-placeholder/lib.mjs | 30 +++++++ vendor/openshell-sdk-placeholder/package.json | 24 ++++++ 7 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 vendor/openshell-sdk-placeholder/index.d.ts create mode 100644 vendor/openshell-sdk-placeholder/index.js create mode 100644 vendor/openshell-sdk-placeholder/lib.d.ts create mode 100644 vendor/openshell-sdk-placeholder/lib.mjs create mode 100644 vendor/openshell-sdk-placeholder/package.json diff --git a/package-lock.json b/package-lock.json index ece8a31b5f..d56a6be423 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", "@oclif/core": "^4.10.5", - "@openshell/sdk": "file:../OpenShell/crates/openshell-sdk-node", + "@openshell/sdk": "file:vendor/openshell-sdk-placeholder", "js-yaml": "^4.1.1", "p-retry": "^4.6.2", "qrcode-terminal": "^0.12.0", @@ -45,6 +45,7 @@ "../OpenShell/crates/openshell-sdk-node": { "name": "@openshell/sdk", "version": "0.0.0-alpha.0", + "extraneous": true, "license": "Apache-2.0", "devDependencies": { "@napi-rs/cli": "^3" @@ -2126,7 +2127,7 @@ } }, "node_modules/@openshell/sdk": { - "resolved": "../OpenShell/crates/openshell-sdk-node", + "resolved": "vendor/openshell-sdk-placeholder", "link": true }, "node_modules/@oxc-project/types": { @@ -6621,6 +6622,14 @@ "peerDependencies": { "zod": "^3.25.28 || ^4" } + }, + "vendor/openshell-sdk-placeholder": { + "name": "@openshell/sdk", + "version": "0.0.0-alpha.0", + "license": "Apache-2.0", + "engines": { + "node": ">= 18" + } } } } diff --git a/package.json b/package.json index eb005bbaf1..95d9275b19 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", "@oclif/core": "^4.10.5", - "@openshell/sdk": "file:../OpenShell/crates/openshell-sdk-node", + "@openshell/sdk": "file:vendor/openshell-sdk-placeholder", "js-yaml": "^4.1.1", "p-retry": "^4.6.2", "qrcode-terminal": "^0.12.0", diff --git a/vendor/openshell-sdk-placeholder/index.d.ts b/vendor/openshell-sdk-placeholder/index.d.ts new file mode 100644 index 0000000000..cd2a4ce3fa --- /dev/null +++ b/vendor/openshell-sdk-placeholder/index.d.ts @@ -0,0 +1,83 @@ +/* auto-generated by NAPI-RS */ +/* eslint-disable */ + +export declare class OidcRefresher { + constructor( + initialToken: string, + initialExpiresAt: number | undefined | null, + callback: () => Promise<{ accessToken: string; expiresAt?: number }>, + ) + currentToken(): string + refresh(): Promise +} + +export declare class OpenShellClient { + static connect(options: ConnectOptions): Promise + health(): Promise + createSandbox(spec: SandboxSpec): Promise + getSandbox(name: string): Promise + listSandboxes(options?: ListOptions | undefined | null): Promise> + deleteSandbox(name: string): Promise + waitReady(name: string, timeoutSecs: number): Promise + waitDeleted(name: string, timeoutSecs: number): Promise + exec( + name: string, + command: Array, + options?: ExecOptions | undefined | null, + ): Promise +} + +export interface ConnectOptions { + gateway: string + caCert?: Buffer + oidcToken?: string + edgeToken?: string + insecureSkipVerify?: boolean +} + +export interface ExecOptions { + workdir?: string + environment?: Record + timeoutSecs?: number + stdin?: Buffer +} + +export interface ExecResult { + exitCode: number + stdout: Buffer + stderr: Buffer +} + +export interface Health { + status: string + version: string +} + +export interface JsRefreshedToken { + accessToken: string + expiresAt?: number +} + +export interface ListOptions { + limit?: number + offset?: number + labelSelector?: string +} + +export interface SandboxRef { + id: string + name: string + phase: string + labels: Record + resourceVersion: string +} + +export interface SandboxSpec { + name?: string + image?: string + labels?: Record + environment?: Record + providers?: Array + gpu?: boolean + gpuDevice?: string +} diff --git a/vendor/openshell-sdk-placeholder/index.js b/vendor/openshell-sdk-placeholder/index.js new file mode 100644 index 0000000000..ab645771ce --- /dev/null +++ b/vendor/openshell-sdk-placeholder/index.js @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const MESSAGE = + "@openshell/sdk is not published yet. This NemoClaw draft vendors only the SDK type surface " + + "so CI can build; replace vendor/openshell-sdk-placeholder with the OpenShell-published " + + "@openshell/sdk package before running SDK-backed sandbox operations."; + +class OpenShellClient { + static async connect() { + throw new Error(MESSAGE); + } +} + +class OidcRefresher { + constructor(initialToken, _initialExpiresAt, callback) { + this.token = initialToken; + this.callback = callback; + } + + currentToken() { + return this.token; + } + + async refresh() { + const refreshed = await this.callback(); + this.token = refreshed.accessToken; + return this.token; + } +} + +module.exports = { + OidcRefresher, + OpenShellClient, +}; diff --git a/vendor/openshell-sdk-placeholder/lib.d.ts b/vendor/openshell-sdk-placeholder/lib.d.ts new file mode 100644 index 0000000000..6d1e0ee83e --- /dev/null +++ b/vendor/openshell-sdk-placeholder/lib.d.ts @@ -0,0 +1 @@ +export * from "./index"; diff --git a/vendor/openshell-sdk-placeholder/lib.mjs b/vendor/openshell-sdk-placeholder/lib.mjs new file mode 100644 index 0000000000..b2c9e59f31 --- /dev/null +++ b/vendor/openshell-sdk-placeholder/lib.mjs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const MESSAGE = + "@openshell/sdk is not published yet. This NemoClaw draft vendors only the SDK type surface " + + "so CI can build; replace vendor/openshell-sdk-placeholder with the OpenShell-published " + + "@openshell/sdk package before running SDK-backed sandbox operations."; + +export class OpenShellClient { + static async connect() { + throw new Error(MESSAGE); + } +} + +export class OidcRefresher { + constructor(initialToken, _initialExpiresAt, callback) { + this.token = initialToken; + this.callback = callback; + } + + currentToken() { + return this.token; + } + + async refresh() { + const refreshed = await this.callback(); + this.token = refreshed.accessToken; + return this.token; + } +} diff --git a/vendor/openshell-sdk-placeholder/package.json b/vendor/openshell-sdk-placeholder/package.json new file mode 100644 index 0000000000..8efa745bfe --- /dev/null +++ b/vendor/openshell-sdk-placeholder/package.json @@ -0,0 +1,24 @@ +{ + "name": "@openshell/sdk", + "version": "0.0.0-alpha.0", + "description": "Temporary NemoClaw draft placeholder for the unpublished OpenShell SDK package.", + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "import": { + "types": "./lib.d.ts", + "default": "./lib.mjs" + }, + "require": { + "types": "./index.d.ts", + "default": "./index.js" + } + }, + "./package.json": "./package.json" + }, + "license": "Apache-2.0", + "engines": { + "node": ">= 18" + } +} From 1f6618924b9acce814395e7ead8720d2591aea79 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 31 May 2026 14:23:46 -0700 Subject: [PATCH 23/26] fix: call OpenShell gRPC endpoints for SDK gaps Use raw OpenShell gRPC for ForwardTcp and log/watch endpoints that are not exposed by the current TypeScript SDK binding. Add a detached dashboard forward runner and raw ExecSandbox fallback so SDK-preview builds can validate runtime flows without the placeholder implementation. Signed-off-by: Aaron Erickson --- package-lock.json | 65 +- package.json | 4 +- scripts/copy-openshell-protos.mjs | 15 + .../adapters/openshell/direct-grpc.test.ts | 162 ++ src/lib/adapters/openshell/direct-grpc.ts | 666 +++++++ .../openshell/forward-bridge-runner.ts | 54 + .../openshell/forward-bridge-state.ts | 67 +- .../openshell/grpc-migration-guard.test.ts | 11 +- src/lib/adapters/openshell/grpc.test.ts | 72 +- src/lib/adapters/openshell/grpc.ts | 98 +- .../adapters/openshell/proto/datamodel.proto | 44 + .../adapters/openshell/proto/openshell.proto | 1773 +++++++++++++++++ .../adapters/openshell/proto/sandbox.proto | 271 +++ 13 files changed, 3259 insertions(+), 43 deletions(-) create mode 100644 scripts/copy-openshell-protos.mjs create mode 100644 src/lib/adapters/openshell/direct-grpc.test.ts create mode 100644 src/lib/adapters/openshell/direct-grpc.ts create mode 100644 src/lib/adapters/openshell/forward-bridge-runner.ts create mode 100644 src/lib/adapters/openshell/proto/datamodel.proto create mode 100644 src/lib/adapters/openshell/proto/openshell.proto create mode 100644 src/lib/adapters/openshell/proto/sandbox.proto diff --git a/package-lock.json b/package-lock.json index d56a6be423..9015698cb3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,8 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", "@oclif/core": "^4.10.5", "@openshell/sdk": "file:vendor/openshell-sdk-placeholder", "js-yaml": "^4.1.1", @@ -1779,6 +1781,37 @@ } } }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@j178/prek": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/@j178/prek/-/prek-0.3.6.tgz", @@ -1849,6 +1882,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@mariozechner/clipboard": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.5.tgz", @@ -2144,35 +2187,30 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/codegen": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dev": true, "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.1", @@ -2183,35 +2221,30 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/inquire": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/utf8": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@rolldown/binding-android-arm64": { @@ -2720,7 +2753,6 @@ "version": "25.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.2.tgz", "integrity": "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -3310,7 +3342,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3769,7 +3800,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4191,7 +4221,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -5142,7 +5171,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "dev": true, "license": "MIT" }, "node_modules/lodash.kebabcase": { @@ -5184,7 +5212,6 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "dev": true, "license": "Apache-2.0" }, "node_modules/lru-cache": { @@ -5694,7 +5721,6 @@ "version": "7.5.8", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.8.tgz", "integrity": "sha512-dvpCIeLPbXZS/Ete7yLaO7RenOdken2NHKykBXbsaGxZT0UTltcarBciw+A78SRQs9iMAAVpsYA+l8b1hTePIA==", - "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -5785,7 +5811,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6237,7 +6262,6 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -6542,7 +6566,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -6567,7 +6590,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -6586,7 +6608,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" diff --git a/package.json b/package.json index 16772e03e9..230fccbaac 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "format:ts": "cd nemoclaw && npm run lint:fix && npm run format", "check:installer-hash": "bash scripts/check-installer-hash.sh", "typecheck": "tsc -p jsconfig.json", - "build:cli": "tsc -p tsconfig.src.json && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", + "build:cli": "tsc -p tsconfig.src.json && node scripts/copy-openshell-protos.mjs && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", "typecheck:cli": "tsc -p tsconfig.cli.json", "validate:configs": "tsx scripts/validate-configs.ts", "type-safety:hotspots": "tsx scripts/type-safety-hotspots.ts", @@ -47,6 +47,8 @@ }, "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.1046.0", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", "@oclif/core": "^4.10.5", "@openshell/sdk": "file:vendor/openshell-sdk-placeholder", "js-yaml": "^4.1.1", diff --git a/scripts/copy-openshell-protos.mjs b/scripts/copy-openshell-protos.mjs new file mode 100644 index 0000000000..3c9c544723 --- /dev/null +++ b/scripts/copy-openshell-protos.mjs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const source = path.join(root, "src", "lib", "adapters", "openshell", "proto"); +const dest = path.join(root, "dist", "lib", "adapters", "openshell", "proto"); + +fs.mkdirSync(dest, { recursive: true }); +for (const entry of fs.readdirSync(source, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(".proto")) continue; + fs.copyFileSync(path.join(source, entry.name), path.join(dest, entry.name)); +} diff --git a/src/lib/adapters/openshell/direct-grpc.test.ts b/src/lib/adapters/openshell/direct-grpc.test.ts new file mode 100644 index 0000000000..8012c4bad6 --- /dev/null +++ b/src/lib/adapters/openshell/direct-grpc.test.ts @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; +import path from "node:path"; + +import * as grpc from "@grpc/grpc-js"; +import * as protoLoader from "@grpc/proto-loader"; +import { describe, expect, it } from "vitest"; + +import { __directGrpcTestHooks, OpenShellDirectGrpcClient } from "./direct-grpc"; + +function unusedLocalPort(): Promise { + const server = net.createServer(); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + server.close((error) => (error ? reject(error) : resolve(port))); + }); + }); +} + +function loadOpenShellService(): grpc.ServiceDefinition { + const protoRoot = __directGrpcTestHooks.protoRoot(); + const definition = protoLoader.loadSync(path.join(protoRoot, "openshell.proto"), { + defaults: false, + enums: String, + includeDirs: [protoRoot], + keepCase: true, + longs: String, + oneofs: true, + bytes: Buffer, + }); + const loaded = grpc.loadPackageDefinition(definition) as any; + return loaded.openshell.v1.OpenShell.service as grpc.ServiceDefinition; +} + +function readOne(socket: net.Socket): Promise { + return new Promise((resolve, reject) => { + socket.once("data", (chunk) => resolve(Buffer.from(chunk))); + socket.once("error", reject); + }); +} + +async function waitFor(predicate: () => boolean, timeoutMs = 2_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error("timed out waiting for condition"); +} + +describe("OpenShell raw gRPC adapter", () => { + it("backs a local TCP forward with OpenShell ForwardTcp frames", async () => { + const service = loadOpenShellService(); + const server = new grpc.Server(); + const frames: any[] = []; + const revoked: string[] = []; + server.addService(service, { + GetSandbox: (_call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) => { + callback(null, { + sandbox: { + metadata: { id: "sandbox-123", name: "alpha" }, + phase: "SANDBOX_PHASE_READY", + }, + }); + }, + CreateSshSession: (_call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) => { + callback(null, { sandbox_id: "sandbox-123", token: "forward-token" }); + }, + RevokeSshSession: (call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) => { + revoked.push(String(call.request.token)); + callback(null, { revoked: true }); + }, + ExecSandbox: (call: grpc.ServerWritableStream) => { + frames.push({ exec: call.request }); + call.write({ stdout: { data: Buffer.from("exec-out") } }); + call.write({ stderr: { data: Buffer.from("exec-err") } }); + call.write({ exit: { exit_code: 7 } }); + call.end(); + }, + ForwardTcp: (call: grpc.ServerDuplexStream) => { + call.on("data", (frame) => { + frames.push(frame); + if (frame.data) call.write({ data: frame.data }); + }); + call.on("end", () => call.end()); + }, + }); + + const gatewayPort = await new Promise((resolve, reject) => { + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => { + if (error) reject(error); + else resolve(port); + }); + }); + const localPort = await unusedLocalPort(); + const client = new OpenShellDirectGrpcClient({ + gatewayEndpoint: `http://127.0.0.1:${gatewayPort}`, + }); + + try { + const exec = await client.execText("alpha", ["sh", "-c", "printf hi"], { + workdir: "/sandbox", + env: { A: "1" }, + timeoutMs: 1_000, + }); + expect(exec).toEqual({ status: 7, stdout: "exec-out", stderr: "exec-err" }); + + const handle = await client.startForward("alpha", { + localHost: "127.0.0.1", + localPort, + targetHost: "127.0.0.1", + targetPort: 8642, + serviceId: "nemoclaw-dashboard-8642", + }); + try { + const socket = net.createConnection({ host: "127.0.0.1", port: localPort }); + await new Promise((resolve, reject) => { + socket.once("connect", resolve); + socket.once("error", reject); + }); + socket.write(Buffer.from("ping")); + expect((await readOne(socket)).toString("utf-8")).toBe("ping"); + const closed = new Promise((resolve) => socket.once("close", () => resolve())); + socket.end(); + await closed; + await waitFor(() => revoked.includes("forward-token")); + } finally { + await handle.close(); + } + + expect(frames[0]).toMatchObject({ + exec: { + sandbox_id: "sandbox-123", + command: ["sh", "-c", "printf hi"], + workdir: "/sandbox", + environment: { A: "1" }, + timeout_seconds: 1, + }, + }); + expect(frames[1]).toMatchObject({ + init: { + sandbox_id: "sandbox-123", + service_id: "nemoclaw-dashboard-8642", + authorization_token: "forward-token", + tcp: { host: "127.0.0.1", port: 8642 }, + }, + }); + expect(frames.some((frame) => Buffer.isBuffer(frame.data) && frame.data.toString("utf-8") === "ping")).toBe( + true, + ); + expect(revoked).toContain("forward-token"); + } finally { + client.close(); + server.forceShutdown(); + } + }); +}); diff --git a/src/lib/adapters/openshell/direct-grpc.ts b/src/lib/adapters/openshell/direct-grpc.ts new file mode 100644 index 0000000000..95f8bbe533 --- /dev/null +++ b/src/lib/adapters/openshell/direct-grpc.ts @@ -0,0 +1,666 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; + +import * as grpc from "@grpc/grpc-js"; +import * as protoLoader from "@grpc/proto-loader"; + +import { + type GatewayMetadataOptions, + type ResolvedGatewayMetadata, + resolveGatewayMetadata, +} from "./gateway-metadata"; + +type GrpcClient = grpc.Client & Record any>; + +const DEFAULT_GRPC_TIMEOUT_MS = 30_000; + +let packageDefinition: grpc.GrpcObject | null = null; + +export interface DirectGrpcClientOptions extends GatewayMetadataOptions { + gateway?: ResolvedGatewayMetadata; +} + +export interface SandboxLogLine { + sandbox_id?: string; + timestamp_ms?: number | string; + level?: string; + target?: string; + message?: string; + source?: string; + fields?: Record; +} + +export interface SandboxLogsResult { + logs: SandboxLogLine[]; + bufferTotal: number; +} + +export type SandboxWatchEvent = + | { type: "sandbox"; sandbox: unknown } + | { type: "log"; log: SandboxLogLine } + | { type: "event"; event: unknown } + | { type: "warning"; message: string }; + +export interface DirectForwardHandle { + sandboxName: string; + localPort: number; + localHost: string; + targetHost: string; + targetPort: number; + close(): Promise; +} + +export interface DirectExecOptions { + workdir?: string; + env?: Record; + timeoutMs?: number; + timeoutSeconds?: number; + stdin?: Buffer | string; + tty?: boolean; + cols?: number; + rows?: number; +} + +export interface DirectStreamResult { + status: number; + stdout: Buffer; + stderr: Buffer; +} + +export interface DirectTextResult { + status: number; + stdout: string; + stderr: string; +} + +function protoRoot(): string { + return path.join(__dirname, "proto"); +} + +function loadOpenShellPackage(): grpc.GrpcObject { + if (packageDefinition) return packageDefinition; + const definition = protoLoader.loadSync(path.join(protoRoot(), "openshell.proto"), { + defaults: false, + enums: String, + includeDirs: [protoRoot()], + keepCase: true, + longs: String, + oneofs: true, + bytes: Buffer, + }); + packageDefinition = grpc.loadPackageDefinition(definition) as grpc.GrpcObject; + return packageDefinition; +} + +function getOpenShellConstructor(): typeof grpc.Client { + const loaded = loadOpenShellPackage() as any; + const ctor = loaded.openshell?.v1?.OpenShell; + if (!ctor) { + throw new Error("Failed to load openshell.v1.OpenShell from vendored proto files."); + } + return ctor; +} + +function readRequiredFile(filePath: string, label: string): Buffer { + try { + return fs.readFileSync(filePath); + } catch (error) { + throw new Error( + `Failed to read OpenShell ${label} at ${filePath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function callCredentials(gateway: ResolvedGatewayMetadata): grpc.CallCredentials | null { + const md = new grpc.Metadata(); + const oidcToken = gateway.connectOptions.oidcToken; + const edgeToken = gateway.connectOptions.edgeToken; + if (oidcToken) { + md.set("authorization", `Bearer ${oidcToken}`); + } else if (edgeToken) { + md.set("cf-access-token", edgeToken); + md.set("cf-access-jwt-assertion", edgeToken); + md.set("cookie", `CF_Authorization=${edgeToken}`); + } + if (md.getMap && Object.keys(md.getMap()).length === 0) return null; + return grpc.credentials.createFromMetadataGenerator((_params, callback) => { + callback(null, md); + }); +} + +function createCredentials(gateway: ResolvedGatewayMetadata): grpc.ChannelCredentials { + if (gateway.authMode === "plaintext" || gateway.endpoint.protocol === "http:") { + return grpc.credentials.createInsecure(); + } + + if (gateway.insecureTls) { + throw new Error( + "OpenShell raw gRPC does not support OPENSHELL_GATEWAY_INSECURE for HTTPS gateways yet. " + + "Use a trusted CA in the gateway metadata, an mTLS gateway, or the upstream SDK transport.", + ); + } + + const rootCerts = gateway.connectOptions.caCert; + let channelCredentials: grpc.ChannelCredentials; + if (gateway.authMode === "mtls") { + if (!gateway.mtlsDir) { + throw new Error( + "OpenShell mTLS gateway metadata did not include a certificate directory. " + + "Run `openshell gateway info` and verify the gateway is registered locally.", + ); + } + const ca = readRequiredFile(path.join(gateway.mtlsDir, "ca.crt"), "mTLS CA"); + const cert = readRequiredFile(path.join(gateway.mtlsDir, "tls.crt"), "mTLS client certificate"); + const key = readRequiredFile(path.join(gateway.mtlsDir, "tls.key"), "mTLS client key"); + channelCredentials = grpc.credentials.createSsl(ca, key, cert); + } else { + channelCredentials = grpc.credentials.createSsl(rootCerts); + } + + const perCallCredentials = callCredentials(gateway); + return perCallCredentials + ? grpc.credentials.combineChannelCredentials(channelCredentials, perCallCredentials) + : channelCredentials; +} + +function method any>(client: GrpcClient, name: string): T { + const lower = name.charAt(0).toLowerCase() + name.slice(1); + const fn = client[lower] || client[name]; + if (typeof fn !== "function") { + throw new Error(`OpenShell gRPC method ${name} is unavailable.`); + } + return fn.bind(client) as T; +} + +function callOptions(timeoutMs?: number): grpc.CallOptions { + const deadlineMs = timeoutMs ?? DEFAULT_GRPC_TIMEOUT_MS; + return deadlineMs > 0 ? { deadline: new Date(Date.now() + deadlineMs) } : {}; +} + +function grpcStatusName(code: number | undefined): string { + if (typeof code !== "number") return "UNKNOWN"; + return grpc.status[code] || String(code); +} + +export function formatDirectGrpcError(error: unknown): string { + if (!error || typeof error !== "object") return String(error); + const maybe = error as Partial; + const status = grpcStatusName(maybe.code); + const details = maybe.details || maybe.message || String(error); + return `${status}: ${details}`; +} + +function bufferFromData(data: unknown): Buffer { + if (Buffer.isBuffer(data)) return data; + if (data instanceof Uint8Array) return Buffer.from(data); + if (typeof data === "string") return Buffer.from(data, "base64"); + return Buffer.alloc(0); +} + +function stdinBuffer(input: Buffer | string | undefined): Buffer | undefined { + if (input === undefined) return undefined; + return Buffer.isBuffer(input) ? input : Buffer.from(input); +} + +function timeoutSeconds(opts: DirectExecOptions): number { + if (opts.timeoutSeconds !== undefined) return opts.timeoutSeconds; + if (opts.timeoutMs === undefined || opts.timeoutMs <= 0) return 0; + return Math.max(1, Math.ceil(opts.timeoutMs / 1000)); +} + +function execRequest(sandboxId: string, argv: string[], opts: DirectExecOptions = {}) { + return { + sandbox_id: sandboxId, + command: argv, + workdir: opts.workdir || "", + environment: opts.env || {}, + timeout_seconds: timeoutSeconds(opts), + stdin: stdinBuffer(opts.stdin), + tty: opts.tty === true, + cols: opts.cols ?? 0, + rows: opts.rows ?? 0, + }; +} + +function makeTimeoutError(label: string, timeoutMs: number): Error { + const error = new Error(`${label} timed out after ${timeoutMs} ms`) as NodeJS.ErrnoException; + error.code = "ETIMEDOUT"; + return error; +} + +function collectExecStream( + stream: grpc.ClientReadableStream, + label: string, + timeoutMs?: number, +): Promise { + return new Promise((resolve, reject) => { + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let exitCode: number | null = null; + let settled = false; + let timer: NodeJS.Timeout | undefined; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + fn(); + }; + + stream.on("data", (event: any) => { + if (event?.stdout?.data !== undefined) stdout.push(bufferFromData(event.stdout.data)); + if (event?.stderr?.data !== undefined) stderr.push(bufferFromData(event.stderr.data)); + if (event?.exit?.exit_code !== undefined) { + const parsed = Number(event.exit.exit_code); + exitCode = Number.isFinite(parsed) ? parsed : 1; + } + }); + stream.on("error", (error) => { + settle(() => reject(new Error(`${label} failed: ${formatDirectGrpcError(error)}`))); + }); + stream.on("end", () => { + settle(() => + resolve({ + status: exitCode ?? 1, + stdout: Buffer.concat(stdout), + stderr: Buffer.concat(stderr), + }), + ); + }); + + if (timeoutMs && timeoutMs > 0) { + timer = setTimeout(() => { + try { + stream.cancel(); + } catch { + /* ignore */ + } + settle(() => reject(makeTimeoutError(label, timeoutMs))); + }, timeoutMs); + } + }); +} + +function sandboxIdFromResponse(response: any, sandboxName: string): string { + const sandbox = response?.sandbox; + const id = sandbox?.metadata?.id || sandbox?.id; + if (typeof id === "string" && id.trim()) return id; + throw new Error(`OpenShell gateway returned no sandbox id for '${sandboxName}'.`); +} + +function sandboxPhase(response: any): string | number | null { + const phase = response?.sandbox?.phase; + return typeof phase === "string" || typeof phase === "number" ? phase : null; +} + +function isReadyPhase(phase: string | number | null): boolean { + return phase === null || phase === "SANDBOX_PHASE_READY" || phase === 2 || phase === "2"; +} + +function callUnary( + client: GrpcClient, + name: string, + request: TRequest, + timeoutMs?: number, +): Promise { + const fn = method< + (request: TRequest, options: grpc.CallOptions, callback: grpc.requestCallback) => void + >(client, name); + return new Promise((resolve, reject) => { + fn(request, callOptions(timeoutMs), (error, value) => { + if (error) reject(new Error(`${name} failed: ${formatDirectGrpcError(error)}`)); + else resolve(value as TResponse); + }); + }); +} + +export class OpenShellDirectGrpcClient { + readonly gateway: ResolvedGatewayMetadata; + private readonly client: GrpcClient; + + constructor(options: DirectGrpcClientOptions = {}) { + this.gateway = options.gateway ?? resolveGatewayMetadata(options); + const OpenShell = getOpenShellConstructor(); + this.client = new OpenShell(this.gateway.target, createCredentials(this.gateway), { + "grpc.max_receive_message_length": 256 * 1024 * 1024, + "grpc.max_send_message_length": 256 * 1024 * 1024, + }) as GrpcClient; + } + + close(): void { + this.client.close(); + } + + async getSandboxId(sandboxName: string, timeoutMs?: number): Promise { + const response = await callUnary<{ name: string }, any>( + this.client, + "GetSandbox", + { name: sandboxName }, + timeoutMs, + ).catch((error) => { + throw new Error(`GetSandbox '${sandboxName}' failed: ${error instanceof Error ? error.message : String(error)}`); + }); + return sandboxIdFromResponse(response, sandboxName); + } + + async getReadySandboxIdForForward(sandboxName: string, timeoutMs?: number): Promise { + const response = await callUnary<{ name: string }, any>( + this.client, + "GetSandbox", + { name: sandboxName }, + timeoutMs, + ).catch((error) => { + throw new Error(`GetSandbox '${sandboxName}' failed: ${error instanceof Error ? error.message : String(error)}`); + }); + const phase = sandboxPhase(response); + if (!isReadyPhase(phase)) { + throw new Error(`sandbox '${sandboxName}' is not ready for forwarding (phase: ${String(phase)})`); + } + return sandboxIdFromResponse(response, sandboxName); + } + + async execBinaryStream( + sandboxName: string, + argv: string[], + opts: DirectExecOptions = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const execSandbox = method< + (request: unknown, options?: grpc.CallOptions) => grpc.ClientReadableStream + >(this.client, "ExecSandbox"); + return collectExecStream( + execSandbox(execRequest(sandboxId, argv, opts), callOptions(opts.timeoutMs)), + `ExecSandbox '${sandboxName}'`, + opts.timeoutMs, + ); + } + + async execText( + sandboxName: string, + argv: string[], + opts: DirectExecOptions = {}, + ): Promise { + const result = await this.execBinaryStream(sandboxName, argv, opts); + return { + status: result.status, + stdout: result.stdout.toString("utf-8"), + stderr: result.stderr.toString("utf-8"), + }; + } + + async createForwardSessionToken(sandboxId: string, timeoutMs?: number): Promise { + const response = await callUnary<{ sandbox_id: string }, any>( + this.client, + "CreateSshSession", + { sandbox_id: sandboxId }, + timeoutMs, + ); + const token = response?.token; + if (typeof token !== "string" || token.length === 0) { + throw new Error("OpenShell gateway returned no ForwardTcp authorization token."); + } + return token; + } + + async revokeForwardSessionToken(token: string, timeoutMs?: number): Promise { + if (!token) return; + try { + await callUnary<{ token: string }, unknown>(this.client, "RevokeSshSession", { token }, timeoutMs); + } catch { + /* best-effort cleanup */ + } + } + + async forwardTcpConnection( + socket: net.Socket, + params: { + sandboxName: string; + sandboxId: string; + targetHost: string; + targetPort: number; + serviceId?: string; + timeoutMs?: number; + }, + ): Promise { + const token = await this.createForwardSessionToken(params.sandboxId, params.timeoutMs); + const forwardTcp = method<() => grpc.ClientDuplexStream>(this.client, "ForwardTcp"); + const stream = forwardTcp(); + let streamEnded = false; + + try { + await new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: Error) => { + if (settled) return; + settled = true; + streamEnded = true; + socket.off("data", onSocketData); + socket.off("end", onSocketEnd); + socket.off("close", onSocketClose); + socket.off("error", onSocketError); + stream.off("data", onStreamData); + stream.off("end", onStreamEnd); + stream.off("error", onStreamError); + stream.on("error", () => { + /* swallow late cancellation after local socket shutdown */ + }); + if (error) reject(error); + else resolve(); + }; + const onSocketData = (chunk: Buffer) => { + if (!streamEnded) stream.write({ data: chunk }); + }; + const onSocketEnd = () => { + streamEnded = true; + stream.end(); + }; + const onSocketClose = () => { + if (!streamEnded) { + try { + stream.end(); + } catch { + /* ignore */ + } + } + streamEnded = true; + finish(); + }; + const onSocketError = (error: Error) => { + streamEnded = true; + try { + stream.cancel(); + } catch { + /* ignore */ + } + finish(error); + }; + const onStreamData = (frame: any) => { + const data = bufferFromData(frame?.data); + if (data.length > 0 && !socket.destroyed) socket.write(data); + }; + const onStreamEnd = () => { + streamEnded = true; + if (!socket.destroyed) socket.end(); + finish(); + }; + const onStreamError = (error: grpc.ServiceError) => { + streamEnded = true; + socket.destroy(); + finish(new Error(`ForwardTcp '${params.sandboxName}' failed: ${formatDirectGrpcError(error)}`)); + }; + + socket.on("data", onSocketData); + socket.on("end", onSocketEnd); + socket.on("close", onSocketClose); + socket.on("error", onSocketError); + stream.on("data", onStreamData); + stream.on("end", onStreamEnd); + stream.on("error", onStreamError); + + stream.write({ + init: { + sandbox_id: params.sandboxId, + service_id: params.serviceId || "", + tcp: { + host: params.targetHost, + port: params.targetPort, + }, + authorization_token: token, + }, + }); + }); + } finally { + await this.revokeForwardSessionToken(token, params.timeoutMs); + } + } + + async startForward( + sandboxName: string, + options: { + localPort: number; + localHost?: string; + targetHost?: string; + targetPort: number; + serviceId?: string; + timeoutMs?: number; + }, + ): Promise { + const localHost = options.localHost || "127.0.0.1"; + const targetHost = options.targetHost || "127.0.0.1"; + const sandboxId = await this.getReadySandboxIdForForward(sandboxName, options.timeoutMs); + const server = net.createServer((socket) => { + this.forwardTcpConnection(socket, { + sandboxName, + sandboxId, + targetHost, + targetPort: options.targetPort, + serviceId: options.serviceId, + timeoutMs: options.timeoutMs, + }).catch(() => socket.destroy()); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(options.localPort, localHost, () => { + server.off("error", reject); + resolve(); + }); + }); + return { + sandboxName, + localPort: options.localPort, + localHost, + targetHost, + targetPort: options.targetPort, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + }; + } + + async getSandboxLogs( + sandboxName: string, + opts: { + lines?: number; + sinceMs?: number; + sources?: string[]; + minLevel?: string; + timeoutMs?: number; + } = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const response = await callUnary( + this.client, + "GetSandboxLogs", + { + sandbox_id: sandboxId, + lines: opts.lines ?? 0, + since_ms: opts.sinceMs ?? 0, + sources: opts.sources ?? [], + min_level: opts.minLevel ?? "", + }, + opts.timeoutMs, + ).catch((error) => { + throw new Error( + `GetSandboxLogs '${sandboxName}' failed: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + return { + logs: Array.isArray(response?.logs) ? response.logs : [], + bufferTotal: Number(response?.buffer_total ?? 0), + }; + } + + async watchSandbox( + sandboxName: string, + opts: { + followStatus?: boolean; + followLogs?: boolean; + followEvents?: boolean; + logTailLines?: number; + eventTail?: number; + stopOnTerminal?: boolean; + logSinceMs?: number; + logSources?: string[]; + logMinLevel?: string; + timeoutMs?: number; + onEvent?: (event: SandboxWatchEvent) => void; + } = {}, + ): Promise { + const sandboxId = await this.getSandboxId(sandboxName, opts.timeoutMs); + const watchSandbox = method< + (request: unknown, options?: grpc.CallOptions) => grpc.ClientReadableStream + >(this.client, "WatchSandbox"); + const stream = watchSandbox( + { + id: sandboxId, + follow_status: opts.followStatus === true, + follow_logs: opts.followLogs === true, + follow_events: opts.followEvents === true, + log_tail_lines: opts.logTailLines ?? 0, + event_tail: opts.eventTail ?? 0, + stop_on_terminal: opts.stopOnTerminal === true, + log_since_ms: opts.logSinceMs ?? 0, + log_sources: opts.logSources ?? [], + log_min_level: opts.logMinLevel ?? "", + }, + callOptions(opts.timeoutMs), + ); + await new Promise((resolve, reject) => { + stream.on("data", (event: any) => { + if (!opts.onEvent) return; + if (event?.sandbox !== undefined) opts.onEvent({ type: "sandbox", sandbox: event.sandbox }); + if (event?.log !== undefined) opts.onEvent({ type: "log", log: event.log }); + if (event?.event !== undefined) opts.onEvent({ type: "event", event: event.event }); + if (event?.warning !== undefined) { + opts.onEvent({ + type: "warning", + message: String(event.warning.message || ""), + }); + } + }); + stream.on("error", (error) => { + reject(new Error(`WatchSandbox '${sandboxName}' failed: ${formatDirectGrpcError(error)}`)); + }); + stream.on("end", () => resolve()); + }); + } +} + +export function __clearDirectGrpcPackageCacheForTests(): void { + packageDefinition = null; +} + +export const __directGrpcTestHooks = { + protoRoot, + createCredentials, + formatDirectGrpcError, + bufferFromData, + isReadyPhase, +}; diff --git a/src/lib/adapters/openshell/forward-bridge-runner.ts b/src/lib/adapters/openshell/forward-bridge-runner.ts new file mode 100644 index 0000000000..d21a41373a --- /dev/null +++ b/src/lib/adapters/openshell/forward-bridge-runner.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createSandboxGrpcClient } from "./grpc"; +import { writeForwardState } from "./forward-bridge-state"; + +type RunnerArgs = { + sandboxName: string; + bind: string; + port: number; + targetHost: string; + targetPort: number; +}; + +async function main(): Promise { + const raw = process.argv[2]; + if (!raw) throw new Error("missing forward bridge args"); + const args = JSON.parse(raw) as RunnerArgs; + const client = createSandboxGrpcClient(); + const handle = await client.startForward(args.sandboxName, { + localHost: args.bind, + localPort: args.port, + targetHost: args.targetHost, + targetPort: args.targetPort, + serviceId: `nemoclaw-dashboard-${args.port}`, + }); + + writeForwardState({ + sandboxName: args.sandboxName, + bind: args.bind, + port: args.port, + targetHost: args.targetHost, + targetPort: args.targetPort, + pid: process.pid, + startedAt: new Date().toISOString(), + }); + + const close = async () => { + try { + await handle.close(); + } catch { + /* ignore */ + } + client.close(); + process.exit(0); + }; + process.on("SIGTERM", () => void close()); + process.on("SIGINT", () => void close()); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index 047e01a738..aeefbdfac9 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -38,7 +38,8 @@ function stateDir(): string { function statePath(sandboxName: string, port: number | string): string { const safeSandbox = sandboxName.replace(/[^A-Za-z0-9._-]/g, "_"); - return path.join(stateDir(), `${safeSandbox}-${String(port)}.json`); + const safePort = String(port).replace(/[^A-Za-z0-9._-]/g, "_"); + return path.join(stateDir(), `${safeSandbox}-${safePort}.json`); } function ensureStateDir(): void { @@ -190,6 +191,18 @@ function isTestForwardPid(pid: number): boolean { return useTestForwardBridge() && pid === testForwardPid(); } +function runnerCommand(): { command: string; args: string[] } { + const built = path.join(__dirname, "forward-bridge-runner.js"); + if (fs.existsSync(built)) return { command: process.execPath, args: [built] }; + + const source = path.join(__dirname, "forward-bridge-runner.ts"); + const root = path.resolve(__dirname, "..", "..", "..", ".."); + const tsxBin = path.join(root, "node_modules", ".bin", process.platform === "win32" ? "tsx.cmd" : "tsx"); + if (fs.existsSync(source) && fs.existsSync(tsxBin)) return { command: tsxBin, args: [source] }; + + throw new Error("OpenShell gRPC forward bridge runner is not available. Run `npm run build:cli` first."); +} + function probeForwardReady(bind: string, port: number): boolean { const host = bind === "0.0.0.0" || bind === "::" ? "127.0.0.1" : bind; const script = @@ -230,12 +243,53 @@ export function startForwardBridgeDetached( writeForwardState(state); return { ok: true, state, diagnostic: "" }; } - void targetHost; - void timeoutMs; + const { command, args } = runnerCommand(); + ensureStateDir(); + const diagnosticPath = path.join( + stateDir(), + `${sandboxName.replace(/[^A-Za-z0-9._-]/g, "_")}-${options.port}.log`, + ); + const out = fs.openSync(diagnosticPath, "w", 0o600); + const child = spawn( + command, + [ + ...args, + JSON.stringify({ + sandboxName, + bind, + port: options.port, + targetHost, + targetPort: options.targetPort, + }), + ], + { detached: true, stdio: ["ignore", out, out], env: process.env }, + ); + child.unref(); + fs.closeSync(out); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const state = getForwardState(sandboxName, options.port); + if (state && state.pid === child.pid && probeForwardReady(state.bind, state.port)) { + return { ok: true, state, diagnostic: "" }; + } + sleepMs(250); + } + + let diagnostic = ""; + try { + diagnostic = fs.readFileSync(diagnosticPath, "utf-8").trim(); + } catch { + /* ignore */ + } + try { + if (child.pid) process.kill(child.pid, "SIGTERM"); + } catch { + /* ignore */ + } return { ok: false, - diagnostic: - "OpenShell SDK forwarding is not available yet. Land an upstream @openshell/sdk forwarding API before enabling dashboard forwarding.", + diagnostic: diagnostic || `forward bridge did not become ready within ${String(timeoutMs)}ms`, }; } @@ -244,4 +298,5 @@ export const __forwardBridgeTestHooks = { probeForwardReady, waitForPortRelease, waitForPidExit, + runnerCommand, }; diff --git a/src/lib/adapters/openshell/grpc-migration-guard.test.ts b/src/lib/adapters/openshell/grpc-migration-guard.test.ts index c521947a33..7369f81903 100644 --- a/src/lib/adapters/openshell/grpc-migration-guard.test.ts +++ b/src/lib/adapters/openshell/grpc-migration-guard.test.ts @@ -23,6 +23,8 @@ const PRODUCTION_TARGETS = [ "src/lib/status-command-deps.ts", "src/lib/tunnel/services.ts", "src/lib/verify-deployment.ts", + "src/lib/adapters/openshell/direct-grpc.ts", + "src/lib/adapters/openshell/forward-bridge-runner.ts", "src/lib/adapters/openshell/grpc.ts", "src/lib/adapters/openshell/sync-runner.ts", "package.json", @@ -30,12 +32,6 @@ const PRODUCTION_TARGETS = [ ]; const BANNED = [ - /@grpc\/grpc-js/, - /@grpc\/proto-loader/, - /\bprotoLoader\b/, - /\bloadPackageDefinition\b/, - /\bCreateSshSession\b/, - /\bForwardTcp\b/, /captureSandboxSshConfig/, /sandbox ssh-config/, /spawnSync\(\s*["']ssh["']/, @@ -44,7 +40,6 @@ const BANNED = [ /sandbox download/, /forward start --background/, /openshell forward start/, - /copy-openshell-protos/, ]; function filesUnder(target: string): string[] { @@ -65,7 +60,7 @@ function filesUnder(target: string): string[] { } describe("OpenShell SDK migration guard", () => { - it("keeps sandbox lifecycle production code off SSH and SSH-backed forwards", () => { + it("keeps sandbox lifecycle production code off SSH shelling and legacy OpenShell CLI forwards", () => { const violations: string[] = []; for (const target of PRODUCTION_TARGETS) { for (const file of filesUnder(target)) { diff --git a/src/lib/adapters/openshell/grpc.test.ts b/src/lib/adapters/openshell/grpc.test.ts index 1206447af4..ffcd2fea0d 100644 --- a/src/lib/adapters/openshell/grpc.test.ts +++ b/src/lib/adapters/openshell/grpc.test.ts @@ -5,13 +5,36 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import * as grpc from "@grpc/grpc-js"; +import * as protoLoader from "@grpc/proto-loader"; import { describe, expect, it, vi } from "vitest"; vi.mock("@openshell/sdk", () => ({ OpenShellClient: { connect: vi.fn() }, })); -import { __grpcTestHooks, execBinaryStreamSync } from "./grpc"; +import { __directGrpcTestHooks } from "./direct-grpc"; +import { + __clearSandboxSdkClientCacheForTests, + __grpcTestHooks, + createSandboxGrpcClient, + execBinaryStreamSync, +} from "./grpc"; + +function loadOpenShellService(): grpc.ServiceDefinition { + const protoRoot = __directGrpcTestHooks.protoRoot(); + const definition = protoLoader.loadSync(path.join(protoRoot, "openshell.proto"), { + defaults: false, + enums: String, + includeDirs: [protoRoot], + keepCase: true, + longs: String, + oneofs: true, + bytes: Buffer, + }); + const loaded = grpc.loadPackageDefinition(definition) as any; + return loaded.openshell.v1.OpenShell.service as grpc.ServiceDefinition; +} describe("OpenShell SDK adapter", () => { it("maps NemoClaw exec options to SDK exec options", () => { @@ -49,6 +72,53 @@ describe("OpenShell SDK adapter", () => { expect(__grpcTestHooks.formatSdkError(new Error("plain failure"))).toBe("plain failure"); }); + it("falls back to raw ExecSandbox when the SDK package is still the placeholder", async () => { + const { OpenShellClient } = await import("@openshell/sdk"); + vi.mocked(OpenShellClient.connect).mockRejectedValueOnce( + new Error("@openshell/sdk is not published yet"), + ); + __clearSandboxSdkClientCacheForTests(); + + const server = new grpc.Server(); + server.addService(loadOpenShellService(), { + GetSandbox: (_call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) => { + callback(null, { + sandbox: { + metadata: { id: "sandbox-123", name: "alpha" }, + phase: "SANDBOX_PHASE_READY", + }, + }); + }, + ExecSandbox: (call: grpc.ServerWritableStream) => { + expect(call.request).toMatchObject({ + sandbox_id: "sandbox-123", + command: ["sh", "-c", "printf ok"], + timeout_seconds: 1, + }); + call.write({ stdout: { data: Buffer.from("ok") } }); + call.write({ exit: { exit_code: 0 } }); + call.end(); + }, + }); + const port = await new Promise((resolve, reject) => { + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, boundPort) => { + if (error) reject(error); + else resolve(boundPort); + }); + }); + const client = createSandboxGrpcClient({ gatewayEndpoint: `http://127.0.0.1:${port}` }); + try { + await expect(client.execText("alpha", ["sh", "-c", "printf ok"], { timeoutMs: 1_000 })).resolves.toEqual({ + status: 0, + stdout: "ok", + stderr: "", + }); + } finally { + client.close(); + server.forceShutdown(); + } + }); + it("preserves large sync-runner binary stdout through the fake SDK runner", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sdk-large-stdout-")); const oldEnv = { diff --git a/src/lib/adapters/openshell/grpc.ts b/src/lib/adapters/openshell/grpc.ts index b530a70816..47a8285518 100644 --- a/src/lib/adapters/openshell/grpc.ts +++ b/src/lib/adapters/openshell/grpc.ts @@ -19,6 +19,14 @@ import { type ResolvedGatewayMetadata, resolveGatewayMetadata, } from "./gateway-metadata"; +import { + formatDirectGrpcError, + OpenShellDirectGrpcClient, + type SandboxLogsResult, + type SandboxWatchEvent, +} from "./direct-grpc"; + +export type { SandboxLogLine, SandboxLogsResult, SandboxWatchEvent } from "./direct-grpc"; export interface SandboxExecOptions { workdir?: string; @@ -155,6 +163,14 @@ export function formatSdkError(error: unknown): string { export const formatGrpcError = formatSdkError; +function shouldTryDirectGrpcFallback(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("@openshell/sdk is not published yet") || + message.includes("does not expose client-certificate auth yet") + ); +} + export class SandboxGrpcClient { readonly gateway: ResolvedGatewayMetadata; @@ -235,6 +251,21 @@ export class SandboxGrpcClient { const result = await (await this.sdk()).exec(sandboxName, argv, toSdkExecOptions(opts)); return sdkResultToStream(result); } catch (error) { + if (shouldTryDirectGrpcFallback(error)) { + const direct = new OpenShellDirectGrpcClient({ gateway: this.gateway }); + try { + return await direct.execBinaryStream(sandboxName, argv, opts); + } catch (directError) { + const detail = + directError instanceof Error ? directError.message : formatDirectGrpcError(directError); + throw new Error( + `OpenShell SDK exec '${sandboxName}' failed: ${formatSdkError(error)}; ` + + `direct gRPC ExecSandbox fallback failed: ${detail}`, + ); + } finally { + direct.close(); + } + } throw new Error(`OpenShell SDK exec '${sandboxName}' failed: ${formatSdkError(error)}`); } } @@ -271,7 +302,7 @@ export class SandboxGrpcClient { async startForward( sandboxName: string, - _options: { + options: { localPort: number; localHost?: string; targetHost?: string; @@ -279,10 +310,66 @@ export class SandboxGrpcClient { serviceId?: string; }, ): Promise { - throw new Error( - `OpenShell SDK forwarding is not available for '${sandboxName}' yet. ` + - "Land an upstream @openshell/sdk forwarding API before enabling dashboard forwarding through the SDK.", - ); + const direct = new OpenShellDirectGrpcClient({ gateway: this.gateway }); + try { + const handle = await direct.startForward(sandboxName, options); + return { + ...handle, + close: async () => { + try { + await handle.close(); + } finally { + direct.close(); + } + }, + }; + } catch (error) { + direct.close(); + const detail = error instanceof Error ? error.message : formatDirectGrpcError(error); + throw new Error(`OpenShell gRPC ForwardTcp '${sandboxName}' failed: ${detail}`); + } + } + + async getSandboxLogs( + sandboxName: string, + opts: { + lines?: number; + sinceMs?: number; + sources?: string[]; + minLevel?: string; + timeoutMs?: number; + } = {}, + ): Promise { + const direct = new OpenShellDirectGrpcClient({ gateway: this.gateway }); + try { + return await direct.getSandboxLogs(sandboxName, opts); + } finally { + direct.close(); + } + } + + async watchSandbox( + sandboxName: string, + opts: { + followStatus?: boolean; + followLogs?: boolean; + followEvents?: boolean; + logTailLines?: number; + eventTail?: number; + stopOnTerminal?: boolean; + logSinceMs?: number; + logSources?: string[]; + logMinLevel?: string; + timeoutMs?: number; + onEvent?: (event: SandboxWatchEvent) => void; + } = {}, + ): Promise { + const direct = new OpenShellDirectGrpcClient({ gateway: this.gateway }); + try { + await direct.watchSandbox(sandboxName, opts); + } finally { + direct.close(); + } } } @@ -400,4 +487,5 @@ export const __grpcTestHooks = { toSdkExecOptions, sdkResultToStream, cacheKey, + shouldTryDirectGrpcFallback, }; diff --git a/src/lib/adapters/openshell/proto/datamodel.proto b/src/lib/adapters/openshell/proto/datamodel.proto new file mode 100644 index 0000000000..f92d7b7a36 --- /dev/null +++ b/src/lib/adapters/openshell/proto/datamodel.proto @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.datamodel.v1; + +// Kubernetes-style metadata shared by all top-level OpenShell domain objects. +// +// This structure provides consistent metadata (identity, labels, timestamps, +// resource versioning) across Sandbox, Provider, SshSession, and other resources. +message ObjectMeta { + // Stable object ID generated by the gateway. + string id = 1; + + // Human-readable object name (unique per object type). + string name = 2; + + // Milliseconds since Unix epoch when the object was created. + int64 created_at_ms = 3; + + // Key-value labels for filtering and organization. + // Labels must follow Kubernetes conventions: alphanumeric + `-._/`, max 63 chars per segment. + map labels = 4; + + // Optimistic concurrency control version. + // Incremented by the gateway on each update. Clients can use this for compare-and-swap operations. + uint64 resource_version = 5; +} + +// Provider model stored by OpenShell. +message Provider { + // Kubernetes-style metadata (id, name, labels, timestamps, resource version). + ObjectMeta metadata = 1; + // Canonical provider type slug (for example: "claude", "gitlab"). + string type = 2; + // Secret values used for authentication. + map credentials = 3; + // Non-secret provider configuration. + map config = 4; + // Expiration timestamps for credential values, keyed by credential/env var + // name. A zero or missing value means the credential does not expire. + map credential_expires_at_ms = 5; +} diff --git a/src/lib/adapters/openshell/proto/openshell.proto b/src/lib/adapters/openshell/proto/openshell.proto new file mode 100644 index 0000000000..90d1594f74 --- /dev/null +++ b/src/lib/adapters/openshell/proto/openshell.proto @@ -0,0 +1,1773 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.v1; + +import "datamodel.proto"; +import "google/protobuf/struct.proto"; +import "sandbox.proto"; + +// OpenShell service provides sandbox, provider, and runtime management capabilities. +// +// Conventions: +// - This file owns the public API resource model exposed to OpenShell clients. +// - `Sandbox`, `SandboxSpec`, `SandboxStatus`, and `SandboxPhase` are gateway-owned +// public types. Internal compute drivers must not import or return them directly. +// - The gateway translates internal compute-driver observations into these public +// resource messages before persisting or returning them to clients. +service OpenShell { + // Check the health of the service. + rpc Health(HealthRequest) returns (HealthResponse); + + // Create a new sandbox. + rpc CreateSandbox(CreateSandboxRequest) returns (SandboxResponse); + + // Fetch a sandbox by name. + rpc GetSandbox(GetSandboxRequest) returns (SandboxResponse); + + // List sandboxes. + rpc ListSandboxes(ListSandboxesRequest) returns (ListSandboxesResponse); + + // List provider records attached to a sandbox. + rpc ListSandboxProviders(ListSandboxProvidersRequest) + returns (ListSandboxProvidersResponse); + + // Attach a provider record to an existing sandbox. + rpc AttachSandboxProvider(AttachSandboxProviderRequest) + returns (AttachSandboxProviderResponse); + + // Detach a provider record from an existing sandbox. + rpc DetachSandboxProvider(DetachSandboxProviderRequest) + returns (DetachSandboxProviderResponse); + + // Delete a sandbox by name. + rpc DeleteSandbox(DeleteSandboxRequest) returns (DeleteSandboxResponse); + + // Create a short-lived SSH session for a sandbox. + rpc CreateSshSession(CreateSshSessionRequest) returns (CreateSshSessionResponse); + + // Create or update a sandbox HTTP service endpoint for local routing. + rpc ExposeService(ExposeServiceRequest) returns (ServiceEndpointResponse); + + // Fetch one sandbox HTTP service endpoint. + rpc GetService(GetServiceRequest) returns (ServiceEndpointResponse); + + // List sandbox HTTP service endpoints. + rpc ListServices(ListServicesRequest) returns (ListServicesResponse); + + // Delete one sandbox HTTP service endpoint. + rpc DeleteService(DeleteServiceRequest) returns (DeleteServiceResponse); + + // Revoke a previously issued SSH session. + rpc RevokeSshSession(RevokeSshSessionRequest) returns (RevokeSshSessionResponse); + + // Execute a command in a ready sandbox and stream output. + rpc ExecSandbox(ExecSandboxRequest) returns (stream ExecSandboxEvent); + + // Forward one CLI-side TCP connection to a loopback TCP target in a sandbox. + rpc ForwardTcp(stream TcpForwardFrame) returns (stream TcpForwardFrame); + + // Execute an interactive command with bidirectional stdin/stdout streaming. + // The first client message MUST carry an ExecSandboxInput with the start + // variant. Subsequent messages carry stdin bytes or window resize events. + rpc ExecSandboxInteractive(stream ExecSandboxInput) returns (stream ExecSandboxEvent); + + // Create a provider. + rpc CreateProvider(CreateProviderRequest) returns (ProviderResponse); + + // Fetch a provider by name. + rpc GetProvider(GetProviderRequest) returns (ProviderResponse); + + // List providers. + rpc ListProviders(ListProvidersRequest) returns (ListProvidersResponse); + + // List available provider type profiles. + rpc ListProviderProfiles(ListProviderProfilesRequest) + returns (ListProviderProfilesResponse); + + // Fetch one provider type profile by id. + rpc GetProviderProfile(GetProviderProfileRequest) + returns (ProviderProfileResponse); + + // Import custom provider type profiles. + rpc ImportProviderProfiles(ImportProviderProfilesRequest) + returns (ImportProviderProfilesResponse); + + // Validate provider type profiles without registering them. + rpc LintProviderProfiles(LintProviderProfilesRequest) + returns (LintProviderProfilesResponse); + + // Update an existing provider by name. + rpc UpdateProvider(UpdateProviderRequest) returns (ProviderResponse); + + // Fetch refresh status for one provider or provider credential. + rpc GetProviderRefreshStatus(GetProviderRefreshStatusRequest) + returns (GetProviderRefreshStatusResponse); + + // Configure gateway-owned refresh material for one provider credential. + rpc ConfigureProviderRefresh(ConfigureProviderRefreshRequest) + returns (ConfigureProviderRefreshResponse); + + // Record a gateway-owned refresh request for one provider credential. + rpc RotateProviderCredential(RotateProviderCredentialRequest) + returns (RotateProviderCredentialResponse); + + // Delete gateway-owned refresh configuration for one provider credential. + rpc DeleteProviderRefresh(DeleteProviderRefreshRequest) + returns (DeleteProviderRefreshResponse); + + // Delete a provider by name. + rpc DeleteProvider(DeleteProviderRequest) returns (DeleteProviderResponse); + + // Delete a custom provider type profile by id. + rpc DeleteProviderProfile(DeleteProviderProfileRequest) + returns (DeleteProviderProfileResponse); + + // Get sandbox settings by id (called by sandbox entrypoint and poll loop). + rpc GetSandboxConfig(openshell.sandbox.v1.GetSandboxConfigRequest) + returns (openshell.sandbox.v1.GetSandboxConfigResponse); + + // Get gateway-global settings. + rpc GetGatewayConfig(openshell.sandbox.v1.GetGatewayConfigRequest) + returns (openshell.sandbox.v1.GetGatewayConfigResponse); + + // Update settings or policy at sandbox or global scope. + rpc UpdateConfig(UpdateConfigRequest) + returns (UpdateConfigResponse); + + // Get the load status of a specific policy version. + rpc GetSandboxPolicyStatus(GetSandboxPolicyStatusRequest) + returns (GetSandboxPolicyStatusResponse); + + // List policy history for a sandbox. + rpc ListSandboxPolicies(ListSandboxPoliciesRequest) + returns (ListSandboxPoliciesResponse); + + // Report policy load result (called by sandbox after reload attempt). + rpc ReportPolicyStatus(ReportPolicyStatusRequest) + returns (ReportPolicyStatusResponse); + + // Get provider environment for a sandbox (called by sandbox supervisor at startup). + rpc GetSandboxProviderEnvironment(GetSandboxProviderEnvironmentRequest) + returns (GetSandboxProviderEnvironmentResponse); + + // Fetch recent sandbox logs (one-shot). + rpc GetSandboxLogs(GetSandboxLogsRequest) returns (GetSandboxLogsResponse); + + // Push sandbox supervisor logs to the server (client-streaming). + rpc PushSandboxLogs(stream PushSandboxLogsRequest) returns (PushSandboxLogsResponse); + + // Persistent supervisor-to-gateway session (bidirectional streaming). + // + // The supervisor opens this stream at startup and keeps it alive for the + // sandbox lifetime. The gateway uses it to coordinate relay channels for + // SSH connect, ExecSandbox, and targetable sandbox services. Raw service + // bytes flow over RelayStream calls (separate HTTP/2 streams on the same + // connection), not over this stream. + rpc ConnectSupervisor(stream SupervisorMessage) returns (stream GatewayMessage); + + // Raw byte relay between supervisor and gateway. + // + // The supervisor initiates this call after receiving a RelayOpen message + // on its ConnectSupervisor stream. The first RelayFrame carries a + // RelayInit with the channel_id to associate the new HTTP/2 stream with + // the pending relay slot on the gateway. Subsequent frames carry raw bytes in either + // direction between the gateway-side waiter (ForwardTcp / exec handler) + // and the supervisor-side target bridge. + // + // This rides the same TCP+TLS+HTTP/2 connection as ConnectSupervisor — + // no new TLS handshake, no reverse HTTP CONNECT. + rpc RelayStream(stream RelayFrame) returns (stream RelayFrame); + + // Watch a sandbox and stream updates. + // + // This stream can include: + // - Sandbox status snapshots (phase/status) + // - OpenShell server process logs correlated by sandbox_id + // - Platform events correlated to the sandbox + rpc WatchSandbox(WatchSandboxRequest) returns (stream SandboxStreamEvent); + + // --------------------------------------------------------------------------- + // Draft policy recommendation RPCs + // --------------------------------------------------------------------------- + + // Submit denial analysis results from sandbox (summaries + proposed chunks). + rpc SubmitPolicyAnalysis(SubmitPolicyAnalysisRequest) + returns (SubmitPolicyAnalysisResponse); + + // Get draft policy recommendations for a sandbox. + rpc GetDraftPolicy(GetDraftPolicyRequest) returns (GetDraftPolicyResponse); + + // Approve a single draft policy chunk (merges into active policy). + rpc ApproveDraftChunk(ApproveDraftChunkRequest) + returns (ApproveDraftChunkResponse); + + // Reject a single draft policy chunk. + rpc RejectDraftChunk(RejectDraftChunkRequest) + returns (RejectDraftChunkResponse); + + // Approve all pending draft chunks (skips security-flagged unless forced). + rpc ApproveAllDraftChunks(ApproveAllDraftChunksRequest) + returns (ApproveAllDraftChunksResponse); + + // Edit a pending draft chunk in-place (e.g. narrow allowed_ips). + rpc EditDraftChunk(EditDraftChunkRequest) returns (EditDraftChunkResponse); + + // Reverse an approval (remove merged rule from active policy). + rpc UndoDraftChunk(UndoDraftChunkRequest) returns (UndoDraftChunkResponse); + + // Clear all pending draft chunks for a sandbox. + rpc ClearDraftChunks(ClearDraftChunksRequest) + returns (ClearDraftChunksResponse); + + // Get decision history for a sandbox's draft policy. + rpc GetDraftHistory(GetDraftHistoryRequest) returns (GetDraftHistoryResponse); + + // Exchange a sandbox-bootstrap credential (e.g. a Kubernetes projected + // ServiceAccount token) for a gateway-minted JWT bound to the calling + // sandbox's UUID. Used by the Kubernetes driver path; singleplayer + // drivers receive the gateway JWT directly from the create-sandbox flow + // and never call this RPC. + rpc IssueSandboxToken(IssueSandboxTokenRequest) returns (IssueSandboxTokenResponse); + + // Renew the calling sandbox's gateway JWT. Older tokens remain valid + // until their own expiry; deployments should keep token TTLs short to + // bound replay exposure. The supervisor calls this from a background + // task at ~80% of the token's lifetime; the new token is cached in + // memory only — the on-disk bootstrap file is intentionally not + // rewritten. + rpc RefreshSandboxToken(RefreshSandboxTokenRequest) + returns (RefreshSandboxTokenResponse); +} + +// IssueSandboxToken request. Empty body; identity is established by the +// authentication credentials carried in the request headers (a projected +// Kubernetes ServiceAccount JWT in the K8s driver path). +message IssueSandboxTokenRequest {} + +// IssueSandboxToken response. The supervisor caches the returned token in +// memory and presents it as `Authorization: Bearer` on every subsequent +// gateway RPC. +message IssueSandboxTokenResponse { + // Gateway-minted JWT bound to the calling sandbox's UUID. + string token = 1; + // Absolute expiry of the issued token, milliseconds since the epoch. + int64 expires_at_ms = 2; +} + +// RefreshSandboxToken request. Empty body; the calling principal must +// already be a sandbox principal (i.e. the request carries a still-valid +// gateway-minted JWT in its Authorization header). +message RefreshSandboxTokenRequest {} + +// RefreshSandboxToken response. The new token replaces the supervisor's +// in-memory bearer credential. +message RefreshSandboxTokenResponse { + // Fresh gateway-minted JWT bound to the same sandbox UUID. + string token = 1; + // Absolute expiry of the new token, milliseconds since the epoch. + int64 expires_at_ms = 2; +} + +// Health check request. +message HealthRequest {} + +// Health check response. +message HealthResponse { + // Service status. + ServiceStatus status = 1; + + // Service version. + string version = 2; +} + +// Public sandbox resource exposed by the OpenShell API. +// +// This is the canonical gateway-owned view of a sandbox. It merges user intent +// (`spec`) with gateway-managed metadata and status derived from internal +// compute-driver observations. +// +// Note: The `namespace` field has been removed from the public API. It remains +// in the internal `DriverSandbox` message as a compute-driver implementation detail. +message Sandbox { + // Kubernetes-style metadata (id, name, labels, timestamps, resource version). + openshell.datamodel.v1.ObjectMeta metadata = 1; + // Desired sandbox configuration submitted through the API. + SandboxSpec spec = 2; + // Latest user-facing observed status derived by the gateway. + SandboxStatus status = 3; + // Gateway-derived lifecycle summary. + SandboxPhase phase = 4; + // Currently active policy version (updated when sandbox reports loaded). + uint32 current_policy_version = 5; +} + +// Desired sandbox configuration provided through the public API. +message SandboxSpec { + // Log level exposed to processes running inside the sandbox. + string log_level = 1; + // Environment variables injected into the sandbox runtime. + map environment = 5; + // Container or VM template used to provision the sandbox. + SandboxTemplate template = 6; + // Required sandbox policy configuration. + openshell.sandbox.v1.SandboxPolicy policy = 7; + // Provider names to attach to this sandbox. + repeated string providers = 8; + // Request NVIDIA GPU resources for this sandbox. + bool gpu = 9; + // Optional PCI BDF address (e.g. "0000:2d:00.0") or device index + // (e.g. "0", "1"). When empty with gpu=true, the driver assigns the + // first available GPU. + string gpu_device = 10; +} + +// Public sandbox template mapped onto compute-driver template inputs. +message SandboxTemplate { + // Fully-qualified OCI image reference used to boot the sandbox. + string image = 1; + // Optional runtime class name requested from the compute platform. + string runtime_class_name = 2; + // Optional agent socket path exposed to the workload. + string agent_socket = 3; + // Labels applied to compute-platform resources for this sandbox. + map labels = 4; + // Annotations applied to compute-platform resources for this sandbox. + map annotations = 5; + // Additional environment variables injected by the template. + map environment = 6; + // Platform-specific compute resource requirements and limits. + google.protobuf.Struct resources = 7; + // Optional platform-specific volume claim templates. + google.protobuf.Struct volume_claim_templates = 9; + // Enable Kubernetes user namespace isolation (hostUsers: false). + // When true, container UID 0 maps to a non-root host UID and capabilities + // become namespaced. Requires Kubernetes 1.33+ with user namespace support + // available (beta through 1.35, GA in 1.36+) and a supporting runtime. + // When unset, the cluster-wide default is used. + optional bool user_namespaces = 10; +} + +// User-facing sandbox status derived by the gateway from compute-driver observations. +// +// Lifecycle summary is exposed separately as `Sandbox.phase`. Public status does +// not embed driver-only flags such as `deleting`. +message SandboxStatus { + // Compute-platform sandbox object name. + string sandbox_name = 1; + // Name of the agent pod or equivalent runtime instance. + string agent_pod = 2; + // File descriptor or endpoint for reaching the agent service, when available. + string agent_fd = 3; + // File descriptor or endpoint for reaching the sandbox service, when available. + string sandbox_fd = 4; + // Latest user-facing readiness and lifecycle conditions. + repeated SandboxCondition conditions = 5; +} + +// User-facing sandbox condition derived from driver-native conditions. +message SandboxCondition { + // Condition class, typically mirroring the underlying platform condition type. + string type = 1; + // Condition status value such as `True`, `False`, or `Unknown`. + string status = 2; + // Short machine-readable reason associated with the condition. + string reason = 3; + // Human-readable condition message. + string message = 4; + // Timestamp reported by the underlying platform for the last transition. + string last_transition_time = 5; +} + +// High-level sandbox lifecycle phase derived by the gateway. +// +// Clients should rely on this normalized lifecycle summary for readiness and +// deletion decisions instead of interpreting raw conditions. +enum SandboxPhase { + SANDBOX_PHASE_UNSPECIFIED = 0; + SANDBOX_PHASE_PROVISIONING = 1; + SANDBOX_PHASE_READY = 2; + SANDBOX_PHASE_ERROR = 3; + SANDBOX_PHASE_DELETING = 4; + SANDBOX_PHASE_UNKNOWN = 5; +} + +// Public platform event exposed on the sandbox watch stream. +message PlatformEvent { + // Event timestamp in milliseconds since epoch. + int64 timestamp_ms = 1; + // Event source (e.g. "kubernetes", "docker", "process"). + string source = 2; + // Event type/severity (e.g. "Normal", "Warning"). + string type = 3; + // Short reason code (e.g. "Started", "Pulled", "Failed"). + string reason = 4; + // Human-readable event message. + string message = 5; + // Optional metadata as key-value pairs. + map metadata = 6; +} + +// Create sandbox request. +message CreateSandboxRequest { + SandboxSpec spec = 1; + // Optional user-supplied sandbox name. When empty the server generates one. + string name = 2; + // Optional labels for the sandbox (key-value metadata). + map labels = 3; +} + +// Get sandbox request. +message GetSandboxRequest { + // Sandbox name (canonical lookup key). + string name = 1; +} + +// List sandboxes request. +message ListSandboxesRequest { + uint32 limit = 1; + uint32 offset = 2; + // Optional label selector for filtering (format: "key1=value1,key2=value2"). + string label_selector = 3; +} + +// List providers attached to a sandbox request. +message ListSandboxProvidersRequest { + // Sandbox name (canonical lookup key). + string sandbox_name = 1; +} + +// Attach provider to sandbox request. +message AttachSandboxProviderRequest { + // Sandbox name (canonical lookup key). + string sandbox_name = 1; + // Provider name to attach. + string provider_name = 2; + // Expected resource version for optimistic concurrency control. + // If 0, the server uses the current version (backward compatibility). + // If non-zero, the server validates that the sandbox's current resource_version + // matches this value before applying the mutation, returning ABORTED on mismatch. + uint64 expected_resource_version = 3; +} + +// Detach provider from sandbox request. +message DetachSandboxProviderRequest { + // Sandbox name (canonical lookup key). + string sandbox_name = 1; + // Provider name to detach. + string provider_name = 2; + // Expected resource version for optimistic concurrency control. + // If 0, the server uses the current version (backward compatibility). + // If non-zero, the server validates that the sandbox's current resource_version + // matches this value before applying the mutation, returning ABORTED on mismatch. + uint64 expected_resource_version = 3; +} + +// Delete sandbox request. +message DeleteSandboxRequest { + // Sandbox name (canonical lookup key). + string name = 1; +} + +// Sandbox response. +message SandboxResponse { + Sandbox sandbox = 1; +} + +// List sandboxes response. +message ListSandboxesResponse { + repeated Sandbox sandboxes = 1; +} + +// List providers attached to a sandbox response. +message ListSandboxProvidersResponse { + repeated openshell.datamodel.v1.Provider providers = 1; +} + +// Attach provider to sandbox response. +message AttachSandboxProviderResponse { + Sandbox sandbox = 1; + // True when the provider was newly attached. False means it was already attached. + bool attached = 2; +} + +// Detach provider from sandbox response. +message DetachSandboxProviderResponse { + Sandbox sandbox = 1; + // True when the provider was removed. False means it was not attached. + bool detached = 2; +} + +// Delete sandbox response. +message DeleteSandboxResponse { + bool deleted = 1; +} + +// Create SSH session request. +message CreateSshSessionRequest { + // Sandbox id. + string sandbox_id = 1; +} + +// Create SSH session response. +// +// Fields are interpolated into an SSH `ProxyCommand` string that OpenSSH +// executes through `/bin/sh -c` on the caller's workstation. Servers MUST +// uphold the charset contract below; clients MUST reject responses that +// violate it. The client's own escaping provides defense-in-depth, but +// narrow charsets close injection vectors at the trust boundary. +message CreateSshSessionResponse { + // Sandbox id. [A-Za-z0-9._-]{1,128}. + string sandbox_id = 1; + + // Session token for the gateway tunnel. URL-safe ASCII + // ([A-Za-z0-9._~+/=-]) up to 4096 bytes. No shell metacharacters or + // whitespace. + string token = 2; + + // Gateway host for SSH proxy connection. IPv4 address, bracketed IPv6 + // address, or DNS hostname (Punycode-encoded for IDN). Alphanumeric plus + // `.-:[]` only, up to 253 bytes. + string gateway_host = 3; + + // Gateway port for SSH proxy connection. Must be in range 1..=65535. + uint32 gateway_port = 4; + + // Gateway scheme. Must be exactly "http" or "https". + string gateway_scheme = 5; + + // Optional host key fingerprint. If non-empty, [A-Za-z0-9:+/=-] only. + string host_key_fingerprint = 7; + + // Expiry timestamp in milliseconds since epoch. 0 means no expiry. + int64 expires_at_ms = 8; +} + +// Request to expose an HTTP service running inside a sandbox. +message ExposeServiceRequest { + // Sandbox name. + string sandbox = 1; + // Service name within the sandbox. + string service = 2; + // Loopback TCP port inside the sandbox. + uint32 target_port = 3; + // Whether to print/use the browser-facing service URL. + bool domain = 4; +} + +// Request to fetch an exposed sandbox service endpoint. +message GetServiceRequest { + // Sandbox name. + string sandbox = 1; + // Service name within the sandbox. Empty selects the unnamed endpoint. + string service = 2; +} + +// Request to list exposed sandbox service endpoints. +message ListServicesRequest { + // Optional sandbox name. Empty lists endpoints for all sandboxes. + string sandbox = 1; + // Page size. Zero uses the server default. + uint32 limit = 2; + // Page offset. + uint32 offset = 3; +} + +// Response containing exposed sandbox service endpoints. +message ListServicesResponse { + repeated ServiceEndpointResponse services = 1; +} + +// Request to delete an exposed sandbox service endpoint. +message DeleteServiceRequest { + // Sandbox name. + string sandbox = 1; + // Service name within the sandbox. Empty selects the unnamed endpoint. + string service = 2; +} + +// Response for deleting an exposed sandbox service endpoint. +message DeleteServiceResponse { + // True when an endpoint existed and was deleted. + bool deleted = 1; +} + +// Persisted sandbox service endpoint. +message ServiceEndpoint { + // Kubernetes-style metadata. + openshell.datamodel.v1.ObjectMeta metadata = 1; + // Sandbox object ID. + string sandbox_id = 2; + // Sandbox name. + string sandbox_name = 3; + // Service name within the sandbox. + string service_name = 4; + // Loopback TCP port inside the sandbox. + uint32 target_port = 5; + // Whether browser-facing service routing is enabled for this endpoint. + bool domain = 6; +} + +// Response containing a service endpoint and, when available, its local URL. +message ServiceEndpointResponse { + ServiceEndpoint endpoint = 1; + string url = 2; +} + +// Revoke SSH session request. +message RevokeSshSessionRequest { + // Session token to revoke. + string token = 1; +} + +// Revoke SSH session response. +message RevokeSshSessionResponse { + // True when a session was revoked. + bool revoked = 1; +} + +// Execute command request. +message ExecSandboxRequest { + // Sandbox id. + string sandbox_id = 1; + + // Command and arguments. + repeated string command = 2; + + // Optional working directory. + string workdir = 3; + + // Optional environment overrides. + map environment = 4; + + // Optional timeout in seconds. 0 means no timeout. + uint32 timeout_seconds = 5; + + // Optional stdin payload passed to the command. + bytes stdin = 6; + + // Request a pseudo-terminal for the remote command. + bool tty = 7; + + // Initial terminal columns (used when tty=true, 0 = use default). + uint32 cols = 8; + + // Initial terminal rows (used when tty=true, 0 = use default). + uint32 rows = 9; +} + +// One stdout chunk from a sandbox exec. +message ExecSandboxStdout { + bytes data = 1; +} + +// One stderr chunk from a sandbox exec. +message ExecSandboxStderr { + bytes data = 1; +} + +// Final exit status for a sandbox exec. +message ExecSandboxExit { + int32 exit_code = 1; +} + +// One event in a sandbox exec stream. +message ExecSandboxEvent { + oneof payload { + ExecSandboxStdout stdout = 1; + ExecSandboxStderr stderr = 2; + ExecSandboxExit exit = 3; + } +} + +// Initial frame for one TCP forward stream. +message TcpForwardInit { + // Sandbox id. + string sandbox_id = 1; + // Optional service identifier for audit/correlation. + string service_id = 4; + // Target the gateway should request from the supervisor. + oneof target { + SshRelayTarget ssh = 5; + TcpRelayTarget tcp = 6; + } + // Optional target-specific authorization token. SSH targets use this as the + // short-lived SSH session token issued by CreateSshSession. + string authorization_token = 7; +} + +// A single frame on the CLI-to-gateway TCP forward stream. +message TcpForwardFrame { + oneof payload { + TcpForwardInit init = 1; + bytes data = 2; + } +} + +// Client-to-server message for interactive exec. +message ExecSandboxInput { + oneof payload { + // First message: exec request metadata. + ExecSandboxRequest start = 1; + // Subsequent messages: raw stdin bytes. + bytes stdin = 2; + // Terminal window size change. + ExecSandboxWindowResize resize = 3; + } +} + +// Terminal window resize event for interactive exec. +message ExecSandboxWindowResize { + uint32 cols = 1; + uint32 rows = 2; +} + + +// SSH session record stored in persistence. +message SshSession { + // Kubernetes-style metadata (id, name, labels, timestamps, resource version). + openshell.datamodel.v1.ObjectMeta metadata = 1; + + // Sandbox id. + string sandbox_id = 2; + + // Session token. + string token = 3; + + // Expiry timestamp in milliseconds since epoch. 0 means no expiry + // (backward-compatible default for sessions created before this field existed). + int64 expires_at_ms = 4; + + // Revoked flag. + bool revoked = 5; +} + +// Watch sandbox request. +message WatchSandboxRequest { + // Sandbox id. + string id = 1; + + // Stream sandbox status snapshots. + bool follow_status = 2; + + // Stream openshell-server process logs correlated to this sandbox. + bool follow_logs = 3; + + // Stream platform events correlated to this sandbox. + bool follow_events = 4; + + // Replay the last N log lines (best-effort) before following. + uint32 log_tail_lines = 5; + + // Replay the last N platform events (best-effort) before following. + uint32 event_tail = 6; + + // Stop streaming once the sandbox reaches a terminal phase (READY or ERROR). + bool stop_on_terminal = 7; + + // Only include log lines with timestamp >= this value (milliseconds since epoch). + // 0 means no time filter. Applies to both tail replay and live streaming. + int64 log_since_ms = 8; + + // Filter by log source (e.g. "gateway", "sandbox"). Empty means all sources. + repeated string log_sources = 9; + + // Minimum log level to include (e.g. "INFO", "WARN", "ERROR"). Empty means all levels. + string log_min_level = 10; +} + +// One event in a sandbox watch stream. +message SandboxStreamEvent { + oneof payload { + // Latest sandbox snapshot. + Sandbox sandbox = 1; + // One server log line/event. + SandboxLogLine log = 2; + // One platform event. + PlatformEvent event = 3; + // Warning from the server (e.g. missed messages due to lag). + SandboxStreamWarning warning = 4; + // Draft policy update notification. + DraftPolicyUpdate draft_policy_update = 5; + } +} + +// Log line correlated to a sandbox. +message SandboxLogLine { + string sandbox_id = 1; + int64 timestamp_ms = 2; + string level = 3; + string target = 4; + string message = 5; + // Log source: "gateway" (server-side) or "sandbox" (supervisor). + // Empty is treated as "gateway" for backward compatibility. + string source = 6; + // Structured key-value fields from the tracing event (e.g. dst_host, action). + map fields = 7; +} + +message SandboxStreamWarning { + string message = 1; +} + +// Create provider request. +message CreateProviderRequest { + openshell.datamodel.v1.Provider provider = 1; +} + +// Get provider request. +message GetProviderRequest { + string name = 1; +} + +// List providers request. +message ListProvidersRequest { + uint32 limit = 1; + uint32 offset = 2; +} + +// Update provider request. +message UpdateProviderRequest { + openshell.datamodel.v1.Provider provider = 1; + // Optional per-credential expiry timestamps to merge into the provider. + // A zero value removes the expiry for that credential. + map credential_expires_at_ms = 2; +} + +// Delete provider request. +message DeleteProviderRequest { + string name = 1; +} + +// Provider response. +message ProviderResponse { + openshell.datamodel.v1.Provider provider = 1; +} + +// List providers response. +message ListProvidersResponse { + repeated openshell.datamodel.v1.Provider providers = 1; +} + +// List provider type profiles request. +message ListProviderProfilesRequest { + uint32 limit = 1; + uint32 offset = 2; +} + +// Fetch provider type profile request. +message GetProviderProfileRequest { + string id = 1; +} + +// Provider profile payload with optional source metadata for diagnostics. +message ProviderProfileImportItem { + ProviderProfile profile = 1; + string source = 2; +} + +// Provider profile validation diagnostic. +message ProviderProfileDiagnostic { + string source = 1; + string profile_id = 2; + string field = 3; + string message = 4; + string severity = 5; +} + +// Provider credential declaration. +message ProviderProfileCredential { + string name = 1; + string description = 2; + repeated string env_vars = 3; + bool required = 4; + string auth_style = 5; + string header_name = 6; + string query_param = 7; + ProviderCredentialRefresh refresh = 8; +} + +enum ProviderCredentialRefreshStrategy { + PROVIDER_CREDENTIAL_REFRESH_STRATEGY_UNSPECIFIED = 0; + PROVIDER_CREDENTIAL_REFRESH_STRATEGY_STATIC = 1; + PROVIDER_CREDENTIAL_REFRESH_STRATEGY_EXTERNAL = 2; + PROVIDER_CREDENTIAL_REFRESH_STRATEGY_OAUTH2_REFRESH_TOKEN = 3; + PROVIDER_CREDENTIAL_REFRESH_STRATEGY_OAUTH2_CLIENT_CREDENTIALS = 4; + PROVIDER_CREDENTIAL_REFRESH_STRATEGY_GOOGLE_SERVICE_ACCOUNT_JWT = 5; +} + +message ProviderCredentialRefreshMaterial { + string name = 1; + string description = 2; + bool required = 3; + bool secret = 4; +} + +message ProviderCredentialRefresh { + ProviderCredentialRefreshStrategy strategy = 1; + string token_url = 2; + repeated string scopes = 3; + int64 refresh_before_seconds = 4; + int64 max_lifetime_seconds = 5; + repeated ProviderCredentialRefreshMaterial material = 6; +} + +message ProviderCredentialRefreshStatus { + string provider_name = 1; + string provider_id = 2; + string credential_key = 3; + ProviderCredentialRefreshStrategy strategy = 4; + string status = 5; + int64 expires_at_ms = 6; + int64 next_refresh_at_ms = 7; + int64 last_refresh_at_ms = 8; + string last_error = 9; +} + +// Provider profile local discovery declaration. +message ProviderProfileDiscovery { + // Credential names from ProviderProfile.credentials eligible for local discovery. + repeated string credentials = 1; +} + +message StoredProviderCredentialRefreshState { + openshell.datamodel.v1.ObjectMeta metadata = 1; + string provider_id = 2; + string provider_name = 3; + string credential_key = 4; + ProviderCredentialRefreshStrategy strategy = 5; + map material = 6; + repeated string secret_material_keys = 7; + int64 expires_at_ms = 8; + int64 next_refresh_at_ms = 9; + int64 last_refresh_at_ms = 10; + string status = 11; + string last_error = 12; + string token_url = 13; + repeated string scopes = 14; + int64 refresh_before_seconds = 15; + int64 max_lifetime_seconds = 16; +} + +message GetProviderRefreshStatusRequest { + string provider = 1; + string credential_key = 2; +} + +message GetProviderRefreshStatusResponse { + repeated ProviderCredentialRefreshStatus credentials = 1; +} + +message ConfigureProviderRefreshRequest { + string provider = 1; + string credential_key = 2; + ProviderCredentialRefreshStrategy strategy = 3; + map material = 4; + repeated string secret_material_keys = 5; + optional int64 expires_at_ms = 6; +} + +message ConfigureProviderRefreshResponse { + ProviderCredentialRefreshStatus status = 1; +} + +message RotateProviderCredentialRequest { + string provider = 1; + string credential_key = 2; +} + +message RotateProviderCredentialResponse { + ProviderCredentialRefreshStatus status = 1; +} + +message DeleteProviderRefreshRequest { + string provider = 1; + string credential_key = 2; +} + +message DeleteProviderRefreshResponse { + bool deleted = 1; +} + +// Stable provider profile categories used by clients for grouping and filtering. +enum ProviderProfileCategory { + PROVIDER_PROFILE_CATEGORY_UNSPECIFIED = 0; + PROVIDER_PROFILE_CATEGORY_OTHER = 1; + PROVIDER_PROFILE_CATEGORY_INFERENCE = 2; + PROVIDER_PROFILE_CATEGORY_AGENT = 3; + PROVIDER_PROFILE_CATEGORY_SOURCE_CONTROL = 4; + PROVIDER_PROFILE_CATEGORY_MESSAGING = 5; + PROVIDER_PROFILE_CATEGORY_DATA = 6; + PROVIDER_PROFILE_CATEGORY_KNOWLEDGE = 7; +} + +// Provider type profile metadata exposed to clients. +message ProviderProfile { + string id = 1; + string display_name = 2; + string description = 3; + ProviderProfileCategory category = 4; + repeated ProviderProfileCredential credentials = 5; + repeated openshell.sandbox.v1.NetworkEndpoint endpoints = 6; + repeated openshell.sandbox.v1.NetworkBinary binaries = 7; + bool inference_capable = 8; + ProviderProfileDiscovery discovery = 9; +} + +// Stored custom provider profile object. +message StoredProviderProfile { + openshell.datamodel.v1.ObjectMeta metadata = 1; + ProviderProfile profile = 2; +} + +// Provider profile response. +message ProviderProfileResponse { + ProviderProfile profile = 1; +} + +// List provider profiles response. +message ListProviderProfilesResponse { + repeated ProviderProfile profiles = 1; +} + +// Import custom provider profiles request. +message ImportProviderProfilesRequest { + repeated ProviderProfileImportItem profiles = 1; +} + +// Import custom provider profiles response. +message ImportProviderProfilesResponse { + repeated ProviderProfileDiagnostic diagnostics = 1; + repeated ProviderProfile profiles = 2; + bool imported = 3; +} + +// Lint provider profiles request. +message LintProviderProfilesRequest { + repeated ProviderProfileImportItem profiles = 1; +} + +// Lint provider profiles response. +message LintProviderProfilesResponse { + repeated ProviderProfileDiagnostic diagnostics = 1; + bool valid = 2; +} + +// Delete provider response. +message DeleteProviderResponse { + bool deleted = 1; +} + +// Delete custom provider profile request. +message DeleteProviderProfileRequest { + string id = 1; +} + +// Delete custom provider profile response. +message DeleteProviderProfileResponse { + bool deleted = 1; +} + +// Get sandbox provider environment request. +message GetSandboxProviderEnvironmentRequest { + // The sandbox ID. + string sandbox_id = 1; +} + +// Get sandbox provider environment response. +message GetSandboxProviderEnvironmentResponse { + // Provider credential environment variables. + map environment = 1; + // Fingerprint for the provider credential inputs that produced environment. + uint64 provider_env_revision = 2; + // Expiration timestamps for returned environment variables. + map credential_expires_at_ms = 3; +} + +// --------------------------------------------------------------------------- +// Policy update messages +// --------------------------------------------------------------------------- + +// Update sandbox policy request. +message UpdateConfigRequest { + // Sandbox name (canonical lookup key). Required for sandbox-scoped updates. + // Not required when `global=true`. + string name = 1; + // The new policy to apply. + // + // Sandbox scope (`global=false`): + // - only network_policies and inference fields may differ from create-time + // policy; static fields must match version 1. + // + // Global scope (`global=true`): + // - applies to all sandboxes in full (no merge). + openshell.sandbox.v1.SandboxPolicy policy = 2; + // Optional single setting key to mutate. + string setting_key = 3; + // Setting value for upsert operations. + openshell.sandbox.v1.SettingValue setting_value = 4; + // Delete the setting key from scope. + // Sandbox-scoped deletes are rejected; only global delete is supported. + bool delete_setting = 5; + // Apply mutation at gateway-global scope. + bool global = 6; + // Batched incremental policy merge operations. Sandbox-scoped only. + repeated PolicyMergeOperation merge_operations = 7; + // Expected resource version for optimistic concurrency control (sandbox-scoped only). + // If 0, the server uses the current version (backward compatibility). + // If non-zero, the server validates that the sandbox's current resource_version + // matches this value before applying the mutation, returning ABORTED on mismatch. + // Ignored for global-scoped updates. + uint64 expected_resource_version = 8; +} + +message PolicyMergeOperation { + oneof operation { + AddNetworkRule add_rule = 1; + RemoveNetworkEndpoint remove_endpoint = 2; + RemoveNetworkRule remove_rule = 3; + AddDenyRules add_deny_rules = 4; + AddAllowRules add_allow_rules = 5; + RemoveNetworkBinary remove_binary = 6; + } +} + +message AddNetworkRule { + string rule_name = 1; + openshell.sandbox.v1.NetworkPolicyRule rule = 2; +} + +message RemoveNetworkEndpoint { + string rule_name = 1; + string host = 2; + uint32 port = 3; +} + +message RemoveNetworkRule { + string rule_name = 1; +} + +message AddDenyRules { + string host = 1; + uint32 port = 2; + repeated openshell.sandbox.v1.L7DenyRule deny_rules = 3; +} + +message AddAllowRules { + string host = 1; + uint32 port = 2; + repeated openshell.sandbox.v1.L7Rule rules = 3; +} + +message RemoveNetworkBinary { + string rule_name = 1; + string binary_path = 2; +} + +// Update sandbox policy response. +message UpdateConfigResponse { + // Assigned policy version (monotonically increasing per sandbox). + uint32 version = 1; + // SHA-256 hash of the serialized policy payload. + string policy_hash = 2; + // Settings revision for the scope that was modified. + uint64 settings_revision = 3; + // True when a setting delete operation removed an existing key. + bool deleted = 4; +} + +// Get sandbox policy status request. +message GetSandboxPolicyStatusRequest { + // Sandbox name (canonical lookup key). Ignored when global is true. + string name = 1; + // The specific policy version to query. 0 means latest. + uint32 version = 2; + // Query global policy revisions instead of a sandbox-scoped one. + bool global = 3; +} + +// Get sandbox policy status response. +message GetSandboxPolicyStatusResponse { + // The queried policy revision. + SandboxPolicyRevision revision = 1; + // The currently active (loaded) policy version for this sandbox. + uint32 active_version = 2; +} + +// List sandbox policies request. +message ListSandboxPoliciesRequest { + // Sandbox name (canonical lookup key). Ignored when global is true. + string name = 1; + uint32 limit = 2; + uint32 offset = 3; + // List global policy revisions instead of sandbox-scoped ones. + bool global = 4; +} + +// List sandbox policies response. +message ListSandboxPoliciesResponse { + repeated SandboxPolicyRevision revisions = 1; +} + +// Report policy load status (called by sandbox runtime after reload attempt). +message ReportPolicyStatusRequest { + // Sandbox id. + string sandbox_id = 1; + // The policy version that was attempted. + uint32 version = 2; + // Load result status. + PolicyStatus status = 3; + // Error message if status is FAILED. + string load_error = 4; +} + +// Report policy status response. +message ReportPolicyStatusResponse {} + +// A versioned policy revision with metadata. +message SandboxPolicyRevision { + // Policy version (monotonically increasing per sandbox). + uint32 version = 1; + // SHA-256 hash of the serialized policy payload. + string policy_hash = 2; + // Load status of this revision. + PolicyStatus status = 3; + // Error message if status is FAILED. + string load_error = 4; + // Milliseconds since epoch when this revision was created. + int64 created_at_ms = 5; + // Milliseconds since epoch when this revision was loaded by the sandbox. + int64 loaded_at_ms = 6; + // The full policy (only populated when explicitly requested). + openshell.sandbox.v1.SandboxPolicy policy = 7; +} + +// Policy load status. +enum PolicyStatus { + POLICY_STATUS_UNSPECIFIED = 0; + // Server received the update; sandbox has not yet loaded it. + POLICY_STATUS_PENDING = 1; + // Sandbox successfully applied this policy version. + POLICY_STATUS_LOADED = 2; + // Sandbox attempted to apply but failed; LKG policy remains active. + POLICY_STATUS_FAILED = 3; + // A newer version was persisted before the sandbox loaded this one. + POLICY_STATUS_SUPERSEDED = 4; +} + +// --------------------------------------------------------------------------- +// Sandbox logs messages +// --------------------------------------------------------------------------- + +// Get sandbox logs request (one-shot fetch). +message GetSandboxLogsRequest { + // Sandbox id. + string sandbox_id = 1; + // Maximum number of log lines to return. 0 means use default (2000). + uint32 lines = 2; + // Only include logs with timestamp >= this value (ms since epoch). 0 means no filter. + int64 since_ms = 3; + // Filter by log source (e.g. "gateway", "sandbox"). Empty means all sources. + repeated string sources = 4; + // Minimum log level to include (e.g. "INFO", "WARN", "ERROR"). Empty means all levels. + string min_level = 5; +} + +// Batch of log lines pushed from sandbox to server. +message PushSandboxLogsRequest { + // The sandbox ID. + string sandbox_id = 1; + // Log lines to ingest. + repeated SandboxLogLine logs = 2; +} + +// Push sandbox logs response. +message PushSandboxLogsResponse {} + +// Get sandbox logs response. +message GetSandboxLogsResponse { + // Log lines in chronological order. + repeated SandboxLogLine logs = 1; + // Total number of lines in the server's buffer for this sandbox. + uint32 buffer_total = 2; +} + +// --------------------------------------------------------------------------- +// Supervisor session messages +// --------------------------------------------------------------------------- + +// Envelope for supervisor-to-gateway messages on the ConnectSupervisor stream. +message SupervisorMessage { + oneof payload { + SupervisorHello hello = 1; + SupervisorHeartbeat heartbeat = 2; + RelayOpenResult relay_open_result = 3; + RelayClose relay_close = 4; + } +} + +// Envelope for gateway-to-supervisor messages on the ConnectSupervisor stream. +message GatewayMessage { + oneof payload { + SessionAccepted session_accepted = 1; + SessionRejected session_rejected = 2; + GatewayHeartbeat heartbeat = 3; + RelayOpen relay_open = 4; + RelayClose relay_close = 5; + } +} + +// Supervisor identifies itself and the sandbox it manages. +message SupervisorHello { + // Sandbox ID this supervisor manages. + string sandbox_id = 1; + // Supervisor instance ID (e.g. boot id or process epoch). + string instance_id = 2; +} + +// Gateway accepts the supervisor session. +message SessionAccepted { + // Gateway-assigned session ID for this connection. + string session_id = 1; + // Recommended heartbeat interval in seconds. + uint32 heartbeat_interval_secs = 2; +} + +// Gateway rejects the supervisor session. +message SessionRejected { + // Human-readable rejection reason. + string reason = 1; +} + +// Supervisor heartbeat. +message SupervisorHeartbeat {} + +// Gateway heartbeat. +message GatewayHeartbeat {} + +// Gateway requests the supervisor to open a relay channel. +// +// On receiving this, the supervisor should initiate a RelayStream RPC to +// the gateway, sending a RelayInit in the first RelayFrame to associate +// the new HTTP/2 stream with the pending relay slot. The supervisor +// bridges that stream to the requested local target. +message RelayOpen { + // Gateway-allocated channel identifier (UUID). + string channel_id = 1; + // Target the supervisor should dial inside the sandbox. + // If absent, supervisors treat the relay as SSH for compatibility. + oneof target { + SshRelayTarget ssh = 2; + TcpRelayTarget tcp = 3; + } + // Optional service identifier for audit/correlation. + string service_id = 5; +} + +// Built-in SSH relay target. +message SshRelayTarget {} + +// TCP target dialed by the supervisor from inside the sandbox. +message TcpRelayTarget { + // Phase 1 accepts loopback only: 127.0.0.1, ::1, or localhost. + string host = 1; + // Target port. Must fit in u16 and be non-zero. + uint32 port = 2; +} + +// Initial RelayStream frame sent by the supervisor to claim a pending relay. +message RelayInit { + // Gateway-allocated channel identifier (UUID). + string channel_id = 1; +} + +// A single frame on the RelayStream RPC. +// +// The supervisor MUST send `init` as the first frame. All subsequent frames +// in either direction carry raw bytes in `data`. +message RelayFrame { + oneof payload { + RelayInit init = 1; + bytes data = 2; + } +} + +// Supervisor reports the result of a relay open request. +message RelayOpenResult { + // Channel identifier from the RelayOpen request. + string channel_id = 1; + // True if the relay was successfully established. + bool success = 2; + // Error message if success is false. + string error = 3; +} + +// Either side requests closure of a relay channel. +message RelayClose { + // Channel identifier to close. + string channel_id = 1; + // Optional reason for closure. + string reason = 2; +} + +// --------------------------------------------------------------------------- +// Service status +// --------------------------------------------------------------------------- + +// Service status enum. +enum ServiceStatus { + SERVICE_STATUS_UNSPECIFIED = 0; + SERVICE_STATUS_HEALTHY = 1; + SERVICE_STATUS_DEGRADED = 2; + SERVICE_STATUS_UNHEALTHY = 3; +} + +// --------------------------------------------------------------------------- +// Draft policy recommendation messages +// --------------------------------------------------------------------------- + +// Observed HTTP method+path pattern from L7 inspection. +message L7RequestSample { + // HTTP method: GET, POST, PUT, DELETE, etc. + string method = 1; + // HTTP path: /v1/models, /repos/myorg/issues + string path = 2; + // L7 decision: "audit" or "deny" (allowed requests not collected). + string decision = 3; + // Number of times this (method, path) was observed. + uint32 count = 4; +} + +// Structured denial summary from sandbox aggregator. +message DenialSummary { + // Sandbox ID that produced this summary. + string sandbox_id = 1; + // Denied destination host. + string host = 2; + // Denied destination port. + uint32 port = 3; + // Binary that attempted the connection. + string binary = 4; + // Process ancestor chain. + repeated string ancestors = 5; + // Denial reason from OPA evaluation. + string deny_reason = 6; + // First denial timestamp (ms since epoch). + int64 first_seen_ms = 7; + // Most recent denial timestamp (ms since epoch). + int64 last_seen_ms = 8; + // Number of denials in the current window. + uint32 count = 9; + // Events dropped during aggregator cooldown. + uint32 suppressed_count = 10; + // Cumulative lifetime count (never resets). + uint32 total_count = 11; + // Distinct cmdline strings observed (sanitized of credentials). + repeated string sample_cmdlines = 12; + // SHA-256 of the binary for audit trail. + string binary_sha256 = 13; + // True if emitted by stale-flush rather than threshold. + bool persistent = 14; + // Denial category: "l4_deny", "l7_deny", "l7_audit", "ssrf". + string denial_stage = 15; + // Observed HTTP request patterns (from L7 inspection). + repeated L7RequestSample l7_request_samples = 16; + // True if L7 inspection was active during observation window. + bool l7_inspection_active = 17; +} + +// A proposed policy rule with rationale and approval status. +message PolicyChunk { + // Unique chunk identifier. + string id = 1; + // Approval status: "pending", "approved", "rejected". + string status = 2; + // Proposed network_policies map key. + string rule_name = 3; + // The proposed network policy rule. + openshell.sandbox.v1.NetworkPolicyRule proposed_rule = 4; + // Human-readable explanation of why this rule is proposed. + string rationale = 5; + // Security concerns flagged by analysis (empty if none). + string security_notes = 6; + // Analysis confidence (0.0-1.0). 0 for mechanistic mode. + float confidence = 7; + // IDs of denial summaries that led to this chunk. + repeated string denial_summary_ids = 8; + // Creation timestamp (ms since epoch). + int64 created_at_ms = 9; + // When the user approved/rejected (ms since epoch). 0 if undecided. + int64 decided_at_ms = 10; + // Recommendation stage: "initial" or "refined" (progressive L7 visibility). + string stage = 11; + // For stage="refined": the initial chunk this replaces. + string supersedes_chunk_id = 12; + // How many times this endpoint has been seen across denial flush cycles. + int32 hit_count = 13; + // First time this endpoint was proposed (ms since epoch). + int64 first_seen_ms = 14; + // Most recent time this endpoint was re-proposed (ms since epoch). + int64 last_seen_ms = 15; + // Binary path that triggered the denial (denormalized for display convenience). + string binary = 16; + // Validation verdict from gateway-side static checks (prover output). + // Free-form summary string for human consumption in the inbox card. + // Empty until the prover has run for this chunk. + string validation_result = 17; + // Operator-supplied free-form text accompanying a rejection. Populated + // when the reviewer rejects via `RejectDraftChunkRequest.reason`; surfaced + // back to the in-sandbox agent so it can revise the proposal. + // Empty for non-rejected chunks. + string rejection_reason = 18; +} + +// Notification that the draft policy was updated. +message DraftPolicyUpdate { + // Current draft version. + uint64 draft_version = 1; + // Number of new chunks added in this update. + uint32 new_chunks = 2; + // Total pending chunks awaiting approval. + uint32 total_pending = 3; + // Brief description of what changed. + string summary = 4; +} + +// Submit analysis results from sandbox to gateway. +message SubmitPolicyAnalysisRequest { + // Aggregated denial summaries. + repeated DenialSummary summaries = 1; + // Proposed policy chunks (validated by sandbox OPA engine). + repeated PolicyChunk proposed_chunks = 2; + // Analysis mode. `mechanistic` is the observation-driven path from the + // denial aggregator — chunks targeting the same host|port|binary fold + // into one row with hit_count incremented. `agent_authored` is an + // intentional proposal from an in-sandbox agent — each submission lands + // as its own chunk so the redraft-after-rejection loop has a stable id + // to watch. Other values are treated as agent-style (no dedup) so a new + // mode does not silently collapse proposals. + string analysis_mode = 3; + // Sandbox name. + string name = 4; +} + +message SubmitPolicyAnalysisResponse { + // Number of chunks accepted by the gateway. + uint32 accepted_chunks = 1; + // Number of chunks rejected by gateway validation. + uint32 rejected_chunks = 2; + // Reasons for each rejected chunk. + repeated string rejection_reasons = 3; + // Server-assigned chunk IDs for the accepted chunks, in submission order. + // Agents use these to watch proposal state via policy.local's + // GET /v1/proposals/{id} and /wait endpoints. + repeated string accepted_chunk_ids = 4; +} + +// Get draft policy for a sandbox. +message GetDraftPolicyRequest { + // Sandbox name. + string name = 1; + // Optional status filter: "pending", "approved", "rejected", or "" for all. + string status_filter = 2; +} + +message GetDraftPolicyResponse { + // Draft policy chunks. + repeated PolicyChunk chunks = 1; + // LLM-generated summary of all analysis (empty in mechanistic mode). + string rolling_summary = 2; + // Current draft version. + uint64 draft_version = 3; + // When the last analysis completed (ms since epoch). + int64 last_analyzed_at_ms = 4; +} + +// Approve a single draft chunk. +message ApproveDraftChunkRequest { + // Sandbox name. + string name = 1; + // Chunk ID to approve. + string chunk_id = 2; +} + +message ApproveDraftChunkResponse { + // New policy version after merge. + uint32 policy_version = 1; + // SHA-256 hash of the new policy. + string policy_hash = 2; +} + +// Reject a single draft chunk. +message RejectDraftChunkRequest { + // Sandbox name. + string name = 1; + // Chunk ID to reject. + string chunk_id = 2; + // Optional reason for rejection (fed to LLM context in future analysis). + string reason = 3; +} + +message RejectDraftChunkResponse {} + +// Approve all pending chunks. +message ApproveAllDraftChunksRequest { + // Sandbox name. + string name = 1; + // Include chunks with security_notes (default false: skips them). + bool include_security_flagged = 2; +} + +message ApproveAllDraftChunksResponse { + // New policy version after merge. + uint32 policy_version = 1; + // SHA-256 hash of the new policy. + string policy_hash = 2; + // Number of chunks approved. + uint32 chunks_approved = 3; + // Number of chunks skipped (security-flagged). + uint32 chunks_skipped = 4; +} + +// Edit a pending chunk in-place. +message EditDraftChunkRequest { + // Sandbox name. + string name = 1; + // Chunk ID to edit. + string chunk_id = 2; + // The modified rule (replaces existing proposed_rule). + openshell.sandbox.v1.NetworkPolicyRule proposed_rule = 3; +} + +message EditDraftChunkResponse {} + +// Reverse an approval (remove merged rule from active policy). +message UndoDraftChunkRequest { + // Sandbox name. + string name = 1; + // Chunk ID to undo. + string chunk_id = 2; +} + +message UndoDraftChunkResponse { + // New policy version after removal. + uint32 policy_version = 1; + // SHA-256 hash of the updated policy. + string policy_hash = 2; +} + +// Clear all pending draft chunks for a sandbox. +message ClearDraftChunksRequest { + // Sandbox name. + string name = 1; +} + +message ClearDraftChunksResponse { + // Number of chunks cleared. + uint32 chunks_cleared = 1; +} + +// Get decision history for a sandbox's draft policy. +message GetDraftHistoryRequest { + // Sandbox name. + string name = 1; +} + +message DraftHistoryEntry { + // Event timestamp (ms since epoch). + int64 timestamp_ms = 1; + // Event type: "denial_detected", "analysis_cycle", "approved", + // "rejected", "edited", "undone", "cleared". + string event_type = 2; + // Human-readable description. + string description = 3; + // Associated chunk ID (if applicable). + string chunk_id = 4; +} + +message GetDraftHistoryResponse { + // Chronological decision history. + repeated DraftHistoryEntry entries = 1; +} + +// Stored payload for a policy revision row in the generic objects table. +message PolicyRevisionPayload { + // Serialized policy contents. + openshell.sandbox.v1.SandboxPolicy policy = 1; + // Deterministic hash of the policy payload. + string hash = 2; + // Load error reported by the sandbox, if any. + string load_error = 3; + // When the policy version was reported as loaded (ms since epoch). 0 if unset. + int64 loaded_at_ms = 4; +} + +// Stored payload for a draft policy chunk row in the generic objects table. +message DraftChunkPayload { + // Proposed network_policies map key. + string rule_name = 1; + // Proposed network policy rule. + openshell.sandbox.v1.NetworkPolicyRule proposed_rule = 2; + // Human-readable explanation of why this rule is proposed. + string rationale = 3; + // Security concerns flagged by analysis (empty if none). + string security_notes = 4; + // Analysis confidence (0.0-1.0). 0 for mechanistic mode. + float confidence = 5; + // When the user approved/rejected (ms since epoch). 0 if undecided. + int64 decided_at_ms = 6; + // Denormalized endpoint host for dedup and display. + string host = 7; + // Denormalized endpoint port for dedup and display. + int32 port = 8; + // Binary path that triggered the denial. + string binary = 9; + // Current draft version for the owning sandbox. + int64 draft_version = 10; + // Gateway prover verdict for this chunk; empty until prover runs. + // Mirrors PolicyChunk.validation_result. + string validation_result = 11; + // Operator-supplied free-form rejection text; empty for non-rejected + // chunks. Mirrors PolicyChunk.rejection_reason. + string rejection_reason = 12; +} + +// Internal stored policy revision row materialized from the generic objects table. +message StoredPolicyRevision { + string id = 1; + string sandbox_id = 2; + int64 version = 3; + bytes policy_payload = 4; + string policy_hash = 5; + string status = 6; + optional string load_error = 7; + int64 created_at_ms = 8; + optional int64 loaded_at_ms = 9; +} + +// Internal stored draft chunk row materialized from the generic objects table. +message StoredDraftChunk { + string id = 1; + string sandbox_id = 2; + int64 draft_version = 3; + string status = 4; + string rule_name = 5; + bytes proposed_rule = 6; + string rationale = 7; + string security_notes = 8; + double confidence = 9; + int64 created_at_ms = 10; + optional int64 decided_at_ms = 11; + string host = 12; + int32 port = 13; + string binary = 14; + int32 hit_count = 15; + int64 first_seen_ms = 16; + int64 last_seen_ms = 17; + // Gateway prover verdict; empty until the prover runs. See PolicyChunk. + string validation_result = 18; + // Operator-supplied free-form rejection text. See PolicyChunk. + string rejection_reason = 19; +} diff --git a/src/lib/adapters/openshell/proto/sandbox.proto b/src/lib/adapters/openshell/proto/sandbox.proto new file mode 100644 index 0000000000..b40d95cb12 --- /dev/null +++ b/src/lib/adapters/openshell/proto/sandbox.proto @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.sandbox.v1; + +// Sandbox-supervisor configuration and policy messages. +// +// Conventions: +// - This file owns messages exchanged between the gateway and the sandbox +// supervisor/runtime. +// - Public sandbox resource types live in `openshell.proto`. +// - Internal compute-driver sandbox observation types live in `compute_driver.proto`. + +// Sandbox security policy configuration. +message SandboxPolicy { + // Policy version. + uint32 version = 1; + // Filesystem access policy. + FilesystemPolicy filesystem = 2; + // Landlock configuration. + LandlockPolicy landlock = 3; + // Process execution policy. + ProcessPolicy process = 4; + // Network access policies keyed by name (e.g. "claude_code", "gitlab"). + map network_policies = 5; +} + +// Filesystem access policy. +message FilesystemPolicy { + // Automatically include the workdir as read-write. + bool include_workdir = 1; + // Read-only directory allow list. + repeated string read_only = 2; + // Read-write directory allow list. + repeated string read_write = 3; +} + +// Landlock policy configuration. +message LandlockPolicy { + // Compatibility mode (e.g. "best_effort", "hard_requirement"). + string compatibility = 1; +} + +// Process execution policy. +message ProcessPolicy { + // User name to run the sandboxed process as. + string run_as_user = 1; + // Group name to run the sandboxed process as. + string run_as_group = 2; +} + +// A named network access policy rule. +message NetworkPolicyRule { + // Human-readable name for this policy rule. + string name = 1; + // Allowed endpoint (host:port) pairs. + repeated NetworkEndpoint endpoints = 2; + // Allowed binary identities. + repeated NetworkBinary binaries = 3; +} + +// A network endpoint (host + port) with optional L7 inspection config. +message NetworkEndpoint { + // Hostname or host glob pattern. Exact match is case-insensitive. + // Glob patterns use "." as delimiter: "*.example.com" matches a single + // subdomain label, "**.example.com" matches across labels. + string host = 1; + // Single port (backwards compat). Use `ports` for multiple ports. + // Mutually exclusive with `ports` — if both are set, `ports` takes precedence. + uint32 port = 2; + // Application protocol for L7 inspection: "rest", "websocket", "graphql", "sql", or "" (L4-only). + string protocol = 3; + // TLS handling: "terminate" or "passthrough" (default). + string tls = 4; + // Enforcement mode: "enforce" or "audit" (default). + string enforcement = 5; + // Access preset shorthand: "read-only", "read-write", "full". + // Mutually exclusive with rules. + string access = 6; + // Explicit L7 rules (mutually exclusive with access). + repeated L7Rule rules = 7; + // Allowed resolved IP addresses or CIDR ranges for this endpoint. + // When non-empty, the SSRF internal-IP check is replaced by an allowlist check: + // - If host is also set: domain must resolve to an IP in this list. + // - If host is empty: any domain is allowed as long as it resolves to an IP in this list. + // Supports exact IPs ("10.0.5.20") and CIDR notation ("10.0.5.0/24"). + // Loopback (127.0.0.0/8) and link-local (169.254.0.0/16) are always blocked + // regardless of this field. + repeated string allowed_ips = 8; + // Multiple ports. When non-empty, this endpoint covers all listed ports. + // If `port` is set and `ports` is empty, `port` is normalized to `ports: [port]`. + // If both are set, `ports` takes precedence. + repeated uint32 ports = 9; + // Explicit L7 deny rules. When present, requests matching any deny rule + // are blocked even if they match an allow rule or access preset. + // Deny rules take precedence over allow rules. + repeated L7DenyRule deny_rules = 10; + // When true, percent-encoded '/' (%2F) is preserved in path segments + // rather than rejected by the L7 path canonicalizer. Required for + // upstreams like GitLab that embed %2F in namespaced resource paths. + // Defaults to false (strict). + bool allow_encoded_slash = 11; + // GraphQL persisted-query behavior for hash-only/saved-query requests: + // "deny" (default) or "allow_registered". + string persisted_queries = 12; + // Trusted GraphQL persisted-query registry keyed by hash or service-specific ID. + // Only used when persisted_queries is "allow_registered". + map graphql_persisted_queries = 13; + // Maximum GraphQL request body bytes to buffer for inspection. + // Defaults to 65536 when unset. + uint32 graphql_max_body_bytes = 14; + // Optional HTTP path glob that scopes this L7 endpoint on shared host:port APIs. + // Example: use path "/graphql" for protocol "graphql" and "/repos/**" for + // protocol "rest" when both surfaces live under api.example.com:443. + // Empty means all paths. + string path = 15; + // When true on a "rest" endpoint, OpenShell rewrites credential placeholders + // inside client-to-server WebSocket text messages after an allowed HTTP 101 + // upgrade. Defaults to false. + bool websocket_credential_rewrite = 16; + // When true on a "rest" endpoint, OpenShell rewrites credential placeholders + // inside supported textual HTTP request bodies before forwarding upstream. + // Defaults to false. + bool request_body_credential_rewrite = 17; +} + +// Trusted GraphQL operation classification. +message GraphqlOperation { + // Operation type: "query", "mutation", or "subscription". + string operation_type = 1; + // Operation name, if known. + string operation_name = 2; + // Root field names selected by the operation. + repeated string fields = 3; +} + +// An L7 deny rule that blocks specific requests. +// Mirrors L7Allow — same fields, same matching semantics, inverted effect. +// Deny rules are evaluated after allow rules and take precedence. +message L7DenyRule { + // HTTP method (REST): GET, POST, etc. or "*" for any. + string method = 1; + // URL path glob pattern (REST): "/repos/*/pulls/*/reviews", "**" for any. + string path = 2; + // SQL command (SQL): SELECT, INSERT, etc. or "*" for any. + string command = 3; + // Query parameter matcher map (REST). + // Same semantics as L7Allow.query. + map query = 4; + // GraphQL operation type: "query", "mutation", "subscription", or "*" for any. + string operation_type = 5; + // GraphQL operation name glob. "*" matches any operation name. + string operation_name = 6; + // GraphQL root field globs. Deny rules match when any selected root field + // matches any configured glob. + repeated string fields = 7; +} + +// An L7 policy rule (allow-only). +message L7Rule { + L7Allow allow = 1; +} + +// Allowed action definition for L7 rules. +message L7Allow { + // HTTP method (REST): GET, POST, etc. or "*" for any. + string method = 1; + // URL path glob pattern (REST): "/repos/**", "**" for any. + string path = 2; + // SQL command (SQL): SELECT, INSERT, etc. or "*" for any. + string command = 3; + // Query parameter matcher map (REST). + // Key is the decoded query parameter name (case-sensitive). + // Value supports either a single glob (`glob`) or a list (`any`). + map query = 4; + // GraphQL operation type: "query", "mutation", "subscription", or "*" for any. + string operation_type = 5; + // GraphQL operation name glob. "*" matches any operation name. + string operation_name = 6; + // GraphQL root field globs. Allow rules match only when every selected root + // field matches one of the configured globs. Omit to match all fields. + repeated string fields = 7; +} + +// Query value matcher for one query parameter key. +message L7QueryMatcher { + // Single glob pattern. + string glob = 1; + // Any-of glob patterns. + repeated string any = 2; +} + +// A binary identity for network policy matching. +message NetworkBinary { + string path = 1; + // Deprecated: the harness concept has been removed. This field is ignored. + bool harness = 2 [deprecated = true]; +} + +// Request to get sandbox settings by sandbox ID. +message GetSandboxConfigRequest { + // The sandbox ID. + string sandbox_id = 1; +} + +// Request to get gateway-global settings. +message GetGatewayConfigRequest {} + +// Response containing gateway-global settings. +message GetGatewayConfigResponse { + // Gateway-global settings map excluding the reserved policy key. + // Registered keys without a configured value are returned with an empty SettingValue. + map settings = 1; + // Monotonically increasing revision for gateway-global settings. + uint64 settings_revision = 2; +} + +// Scope that currently controls a setting. +enum SettingScope { + SETTING_SCOPE_UNSPECIFIED = 0; + SETTING_SCOPE_SANDBOX = 1; + SETTING_SCOPE_GLOBAL = 2; +} + +// Type-aware setting value for sandbox/gateway settings. +message SettingValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + bytes bytes_value = 4; + } +} + +// Effective setting value and the scope it was resolved from. +message EffectiveSetting { + SettingValue value = 1; + SettingScope scope = 2; +} + +// Source used for the policy payload in GetSandboxConfigResponse. +enum PolicySource { + POLICY_SOURCE_UNSPECIFIED = 0; + POLICY_SOURCE_SANDBOX = 1; + POLICY_SOURCE_GLOBAL = 2; +} + +// Response containing effective sandbox settings and policy. +message GetSandboxConfigResponse { + // The sandbox policy configuration. + SandboxPolicy policy = 1; + // Current policy version (monotonically increasing per sandbox). + uint32 version = 2; + // SHA-256 hash of the serialized policy payload. + string policy_hash = 3; + // Effective settings resolved for this sandbox, excluding the reserved policy key. + // Registered keys without a configured value are returned with an empty EffectiveSetting.value. + map settings = 4; + // Fingerprint for effective config (policy + settings). Changes when any effective input changes. + uint64 config_revision = 5; + // Source of the policy payload for this response. + PolicySource policy_source = 6; + // When policy_source is GLOBAL, the version of the global policy revision. + // Zero when no global policy is active or when policy_source is SANDBOX. + uint32 global_policy_version = 7; + // Fingerprint for provider credential inputs attached to this sandbox. + // Changes when attached provider names or attached provider records change. + uint64 provider_env_revision = 8; +} From 2a94b6dee669617984e96ad688cc6c79bddadfea Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 31 May 2026 14:38:42 -0700 Subject: [PATCH 24/26] fix: harden nightly e2e for grpc sandbox ops Signed-off-by: Aaron Erickson --- scripts/nemoclaw-start.sh | 25 +++++++- test/e2e/test-hermes-e2e.sh | 60 ++++++++++++++++--- .../test-issue-4462-scope-upgrade-approval.sh | 12 ++++ test/nemoclaw-start.test.ts | 31 ++++++++++ 4 files changed, 118 insertions(+), 10 deletions(-) diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 7a99f7e00c..bf485ee099 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -1869,7 +1869,30 @@ openclaw() { # upgrades through the gateway without requesting the upgraded scopes for # the approval command itself. Other commands keep OPENCLAW_GATEWAY_URL. if [ "${1:-}" = "devices" ] && [ "${2:-}" = "approve" ]; then - ( unset OPENCLAW_GATEWAY_URL; command openclaw "$@" ) + ( + _nemoclaw_approve_request_id="${3:-}" + unset OPENCLAW_GATEWAY_URL + _nemoclaw_approve_output="$(command openclaw "$@" 2>&1)" + _nemoclaw_approve_rc=$? + if [ "$_nemoclaw_approve_rc" -ne 0 ] && [ -n "$_nemoclaw_approve_request_id" ]; then + _nemoclaw_replacement_request_id="$( + printf '%s\n' "$_nemoclaw_approve_output" \ + | sed -nE 's/.*scope upgrade pending approval \(requestId: ([A-Za-z0-9_-]+)\).*/\1/p' \ + | tail -1 + )" + if [ -n "$_nemoclaw_replacement_request_id" ] \ + && [ "$_nemoclaw_replacement_request_id" != "$_nemoclaw_approve_request_id" ] \ + && printf '%s\n' "$_nemoclaw_approve_output" | grep -Fq "unknown requestId"; then + shift 3 + command openclaw devices approve "$_nemoclaw_replacement_request_id" "$@" + exit $? + fi + fi + if [ -n "$_nemoclaw_approve_output" ]; then + printf '%s\n' "$_nemoclaw_approve_output" + fi + exit "$_nemoclaw_approve_rc" + ) return $? fi case "$1" in diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index c655c9a0e1..a6d37f4bb7 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -147,6 +147,44 @@ sys.exit(1) PY } +nemoclaw_forward_state_list() { + python3 <<'PY' +import json +import os + +state_dir = os.path.join(os.path.expanduser("~"), ".nemoclaw", "forwards") +print("SANDBOX BIND PORT PID STATUS") +try: + names = sorted(os.listdir(state_dir)) +except OSError: + raise SystemExit(0) + +for name in names: + if not name.endswith(".json"): + continue + try: + with open(os.path.join(state_dir, name), encoding="utf-8") as fh: + state = json.load(fh) + except Exception: + continue + sandbox = state.get("sandboxName") + bind = state.get("bind") + port = state.get("port") + pid = state.get("pid") + if not isinstance(sandbox, str) or not isinstance(bind, str): + continue + if not isinstance(port, int) or not isinstance(pid, int) or pid <= 0: + continue + try: + os.kill(pid, 0) + except ProcessLookupError: + continue + except PermissionError: + pass + print(f"{sandbox} {bind} {port} {pid} running") +PY +} + # Determine repo root if [ -d /workspace ] && [ -f /workspace/install.sh ]; then REPO="/workspace" @@ -534,18 +572,22 @@ PY fail "Registry did not record Hermes dashboard metadata: ${registry_check:0:240}" fi - forward_list=$(openshell forward list 2>&1 || true) - if forward_list_has_running_port "$SANDBOX_NAME" "8642" "$forward_list"; then - pass "OpenShell forward list shows Hermes API port 8642 running" + forward_state_list=$(nemoclaw_forward_state_list 2>&1 || true) + if forward_list_has_running_port "$SANDBOX_NAME" "8642" "$forward_state_list"; then + pass "NemoClaw forward state shows Hermes API port 8642 running" else - fail "OpenShell forward list does not show Hermes API port 8642 running" - info "forward list: ${forward_list:0:300}" + fail "NemoClaw forward state does not show Hermes API port 8642 running" + info "NemoClaw forward state: ${forward_state_list:0:300}" + legacy_forward_list=$(openshell forward list 2>&1 || true) + info "OpenShell forward list: ${legacy_forward_list:0:300}" fi - if forward_list_has_running_port "$SANDBOX_NAME" "$HERMES_DASHBOARD_PORT" "$forward_list"; then - pass "OpenShell forward list shows Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" + if forward_list_has_running_port "$SANDBOX_NAME" "$HERMES_DASHBOARD_PORT" "$forward_state_list"; then + pass "NemoClaw forward state shows Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" else - fail "OpenShell forward list does not show Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" - info "forward list: ${forward_list:0:300}" + fail "NemoClaw forward state does not show Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" + info "NemoClaw forward state: ${forward_state_list:0:300}" + legacy_forward_list=$(openshell forward list 2>&1 || true) + info "OpenShell forward list: ${legacy_forward_list:0:300}" fi dashboard_body="$(mktemp)" diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 298be83096..c659f38e5f 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -432,6 +432,18 @@ exit "$approve_rc" fi approved_id=$(printf '%s' "$approve_json" | json_field requestId) if [ "$approved_id" != "$request_id" ]; then + if [ "$allow_already_approved" = "1" ] && [ -n "$approved_id" ]; then + state_after_approve="$(device_state_json 2>&1)" || state_after_approve="" + if [ -n "$state_after_approve" ]; then + printf '=== state after replacement approve %s request=%s approved=%s ===\n%s\n' "$label" "$request_id" "$approved_id" "$state_after_approve" >>"$STATE_LOG" + approved_after_approve=$(printf '%s' "$state_after_approve" | select_cli_paired_with_agent_scopes 2>/dev/null) || approved_after_approve="" + pending_after_approve=$(printf '%s' "$state_after_approve" | select_cli_request scope-upgrade 2>/dev/null) || pending_after_approve="" + if [ -n "$approved_after_approve" ] && [ -z "$pending_after_approve" ]; then + pass "${label}: openclaw devices approve ${request_id} --json succeeded via replacement request ${approved_id}" + return 0 + fi + fi + fi fail "${label}: approve returned requestId=${approved_id:-empty}, expected ${request_id}" return 1 fi diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index ddd682f718..6369f7e72f 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -926,6 +926,37 @@ describe("nemoclaw-start configure guard behavior", () => { } }); + it("#4462: retries replacement scope-upgrade request ids from devices approve", () => { + const setup = writeProxyEnvWithGuard(); + try { + fs.writeFileSync( + path.join(setup.fakeBin, "openclaw"), + `#!/usr/bin/env bash +printf 'ARGS=%s URL=%s\\n' "$*" "\${OPENCLAW_GATEWAY_URL-unset}" >> ${JSON.stringify(setup.commandLog)} +if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "approve" ] && [ "\${3:-}" = "request-1" ]; then + printf 'gateway connect failed: GatewayClientRequestError: scope upgrade pending approval (requestId: request-2)\\n' + printf 'unknown requestId\\n' + exit 1 +fi +printf '{"requestId":"%s"}\\n' "\${3:-}" +exit 0 +`, + { mode: 0o755 }, + ); + + const result = runGuardedOpenclaw(setup, ["devices", "approve", "request-1", "--json"]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('"requestId":"request-2"'); + expect(fs.readFileSync(setup.commandLog, "utf-8").trim().split("\n")).toEqual([ + "ARGS=devices approve request-1 --json URL=unset", + "ARGS=devices approve request-2 --json URL=unset", + ]); + } finally { + fs.rmSync(setup.tmpDir, { recursive: true, force: true }); + } + }); + // #2592 reported the guard did not fire for `openclaw channels add telegram` // and `openclaw channels remove telegram` from inside the sandbox. The // existing test above only exercises `add slack`. Lock in coverage for every From d6029961013fe1168dfef9bc6b0587a7ba6a6e6d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 31 May 2026 14:47:08 -0700 Subject: [PATCH 25/26] test: update scope approval guard probe Signed-off-by: Aaron Erickson --- test/e2e/test-issue-4462-scope-upgrade-approval.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index c659f38e5f..6003c571e3 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -754,7 +754,8 @@ fi . /tmp/nemoclaw-proxy-env.sh printf "OPENCLAW_GATEWAY_URL=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" type openclaw 2>/dev/null | sed -n "1,12p" -grep -F "unset OPENCLAW_GATEWAY_URL; command openclaw" /tmp/nemoclaw-proxy-env.sh >/dev/null \ +grep -F "unset OPENCLAW_GATEWAY_URL" /tmp/nemoclaw-proxy-env.sh >/dev/null \ + && grep -F "command openclaw devices approve" /tmp/nemoclaw-proxy-env.sh >/dev/null \ && echo "APPROVE_GUARD_PRESENT" ' 2>&1) guard_rc=$? From 1999ea448d34ae95764ffb993fd8e9b190f03d13 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 31 May 2026 15:32:11 -0700 Subject: [PATCH 26/26] fix: close forward diagnostic race Signed-off-by: Aaron Erickson --- .../openshell/forward-bridge-state.ts | 35 ++++++++++++------- src/lib/adapters/openshell/grpc.ts | 1 - 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/lib/adapters/openshell/forward-bridge-state.ts b/src/lib/adapters/openshell/forward-bridge-state.ts index aeefbdfac9..965dc98d70 100644 --- a/src/lib/adapters/openshell/forward-bridge-state.ts +++ b/src/lib/adapters/openshell/forward-bridge-state.ts @@ -36,10 +36,12 @@ function stateDir(): string { return path.join(process.env.HOME || os.homedir(), ".nemoclaw", "forwards"); } +function safeStateComponent(value: number | string): string { + return String(value).replace(/[^A-Za-z0-9._-]/g, "_"); +} + function statePath(sandboxName: string, port: number | string): string { - const safeSandbox = sandboxName.replace(/[^A-Za-z0-9._-]/g, "_"); - const safePort = String(port).replace(/[^A-Za-z0-9._-]/g, "_"); - return path.join(stateDir(), `${safeSandbox}-${safePort}.json`); + return path.join(stateDir(), `${safeStateComponent(sandboxName)}-${safeStateComponent(port)}.json`); } function ensureStateDir(): void { @@ -222,6 +224,19 @@ function probeForwardReady(bind: string, port: number): boolean { return result.status === 0; } +function readDiagnosticFromFd(fd: number): string { + try { + const stat = fs.fstatSync(fd); + const length = Math.min(stat.size, 64 * 1024); + if (length <= 0) return ""; + const buffer = Buffer.alloc(length); + const bytes = fs.readSync(fd, buffer, 0, length, Math.max(0, stat.size - length)); + return buffer.toString("utf-8", 0, bytes).trim(); + } catch { + return ""; + } +} + export function startForwardBridgeDetached( sandboxName: string, options: ForwardBridgeStartOptions, @@ -247,9 +262,9 @@ export function startForwardBridgeDetached( ensureStateDir(); const diagnosticPath = path.join( stateDir(), - `${sandboxName.replace(/[^A-Za-z0-9._-]/g, "_")}-${options.port}.log`, + `${safeStateComponent(sandboxName)}-${safeStateComponent(options.port)}.log`, ); - const out = fs.openSync(diagnosticPath, "w", 0o600); + const out = fs.openSync(diagnosticPath, "w+", 0o600); const child = spawn( command, [ @@ -265,23 +280,19 @@ export function startForwardBridgeDetached( { detached: true, stdio: ["ignore", out, out], env: process.env }, ); child.unref(); - fs.closeSync(out); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const state = getForwardState(sandboxName, options.port); if (state && state.pid === child.pid && probeForwardReady(state.bind, state.port)) { + fs.closeSync(out); return { ok: true, state, diagnostic: "" }; } sleepMs(250); } - let diagnostic = ""; - try { - diagnostic = fs.readFileSync(diagnosticPath, "utf-8").trim(); - } catch { - /* ignore */ - } + const diagnostic = readDiagnosticFromFd(out); + fs.closeSync(out); try { if (child.pid) process.kill(child.pid, "SIGTERM"); } catch { diff --git a/src/lib/adapters/openshell/grpc.ts b/src/lib/adapters/openshell/grpc.ts index 47a8285518..188424ee50 100644 --- a/src/lib/adapters/openshell/grpc.ts +++ b/src/lib/adapters/openshell/grpc.ts @@ -64,7 +64,6 @@ export interface SandboxGrpcClientOptions extends GatewayMetadataOptions { gateway?: ResolvedGatewayMetadata; } -const DEFAULT_SDK_TIMEOUT_MS = 30_000; const MAX_SYNC_STDIO_BUFFER = 512 * 1024 * 1024; type OpenShellClientConstructor = typeof import("@openshell/sdk").OpenShellClient;