From 989ce480189684b724c7887776938a1f8a6e981f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 4 Sep 2026 20:56:59 -0700 Subject: [PATCH 01/20] perf(ci): rebalance CLI coverage shards Signed-off-by: Prekshi Vyas --- .github/workflows/main.yaml | 10 +- .github/workflows/pr.yaml | 10 +- ci/cli-test-timing-hints.json | 387 +++--- .../pr-workflow-contract.test.ts | 8 +- test/helpers/cli-coverage-sequencer.ts | 6 +- .../onboard-fresh-create-identity-suite.ts | 1107 +++++++++++++++++ ...fresh-create-cancellation-identity.test.ts | 6 + .../onboard-fresh-create-identity.test.ts | 1080 ---------------- ...ard-fresh-create-provider-identity.test.ts | 6 + ...ard-fresh-create-recovery-identity.test.ts | 6 + .../repository/cli-coverage-sequencer.test.ts | 54 +- 11 files changed, 1361 insertions(+), 1319 deletions(-) create mode 100644 test/helpers/onboard-fresh-create-identity-suite.ts create mode 100644 test/onboarding/onboard-fresh-create-cancellation-identity.test.ts delete mode 100644 test/onboarding/onboard-fresh-create-identity.test.ts create mode 100644 test/onboarding/onboard-fresh-create-provider-identity.test.ts create mode 100644 test/onboarding/onboard-fresh-create-recovery-identity.test.ts diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0b0efb4215d..6a11089b155 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -193,7 +193,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -219,7 +219,7 @@ jobs: uses: ./.github/actions/ci-cli-coverage-shard with: shard: ${{ matrix.shard }} - shard-count: "12" + shard-count: "10" cli-tests: needs: [build-typecheck, cli-test-shards] @@ -258,10 +258,10 @@ jobs: else [.jobs[] | select((.name | type) == "string") | - select(.name | test("^cli-test-shards \\(([1-9]|1[0-2])\\)$")) | + select(.name | test("^cli-test-shards \\(([1-9]|10)\\)$")) | select(.conclusion != "success")] as $failed | if ($failed | length) == 0 or - ($failed | length) > 12 or + ($failed | length) > 10 or (($failed | map(.name) | unique | length) != ($failed | length)) then error("invalid failed CLI shard listing") else @@ -297,7 +297,7 @@ jobs: - name: Merge CLI coverage uses: ./.github/actions/ci-cli-coverage-merge with: - shard-count: "12" + shard-count: "10" plugin-tests: permissions: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1fb2ffa0a89..590c5e5ac85 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -573,7 +573,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -631,7 +631,7 @@ jobs: uses: ./.trusted-ci-actions/.github/actions/ci-cli-coverage-shard with: shard: ${{ matrix.shard }} - shard-count: "12" + shard-count: "10" cli-tests: needs: @@ -673,10 +673,10 @@ jobs: else [.jobs[] | select((.name | type) == "string") | - select(.name | test("^cli-test-shards \\(([1-9]|1[0-2])\\)$")) | + select(.name | test("^cli-test-shards \\(([1-9]|10)\\)$")) | select(.conclusion != "success")] as $failed | if ($failed | length) == 0 or - ($failed | length) > 12 or + ($failed | length) > 10 or (($failed | map(.name) | unique | length) != ($failed | length)) then error("invalid failed CLI shard listing") else @@ -744,7 +744,7 @@ jobs: - name: Merge CLI coverage uses: ./.trusted-ci-actions/.github/actions/ci-cli-coverage-merge with: - shard-count: "12" + shard-count: "10" plugin-tests: needs: [changes, openshell-sdk-package] diff --git a/ci/cli-test-timing-hints.json b/ci/cli-test-timing-hints.json index 1d852284e49..f45abd98e27 100644 --- a/ci/cli-test-timing-hints.json +++ b/ci/cli-test-timing-hints.json @@ -1,215 +1,200 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", "schemaVersion": 2, "defaultDurationMs": 5000, "sources": [ { - "runId": 32538808045, - "artifactId": 9467034647, - "headSha": "9577b175338d6cf1ead335452ade76470f1593a9", - "recordedAt": "2026-08-22T00:40:44Z" + "runId": 33895428444, + "artifactId": 9946097324, + "headSha": "58732cfd84267842e6d8caa08ee9c7eeb01bf41a", + "recordedAt": "2026-09-04T16:46:06Z" }, { - "runId": 32541609216, - "artifactId": 9467388387, - "headSha": "1080ecce4fd4d0366e546b3e92a25c3ec158af61", - "recordedAt": "2026-08-22T01:03:19Z" + "runId": 33898651121, + "artifactId": 9947254151, + "headSha": "d99d1dc579b26be7b240cf50d26b210b7a8d1b1d", + "recordedAt": "2026-09-04T17:20:15Z" }, { - "runId": 32542347175, - "artifactId": 9467589302, - "headSha": "bb686324dd2ce19f3708c6900b3d22110e198662", - "recordedAt": "2026-08-22T01:16:45Z" + "runId": 33917968257, + "artifactId": 9954274692, + "headSha": "37cedc99007a776d4ea4591fcc74eb14bdcc0ef9", + "recordedAt": "2026-09-04T21:00:44Z" + }, + { + "runId": 33921927346, + "artifactId": 9955756307, + "headSha": "94bb868ea0f3e98533b06196def74b7559a1d1d4", + "recordedAt": "2026-09-04T21:52:12Z" + }, + { + "runId": 33930230351, + "artifactId": 9958570313, + "headSha": "5b74336486a8492ad87dd7eaba3d1bc06abbadf6", + "recordedAt": "2026-09-04T23:53:49Z" } ], "files": { - "src/lib/actions/sandbox/auto-pair-approval.test.ts": 10074, - "src/lib/actions/sandbox/destroy-flow.test.ts": 25594, - "src/lib/actions/sandbox/doctor-flow.test.ts": 24265, - "src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.test.ts": 8470, - "src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts": 5667, - "src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts": 7973, - "src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts": 34602, - "src/lib/actions/sandbox/mcp-bridge-status-state.test.ts": 10485, - "src/lib/actions/sandbox/start.test.ts": 21550, - "src/lib/actions/uninstall/bedrock-runtime-adapter-cleanup.test.ts": 5619, - "src/lib/actions/uninstall/hermes-portable-uninstall.test.ts": 20347, - "src/lib/actions/uninstall/portable-runtime-cleanup.test.ts": 13026, - "src/lib/actions/uninstall/run-plan-portable-runtime.test.ts": 5406, - "src/lib/agent/base-image-hermes-resolution.test.ts": 5182, - "src/lib/agent/base-image.test.ts": 9684, - "src/lib/inference/serving/catalog.test.ts": 20421, - "src/lib/onboard/compatible-endpoint-smoke.test.ts": 5086, - "src/lib/onboard/exit-step-failure.test.ts": 5262, - "src/lib/onboard/experimental/hermes-portable-build-context.test.ts": 7152, - "src/lib/onboard/gateway-host-runtime.test.ts": 14070, - "src/lib/onboard/setup-nim-flow.test.ts": 5286, - "src/lib/state/onboard-session-station-express.test.ts": 11308, - "src/lib/state/onboard-session.test.ts": 6062, - "src/lib/state/portable-uninstall-retirement.test.ts": 27072, - "src/lib/tunnel/services.test.ts": 5573, - "test/agents/deepagents/dcode-session-supervisor.test.ts": 7644, - "test/agents/deepagents/dcode-wrapper-identity.test.ts": 11176, - "test/agents/deepagents/deepagents-code-tui-startup-check.test.ts": 28222, - "test/agents/deepagents/langchain-deepagents-code-config.test.ts": 9774, - "test/agents/deepagents/langchain-deepagents-code-direct-module-patch.test.ts": 31234, - "test/agents/deepagents/langchain-deepagents-code-image-credentials.test.ts": 31365, - "test/agents/deepagents/langchain-deepagents-code-image.test.ts": 17504, - "test/agents/deepagents/langchain-deepagents-code-nemotron-profile-plugin.test.ts": 7530, - "test/agents/deepagents/nemo-deepagents-alias.test.ts": 17997, - "test/agents/hermes/hermes-kanban-snapshot.test.ts": 10528, - "test/agents/hermes/hermes-provider-foundation.test.ts": 5353, - "test/agents/hermes/hermes-runtime-api-key.test.ts": 9872, - "test/agents/hermes/hermes-state-ledger-snapshot.test.ts": 5742, - "test/agents/hermes/hermes-tool-gateway-broker.test.ts": 8095, - "test/agents/hermes/nemohermes-alias.test.ts": 14670, - "test/agents/openclaw/openclaw-chat-send-patch.test.ts": 8126, - "test/agents/openclaw/openclaw-config-guard.test.ts": 22936, - "test/agents/openclaw/openclaw-config-snapshot.test.ts": 5147, - "test/agents/openclaw/openclaw-device-self-approval-patch.test.ts": 36649, - "test/agents/openclaw/openclaw-device-stored-auth-patch.test.ts": 8297, - "test/agents/openclaw/openclaw-gemini-inference-compat-runtime.test.ts": 36915, - "test/agents/openclaw/openclaw-integrity-pin-base.test.ts": 10104, - "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 28600, - "test/automation/e2e/e2e-recommendations.test.ts": 45421, - "test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts": 24108, - "test/automation/pull-requests/pr-review-advisor-writing-guide.test.ts": 8835, - "test/automation/releases/release-latest-tag.test.ts": 20264, - "test/automation/releases/retire-release-label.test.ts": 5642, - "test/channels/channels-remove-full-teardown.test.ts": 9727, - "test/cli/cli-oclif-compatibility.test.ts": 7416, - "test/cli/connect-recovery.test.ts": 14143, - "test/cli/credentials-command.test.ts": 6511, - "test/cli/debug-command.test.ts": 27452, - "test/cli/destroy-gateway-cleanup.test.ts": 20186, - "test/cli/dispatch-basics.test.ts": 10094, - "test/cli/docker-outage.test.ts": 12355, - "test/cli/doctor-gateway-token.test.ts": 19537, - "test/cli/exit-code-user-error-surfaces.test.ts": 13250, - "test/cli/helpers.test.ts": 7927, - "test/cli/internal-cli.test.ts": 16451, - "test/cli/launch-routing.test.ts": 13502, - "test/cli/list-command-helper-uses.test.ts": 7294, - "test/cli/list-inference.test.ts": 25333, - "test/cli/list-share-live-inference.test.ts": 26393, - "test/cli/logs-documented-invocations.test.ts": 18011, - "test/cli/logs.test.ts": 17679, - "test/cli/maintenance-command.test.ts": 10074, - "test/cli/onboard-compatibility.test.ts": 13249, - "test/cli/repro-2666-silent-list-status.test.ts": 12380, - "test/cli/sandbox-mutations.test.ts": 22796, - "test/cli/sandbox-skill-install.test.ts": 5955, - "test/cli/sandbox-status-json.test.ts": 31057, - "test/cli/sandbox-status-text.test.ts": 14597, - "test/cli/tunnel-command.test.ts": 16882, - "test/cli/update.test.ts": 10077, - "test/credentials/credentials.test.ts": 6328, - "test/credentials/local-credential-helper-session.test.ts": 5247, - "test/credentials/rebuild-credential-hydration.test.ts": 9487, - "test/credentials/rebuild-credential-preflight.test.ts": 22966, - "test/e2e-runtime/brev-launchable-e2e.test.ts": 34518, - "test/e2e/support/base-image-publication-workflow-boundary.test.ts": 5429, - "test/e2e/support/cli-artifact-workflow-boundary.test.ts": 5392, - "test/e2e/support/dcode-profile-import-gate-workflow-boundary.test.ts": 6533, - "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts": 8284, - "test/e2e/support/e2e-live-target-gating.test.ts": 27808, - "test/e2e/support/e2e-operations-workflow-boundary.test.ts": 26042, - "test/e2e/support/e2e-progress-outcome.test.ts": 11907, - "test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts": 7706, - "test/e2e/support/e2e-semantic-phase-check.test.ts": 8397, - "test/e2e/support/e2e-workflow-trace.test.ts": 6200, - "test/e2e/support/hermes-workflow-boundary.test.ts": 7516, - "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts": 5436, - "test/e2e/support/launch-agent-turn.test.ts": 62646, - "test/e2e/support/live-test-outcome-invocation.test.ts": 5281, - "test/e2e/support/mcp-bridge-runtime-compatibility-cli.test.ts": 5746, - "test/e2e/support/mcp-workflow-boundary.test.ts": 10250, - "test/e2e/support/podman-cpu-lifecycle-helpers.test.ts": 6203, - "test/e2e/support/portable-profile-systemctl-shim.test.ts": 10124, - "test/e2e/support/shared-e2e-workflow-boundary.test.ts": 5690, - "test/e2e/support/workflow-plan.test.ts": 39886, - "test/generation/generate-managed-startup-profile-fixture.test.ts": 5664, - "test/generation/portable-cpu-delegation-docs.test.ts": 39159, - "test/inference/managed/managed-bootstrap-trampoline.test.ts": 14561, - "test/inference/ollama/ollama-auth-proxy-handler.test.ts": 6722, - "test/inference/ollama/ollama-proxy-recovery.test.ts": 34833, - "test/inference/ollama/ollama-proxy-startup.test.ts": 8094, - "test/install/install-gateway-state-root.test.ts": 5494, - "test/install/install-onboard-yes.test.ts": 11335, - "test/install/install-openshell-gateway-service.test.ts": 5873, - "test/install/install-openshell-upgrade-prompt.test.ts": 77471, - "test/install/installer-hash-check.test.ts": 41961, - "test/install/installer-sandbox-build-trust.test.ts": 6681, - "test/install/installer-supervisor-manifest-trust.test.ts": 6603, - "test/install/uninstall-prompt-pty.test.ts": 17156, - "test/install/uninstall.test.ts": 14144, - "test/mcp/mcp-add-crash-consistency.test.ts": 43147, - "test/mcp/mcp-bridge-destroy-marker-recovery.test.ts": 14345, - "test/mcp/mcp-bridge-servers.test.ts": 15199, - "test/mcp/mcp-provider-ownership.test.ts": 9896, - "test/mcp/mcp-tool-discovery-image-contract.test.ts": 8489, - "test/networking/dns-proxy.test.ts": 5827, - "test/onboarding/effective-policy-contracts.test.ts": 7870, - "test/onboarding/onboard-brave-validation.test.ts": 8395, - "test/onboarding/onboard-custom-dockerfile.test.ts": 19813, - "test/onboarding/onboard-dashboard.test.ts": 8568, - "test/onboarding/onboard-endpoint-url-rejection.test.ts": 18559, - "test/onboarding/onboard-exit-handler.test.ts": 8716, - "test/onboarding/onboard-extra-provider-reconciliation.test.ts": 12683, - "test/onboarding/onboard-finalization-dashboard-forward.test.ts": 5857, - "test/onboarding/onboard-fsm-live-slices.test.ts": 20855, - "test/onboarding/onboard-inference-reconciliation.test.ts": 10281, - "test/onboarding/onboard-installer-restore-intent.test.ts": 14131, - "test/onboarding/onboard-lifecycle.test.ts": 6967, - "test/onboarding/onboard-managed-image-buildless-e2e.test.ts": 14383, - "test/onboarding/onboard-messaging.test.ts": 72008, - "test/onboarding/onboard-ollama-autostart.test.ts": 17749, - "test/onboarding/onboard-ollama-context-floor.test.ts": 5807, - "test/onboarding/onboard-prepared-build-context.test.ts": 8435, - "test/onboarding/onboard-prompt-default-case.test.ts": 14894, - "test/onboarding/onboard-provider-key-bridge.test.ts": 7265, - "test/onboarding/onboard-remote-recreate-credential-reuse.test.ts": 5737, - "test/onboarding/onboard-reservation-recreate.test.ts": 18690, - "test/onboarding/onboard-sandbox-build.test.ts": 28225, - "test/onboarding/onboard-sandbox-recreation.test.ts": 68947, - "test/onboarding/onboard-selection-vllm.test.ts": 13182, - "test/onboarding/onboard-selection.test.ts": 50371, - "test/onboarding/onboard-terminal-dashboard.test.ts": 11927, - "test/onboarding/onboard.test.ts": 7899, - "test/repository/layer-import-boundaries.test.ts": 5292, - "test/repository/plugin-vitest-project.test.ts": 7766, - "test/repository/source-architecture.test.ts": 5535, - "test/repository/source-require-loader.test.ts": 5155, - "test/runtime/gateway/gateway-drift-preflight.test.ts": 7324, - "test/runtime/gateway/gateway-serving-watchdog.test.ts": 12304, - "test/runtime/gateway/gateway-state-reconcile-2276.test.ts": 29961, - "test/runtime/gateway/recover-port-forward.test.ts": 12821, - "test/runtime/messaging/telegram-diagnostics.test.ts": 5127, - "test/runtime/policy/policies.test.ts": 5377, - "test/runtime/policy/policy-channel-agent-resolution.test.ts": 8083, - "test/runtime/policy/policy-explain-cli.test.ts": 5249, - "test/runtime/policy/policy-openclaw-npm-compatibility.test.ts": 14295, - "test/runtime/policy/policy-preset-noop-disclosure.test.ts": 9796, - "test/runtime/policy/policy-tiers-onboard.test.ts": 8804, - "test/runtime/policy/portable-policy-failure-finality.test.ts": 8392, - "test/runtime/sandbox/reboot-identity-drift.test.ts": 13632, - "test/runtime/sandbox/sandbox-agent-surface-parity.test.ts": 6276, - "test/runtime/sandbox/sandbox-build-context.test.ts": 33488, - "test/runtime/sandbox/sandbox-download-upload-cli.test.ts": 6982, - "test/runtime/sandbox/sandbox-sessions-admin-agent-cli.test.ts": 7935, - "test/runtime/sandbox/sandbox-sessions-export-cli.test.ts": 20387, - "test/runtime/sandbox/sandbox-stuck-recovery.test.ts": 8760, - "test/sandbox-connect-inference/auto-pair-approval.test.ts": 20974, - "test/sandbox-connect-inference/route-swap-repair.test.ts": 5822, - "test/security/strict-tool-call-probe.test.ts": 31346, - "test/skills/check-gates-actions-evidence.test.ts": 21898, - "test/skills/check-gates-compliance.test.ts": 22684, - "test/skills/dependency-upgrade-skill-security.test.ts": 5353, - "test/skills/dependency-upgrade-skill.test.ts": 19850, - "test/state/snapshot-gateway-guard.test.ts": 17301, - "test/state/snapshot-openclaw-managed-extensions.test.ts": 18060, - "test/state/snapshot-restore-existing-dest.test.ts": 17653, - "test/state/snapshot-stale-directory-restore.test.ts": 7384, - "test/state/snapshot.test.ts": 23861 + "src/lib/actions/sandbox/destroy-flow.test.ts": 27486, + "src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts": 8374, + "src/lib/actions/sandbox/mcp-bridge-provider.test.ts": 9113, + "src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts": 28100, + "src/lib/actions/sandbox/mcp-bridge-status-state.test.ts": 6257, + "src/lib/actions/sandbox/policy-channel-conflict.test.ts": 20022, + "src/lib/actions/sandbox/start.test.ts": 24254, + "src/lib/actions/uninstall/bedrock-runtime-adapter-cleanup.test.ts": 6039, + "src/lib/actions/uninstall/hermes-portable-uninstall.test.ts": 10241, + "src/lib/actions/uninstall/portable-runtime-cleanup.test.ts": 5803, + "src/lib/actions/uninstall/run-plan-gateway-segregation-selected-port.test.ts": 7651, + "src/lib/agent/base-image.test.ts": 5011, + "src/lib/inference/serving/catalog.test.ts": 13164, + "src/lib/onboard/experimental/hermes-portable-build-context.test.ts": 19071, + "src/lib/onboard/experimental/hermes-portable-onboarding.test.ts": 8765, + "src/lib/onboard/gateway-host-runtime.test.ts": 11334, + "src/lib/onboard/portable-resume-lock-boundary.test.ts": 5829, + "src/lib/state/onboard-session-station-express.test.ts": 8205, + "src/lib/state/onboard-session.test.ts": 5234, + "src/lib/state/portable-uninstall-retirement.test.ts": 19790, + "src/lib/tunnel/services.test.ts": 5477, + "test/agents/deepagents/dcode-wrapper-identity.test.ts": 11455, + "test/agents/deepagents/deepagents-code-tui-startup-check.test.ts": 25675, + "test/agents/deepagents/deepagents-mcp-legacy-lifecycle.test.ts": 6508, + "test/agents/deepagents/langchain-deepagents-code-config.test.ts": 10901, + "test/agents/deepagents/langchain-deepagents-code-direct-module-patch.test.ts": 30009, + "test/agents/deepagents/langchain-deepagents-code-image-credentials.test.ts": 22926, + "test/agents/deepagents/langchain-deepagents-code-image.test.ts": 16810, + "test/agents/deepagents/langchain-deepagents-code-managed-entrypoints.test.ts": 5269, + "test/agents/deepagents/langchain-deepagents-code-nemotron-profile-plugin.test.ts": 10534, + "test/agents/deepagents/nemo-deepagents-alias.test.ts": 16325, + "test/agents/hermes/hermes-gateway-wrapper.test.ts": 6580, + "test/agents/hermes/hermes-provider-foundation.test.ts": 6073, + "test/agents/hermes/hermes-runtime-api-key.test.ts": 6496, + "test/agents/hermes/hermes-tool-gateway-broker.test.ts": 5939, + "test/agents/hermes/nemohermes-alias.test.ts": 13175, + "test/agents/openclaw/openclaw-config-guard.test.ts": 14176, + "test/agents/openclaw/openclaw-device-self-approval-patch.test.ts": 8453, + "test/agents/openclaw/openclaw-integrity-pin-base.test.ts": 10604, + "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 13566, + "test/automation/classify-ci-failure.test.ts": 32936, + "test/automation/pull-requests/analyze-pr-value-stream.test.ts": 10435, + "test/automation/releases/release-latest-tag.test.ts": 17720, + "test/automation/releases/reviewed-npm-audit-workflow.test.ts": 6952, + "test/channels/channels-remove-full-teardown.test.ts": 21054, + "test/cli/cli-oclif-compatibility.test.ts": 6897, + "test/cli/connect-recovery.test.ts": 6960, + "test/cli/credentials-command.test.ts": 5826, + "test/cli/debug-command.test.ts": 24056, + "test/cli/destroy-gateway-cleanup.test.ts": 19332, + "test/cli/dispatch-basics.test.ts": 17126, + "test/cli/docker-outage.test.ts": 11383, + "test/cli/doctor-gateway-token.test.ts": 16636, + "test/cli/exit-code-user-error-surfaces.test.ts": 12301, + "test/cli/helpers.test.ts": 7270, + "test/cli/internal-cli.test.ts": 16183, + "test/cli/launch-routing.test.ts": 13048, + "test/cli/list-inference.test.ts": 33405, + "test/cli/list-share-live-inference.test.ts": 23601, + "test/cli/logs-documented-invocations.test.ts": 17171, + "test/cli/logs.test.ts": 15994, + "test/cli/maintenance-command.test.ts": 9085, + "test/cli/onboard-compatibility.test.ts": 11449, + "test/cli/repro-2666-silent-list-status.test.ts": 11598, + "test/cli/sandbox-mutations.test.ts": 21620, + "test/cli/sandbox-skill-install.test.ts": 5685, + "test/cli/sandbox-status-json.test.ts": 29822, + "test/cli/sandbox-status-text.test.ts": 13578, + "test/cli/tunnel-command.test.ts": 15113, + "test/cli/update.test.ts": 8759, + "test/credentials/credentials.test.ts": 5295, + "test/credentials/rebuild-credential-hydration.test.ts": 10594, + "test/credentials/rebuild-credential-preflight.test.ts": 7504, + "test/e2e-runtime/brev-launchable-e2e.test.ts": 40750, + "test/e2e-runtime/brev-launchable-gateway-diagnostics.test.ts": 12525, + "test/e2e/support/e2e-live-target-gating.test.ts": 37684, + "test/e2e/support/e2e-operations-workflow-boundary.test.ts": 19501, + "test/e2e/support/e2e-progress-outcome.test.ts": 8623, + "test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts": 6449, + "test/e2e/support/e2e-semantic-phase-check.test.ts": 6122, + "test/e2e/support/hermes-workflow-boundary.test.ts": 5155, + "test/e2e/support/launch-agent-turn.test.ts": 50714, + "test/e2e/support/managed-image-protected-runtime-workflow.test.ts": 5213, + "test/e2e/support/mcp-workflow-boundary.test.ts": 8713, + "test/e2e/support/portable-profile-systemctl-shim.test.ts": 8548, + "test/e2e/support/shared-e2e-workflow-boundary.test.ts": 5206, + "test/e2e/support/workflow-plan.test.ts": 33835, + "test/inference/llama/llama-cpp-image.test.ts": 5513, + "test/inference/managed/managed-bootstrap-trampoline.test.ts": 12582, + "test/inference/ollama/ollama-proxy-recovery.test.ts": 40052, + "test/inference/ollama/ollama-proxy-startup.test.ts": 9872, + "test/install/install-openshell-upgrade-prompt.test.ts": 9316, + "test/install/installer-hash-check.test.ts": 13746, + "test/install/uninstall-prompt-pty.test.ts": 11817, + "test/install/uninstall.test.ts": 17541, + "test/mcp/mcp-add-crash-consistency.test.ts": 67941, + "test/mcp/mcp-bridge-destroy-marker-recovery.test.ts": 14870, + "test/mcp/mcp-bridge-servers.test.ts": 12361, + "test/mcp/mcp-destroy-lifecycle.test.ts": 15014, + "test/mcp/mcp-provider-ownership.test.ts": 9141, + "test/mcp/mcp-restart-policy-order.test.ts": 15551, + "test/networking/dns-proxy.test.ts": 5742, + "test/onboarding/effective-policy-contracts.test.ts": 7871, + "test/onboarding/onboard-brave-validation.test.ts": 8824, + "test/onboarding/onboard-custom-dockerfile.test.ts": 14734, + "test/onboarding/onboard-endpoint-url-rejection.test.ts": 14806, + "test/onboarding/onboard-exit-handler.test.ts": 9068, + "test/onboarding/onboard-extra-provider-reconciliation.test.ts": 7915, + "test/onboarding/onboard-fresh-create-cancellation-identity.test.ts": 49272, + "test/onboarding/onboard-fresh-create-provider-identity.test.ts": 30497, + "test/onboarding/onboard-fresh-create-recovery-identity.test.ts": 44995, + "test/onboarding/onboard-fsm-live-slices.test.ts": 29983, + "test/onboarding/onboard-hermes-portable-provider-publication.test.ts": 7087, + "test/onboarding/onboard-inference-reconciliation.test.ts": 12166, + "test/onboarding/onboard-installer-restore-intent.test.ts": 11131, + "test/onboarding/onboard-lifecycle.test.ts": 7372, + "test/onboarding/onboard-managed-image-buildless-e2e.test.ts": 14585, + "test/onboarding/onboard-messaging.test.ts": 63014, + "test/onboarding/onboard-nim-memory-selection.test.ts": 5815, + "test/onboarding/onboard-ollama-autostart.test.ts": 19183, + "test/onboarding/onboard-ollama-context-floor.test.ts": 6122, + "test/onboarding/onboard-prepared-build-context.test.ts": 8692, + "test/onboarding/onboard-prompt-default-case.test.ts": 16047, + "test/onboarding/onboard-provider-key-bridge.test.ts": 8723, + "test/onboarding/onboard-remote-recreate-credential-reuse.test.ts": 5997, + "test/onboarding/onboard-reservation-recreate.test.ts": 40804, + "test/onboarding/onboard-sandbox-build.test.ts": 23820, + "test/onboarding/onboard-sandbox-name.test.ts": 5063, + "test/onboarding/onboard-sandbox-recreation.test.ts": 40625, + "test/onboarding/onboard-selection-vllm.test.ts": 11283, + "test/onboarding/onboard-selection.test.ts": 53546, + "test/onboarding/onboard-terminal-dashboard.test.ts": 5128, + "test/onboarding/onboard.test.ts": 7766, + "test/repository/e2e-assertion-census.test.ts": 6151, + "test/repository/layer-import-boundaries.test.ts": 9165, + "test/repository/plugin-vitest-project.test.ts": 6683, + "test/repository/source-architecture.test.ts": 15693, + "test/repository/test-registration-boundary.test.ts": 11385, + "test/runtime/gateway/gateway-serving-watchdog.test.ts": 12216, + "test/runtime/gateway/gateway-state-reconcile-2276.test.ts": 6884, + "test/runtime/gateway/recover-port-forward.test.ts": 8596, + "test/runtime/messaging/telegram-diagnostics.test.ts": 5100, + "test/runtime/messaging/wechat-diagnostics.test.ts": 5099, + "test/runtime/policy/policy-channel-agent-resolution.test.ts": 5285, + "test/runtime/policy/policy-explain-cli.test.ts": 5487, + "test/runtime/policy/policy-preset-noop-disclosure.test.ts": 5802, + "test/runtime/sandbox/reboot-identity-drift.test.ts": 10553, + "test/runtime/sandbox/sandbox-agent-surface-parity.test.ts": 6068, + "test/runtime/sandbox/sandbox-base-security-packages.test.ts": 5110, + "test/runtime/sandbox/sandbox-build-context.test.ts": 8726, + "test/runtime/sandbox/sandbox-download-upload-cli.test.ts": 5925, + "test/runtime/sandbox/sandbox-sessions-admin-agent-cli.test.ts": 7381, + "test/runtime/sandbox/sandbox-sessions-export-cli.test.ts": 18932, + "test/runtime/sandbox/sandbox-stuck-recovery.test.ts": 8172, + "test/sandbox-connect-inference/auto-pair-approval.test.ts": 16670, + "test/sandbox-connect-inference/route-swap-repair.test.ts": 5131, + "test/skills/check-gates-actions-evidence.test.ts": 10111, + "test/skills/check-gates-compliance.test.ts": 11449, + "test/skills/dependency-upgrade-skill-security.test.ts": 5001, + "test/skills/dependency-upgrade-skill.test.ts": 18716, + "test/state/snapshot-gateway-guard.test.ts": 14069, + "test/state/snapshot-restore-existing-dest.test.ts": 15360 } } diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index f2e871942eb..2ba3fb4f57a 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -91,7 +91,7 @@ const trustedActionDirs = [ ".github/actions/ci-installer-integration", ] as const; -const cliShardCount = "12"; +const cliShardCount = "10"; const cliShardTimeoutMinutes = 30; const dependencyInstallJobs = [ "build-typecheck", @@ -1036,7 +1036,7 @@ describe("pull request and main workflow contracts", () => { GITHUB_OUTPUT: output, }); const invalidRange = runWorkflowShellStep(shardValidationStep, { - CLI_SHARD: "13", + CLI_SHARD: "11", CLI_SHARD_COUNT: cliShardCount, GITHUB_OUTPUT: join(temp, "github-output"), }); @@ -1143,7 +1143,7 @@ describe("pull request and main workflow contracts", () => { const failedShards = workflowJobListing([ workflowJob(101, "cli-test-shards (1)", "success"), workflowJob(102, "cli-test-shards (2)", "failure"), - workflowJob(112, "cli-test-shards (12)", "cancelled"), + workflowJob(110, "cli-test-shards (10)", "cancelled"), workflowJob(109, "plugin-tests", "success"), ]); const malformedShards = workflowJobListing([ @@ -1181,7 +1181,7 @@ describe("pull request and main workflow contracts", () => { expect(failure.status, `${workflowName}: ${failure.stderr}`).not.toBe(0); expect(failure.stdout).toContain(`${runUrl}/job/102`); - expect(failure.stdout).toContain(`${runUrl}/job/112`); + expect(failure.stdout).toContain(`${runUrl}/job/110`); expect(malformed.status).not.toBe(0); expect(malformed.stdout).toContain(`Details: ${runUrl}`); expect(malformed.stdout).not.toContain(`${runUrl}/job/`); diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index 5f965f21ec3..6e18167424b 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -41,9 +41,9 @@ const cliCoverageProjects = new Set(["cli", "integration", "e2e-support"]); // ordinary roster changes preserve ownership between profile refreshes. // Integration coverage is serialized, so it needs an independent salt instead // of relying on combined weight from the parallel CLI and E2E-support lanes. -const stableShardSalt = "7257"; -const integrationShardSalt = "18608"; -const e2eSupportShardSalt = "25980"; +const stableShardSalt = "5276"; +const integrationShardSalt = "45487"; +const e2eSupportShardSalt = "9870"; // Only measured outliers are stored; new and ordinary files share the // conservative fallback used to estimate each stable shard's load. const timingHintsUrl = new URL("../../ci/cli-test-timing-hints.json", import.meta.url); diff --git a/test/helpers/onboard-fresh-create-identity-suite.ts b/test/helpers/onboard-fresh-create-identity-suite.ts new file mode 100644 index 00000000000..c0f914d6ec6 --- /dev/null +++ b/test/helpers/onboard-fresh-create-identity-suite.ts @@ -0,0 +1,1107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { beforeEach, describe, it, vi } from "vitest"; +import { encodeMessagingPlan, makeMessagingPlan } from "./messaging-plan-fixtures"; +import { writeOkOpenshell } from "./onboard-openshell-fixture"; +import { type CommandEntry, onboardScriptMocksPath } from "./onboard-split-context"; + +beforeEach(() => { + vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); + vi.stubEnv("NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE", "1"); + vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); +}); + +const freshCreateIdentityCases = [ + { + group: "provider" as const, + title: "binds ordinary providers at create time before managed registration (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "managed-provider" as const, + }, + { + group: "provider" as const, + title: "rejects provider-backed APF creation before sandbox or provider effects (#9833)", + apfInterceptorRequested: true, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "provider-refusal" as const, + }, + { + group: "provider" as const, + title: "rejects a nondefault agent before credential reads or sandbox inspection (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: { name: "hermes" }, + expectedOutcome: "unsupported-agent-refusal" as const, + }, + { + group: "provider" as const, + title: + "rejects pre-resolved nondefault agent intent before credential reads or sandbox inspection (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "resolved-agent-refusal" as const, + }, + { + group: "provider" as const, + title: + "registers providerless APF only after identity, policy, and checkpoint verification (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "providerless-apf" as const, + }, + { + group: "provider" as const, + title: "rejects mismatched selector and get identities before later effects (#10463)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "identity-mismatch-refusal" as const, + }, + { + group: "recovery" as const, + title: "retains recovery state when the create runner fails after verification (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-runner-refusal" as const, + }, + { + group: "recovery" as const, + title: "retains recovery state when registry publication fails after create (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-registration-refusal" as const, + }, + { + group: "recovery" as const, + title: "blocks every reentry when registry-failure recovery has no durable journal (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-registration-recovery-readback-failure" as const, + }, + { + group: "recovery" as const, + title: "retries registry-failure recovery from the process-exit owner (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-registration-recovery-retry" as const, + }, + { + group: "recovery" as const, + title: "accepts an external policy change after registration (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-policy-change" as const, + }, + { + group: "provider" as const, + title: "rejects staged messaging intent before any onboarding side effect (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "staged-messaging-refusal" as const, + }, + { + group: "cancellation" as const, + title: "makes a tier-cancelled created sandbox recovery-only (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "cancel-after-create-tier" as const, + }, + { + group: "cancellation" as const, + title: "makes a tier-preset-cancelled created sandbox recovery-only (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "cancel-after-create-tier-presets" as const, + }, + { + group: "cancellation" as const, + title: "makes a custom-preset-cancelled created sandbox recovery-only (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "cancel-after-create-custom-presets" as const, + }, +] as const; + +export type FreshCreateIdentityGroup = (typeof freshCreateIdentityCases)[number]["group"]; + +export function registerFreshCreateIdentityTests(group: FreshCreateIdentityGroup): void { + describe(`fresh create identity (${group})`, () => { + it.each(freshCreateIdentityCases.filter((testCase) => testCase.group === group))( + "$title", + { + timeout: 45000, + }, + async ({ agent, apfInterceptorRequested, expectedOutcome, model, provider }) => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-ready-check.js"); + const payloadPath = path.join(tmpDir, "payload.json"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const recreateJournalPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "onboard-recreate-journal.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const entryOptionsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "entry-options.ts"), + ); + const retainedRecoveryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), + ); + const dockerExecPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "adapters", "docker", "exec.ts"), + ); + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; +const registry = require(${registryPath}); +const recreateJournal = require(${recreateJournalPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const entryOptions = require(${entryOptionsPath}); +const retainedRecovery = require(${retainedRecoveryPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const dockerExec = require(${dockerExecPath}); +dockerExec.dockerSpawn = () => { + const child = new EventEmitter(); + process.nextTick(() => child.emit("close", 0)); + return child; +}; +const fs = require("node:fs"); + +const commands = []; +const lifecycleObservationCommands = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ + sandboxName: "my-assistant", + sandboxId: "sbx-fresh-create", + gatewayName: "nemoclaw-18080", +}); +const mismatchedSandboxId = createdSandbox.state.sandboxId + "-mismatch"; +let sandboxListCalls = 0; +let dockerPsCalls = 0; +let registeredSandbox = null; +let credentialReadCalls = 0; +let identityMismatchGetCalls = 0; +let routeReservationCalls = 0; +const keepAlive = setInterval(() => {}, 1000); +const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; +const agent = ${JSON.stringify(agent)}; +const model = ${JSON.stringify(model)}; +const provider = ${JSON.stringify(provider)}; +const selectedChannels = ${JSON.stringify(expectedOutcome === "provider-refusal" ? ["telegram"] : null)}; +const cancellationSelector = ${JSON.stringify( + expectedOutcome.startsWith("cancel-after-create-") + ? expectedOutcome.slice("cancel-after-create-".length) + : null, + )}; +const cancelAfterCreate = cancellationSelector !== null; +const recoveryReentry = process.env.NEMOCLAW_RECOVERY_REENTRY || ""; +const identityMismatchRefusal = ${JSON.stringify(expectedOutcome === "identity-mismatch-refusal")}; +const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-messaging-refusal")}; +const postCreateRunnerRefusal = ${JSON.stringify(expectedOutcome === "post-create-runner-refusal")}; +const postCreateRegistrationRefusal = ${JSON.stringify( + expectedOutcome === "post-create-registration-refusal" || + expectedOutcome === "post-create-registration-recovery-readback-failure" || + expectedOutcome === "post-create-registration-recovery-retry", + )}; +let recoveryJournalReadbackFailuresRemaining = ${JSON.stringify( + expectedOutcome === "post-create-registration-recovery-readback-failure" + ? 100 + : expectedOutcome === "post-create-registration-recovery-retry" || + expectedOutcome === "post-create-runner-refusal" + ? 1 + : 0, + )}; +const postCreatePolicyChange = ${JSON.stringify(expectedOutcome === "post-create-policy-change")}; +let cancelPrompt = false; +const originalGetCredential = credentials.getCredential; +credentials.getCredential = (...args) => { + credentialReadCalls += 1; + if (typeof args[0] !== "string") return null; + return originalGetCredential(...args); +}; +const originalReserveSandboxInferenceRoute = registry.reserveSandboxInferenceRoute; +registry.reserveSandboxInferenceRoute = (...args) => { + routeReservationCalls += 1; + return originalReserveSandboxInferenceRoute(...args); +}; +runner.run = (command, opts = {}) => { + const cmd = _n(command); + _deleted = _deleted || cmd.includes("sandbox delete"); + commands.push({ command: cmd, env: opts.env || null }); + const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); + if (profileResult !== null) return profileResult; + const providerResult = require(${onboardScriptMocksPath}).mockNvidiaProviderGetRun(command, "nemoclaw-18080"); + if (providerResult !== null) return providerResult; + if (cmd.includes("sandbox delete") && createdSandbox.state.lifecycleState === "created") { + createdSandbox.delete(); + } + const sandboxResult = createdSandbox.run(command); + return sandboxResult ?? { status: 0 }; +}; + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:18080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) { + if (postCreatePolicyChange && registeredSandbox) { + throw new Error("final onboarding policy check failed"); + } + return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + } + if (cmd.includes("sandbox get") || cmd.includes("sandbox list")) { + lifecycleObservationCommands.push(cmd); + } + if (cmd.includes("sandbox list") && !cmd.includes("--selector")) { + sandboxListCalls += 1; + createdSandbox.setPhase(sandboxListCalls >= 2 ? "Ready" : "Pending"); + } + const sandboxCapture = createdSandbox.capture(command); + if (sandboxCapture !== null) { + if ( + identityMismatchRefusal && + cmd.includes("sandbox get") && + sandboxCapture.includes("Id: " + createdSandbox.state.sandboxId) + ) { + identityMismatchGetCalls += 1; + return sandboxCapture.replace(createdSandbox.state.sandboxId, mismatchedSandboxId); + } + return sandboxCapture; + } + if (cmd.startsWith("docker ps -a --no-trunc ")) { + dockerPsCalls += 1; + if (dockerPsCalls === 1) return "a".repeat(64); + } + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; + const retainedRegistryEntry = recoveryReentry && fs.existsSync(${JSON.stringify(payloadPath)}) + ? JSON.parse(fs.readFileSync(${JSON.stringify(payloadPath)}, "utf8")).currentRegistryEntry + : null; + const registryMutationCalls = []; + let checkpointReadCalls = 0; + if (!recoveryReentry) { + const session = retainedRecovery.createSession({ + sessionId: "session-owner", + sandboxName: "my-assistant", + agent: agent?.name ?? "openclaw", + }); + retainedRecovery.saveSession(session); + registry.save({ + defaultSandbox: null, + sandboxes: { + "my-assistant": { + name: "my-assistant", + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + provider, + model, + endpointUrl: null, + endpointSource: null, + credentialEnv: null, + preferredInferenceApi: null, + pendingRouteReservation: true, + reservationSessionId: "session-owner", + }, + }, + }); + recreateJournal.openOnboardRecreateJournal({ + target: { + sandboxName: "my-assistant", + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + }, + agentName: agent?.name ?? "openclaw", + note: () => {}, + observe: () => ({ state: "missing", liveIdentityFingerprint: null }), + intent: { + agent: agent?.name ?? "openclaw", + fromDockerfile: null, + provider, + model, + preferredInferenceApi: null, + sandboxGpuConfig: null, + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + toolDisclosure: "progressive", + dcodeAutoApprovalMode: null, + observabilityEnabled: false, + }, + }); + } + const durableGetSandbox = registry.getSandbox.bind(registry); + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + provider, + model, + sessionId: "session-owner", + apfInterceptorRequested, + getSandbox: (name) => retainedRegistryEntry ?? durableGetSandbox(name), + registerSandbox: (entry) => { + if (postCreateRegistrationRefusal) { + throw new Error("registry publication failed"); + } + registeredSandbox = entry; + registryMutationCalls.push({ operation: "register", name: entry.name }); + }, + updateSandbox: (name) => { registryMutationCalls.push({ operation: "update", name }); }, + setDefault: (name) => { registryMutationCalls.push({ operation: "set-default", name }); }, + removeSandbox: (name) => { registryMutationCalls.push({ operation: "remove", name }); }, + }); +if (postCreateRunnerRefusal) { + const requireCurrentCheckpoint = registry.requireCurrentPendingSandboxCreateIdentity; + registry.requireCurrentPendingSandboxCreateIdentity = (...args) => { + checkpointReadCalls += 1; + if (checkpointReadCalls === 6) { + throw new Error("post-verification create runner checkpoint failed"); + } + return requireCurrentCheckpoint(...args); + }; +} +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => { + if (cancelPrompt) { + throw Object.assign(new Error("Prompt interrupted"), { code: "SIGINT" }); + } + return ""; +}; + +const groupKillCalls = []; +const realProcessKill = process.kill.bind(process); +process.kill = (pid, signal) => { + if (pid < 0) { + groupKillCalls.push({ pid, signal }); + const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); + process.nextTick(() => createCommand.child.emit("close", signal === "SIGTERM" ? 0 : 1)); + return true; + } + return realProcessKill(pid, signal); +}; + +childProcess.spawn = (...args) => { + const command = [args[0], ...(Array.isArray(args[1]) ? args[1] : [])]; + createdSandbox.create(command); + if (_n(command).includes("sandbox create")) _deleted = false; + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + child.killCalls = []; + child.unrefCalls = 0; + child.stdout.destroyCalls = 0; + child.stderr.destroyCalls = 0; + child.stdout.destroy = () => { + child.stdout.destroyCalls += 1; + }; + child.stderr.destroy = () => { + child.stderr.destroyCalls += 1; + }; + child.unref = () => { + child.unrefCalls += 1; + }; + child.kill = (signal) => { + child.killCalls.push(signal); + process.nextTick(() => child.emit("close", signal === "SIGTERM" ? 0 : 1)); + return true; + }; + commands.push({ command: _n(command), env: args[2]?.env || null, child }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); + }); + return child; +}; + +const onboardModule = require(${onboardPath}); +const { createSandbox } = onboardModule; +if (recoveryJournalReadbackFailuresRemaining > 0) { + const renameSync = fs.renameSync.bind(fs); + fs.renameSync = (source, destination) => { + renameSync(source, destination); + if ( + recoveryJournalReadbackFailuresRemaining > 0 && + String(destination).endsWith("retained-sandbox-recovery.json") + ) { + recoveryJournalReadbackFailuresRemaining -= 1; + fs.writeFileSync( + destination, + JSON.stringify({ schemaVersion: 1, unresolved: [], resolutions: [] }), + ); + } + }; +} +if (cancelAfterCreate && !recoveryReentry) { + const session = onboardModule.onboardSession.loadSession(); + if (!session) throw new Error("missing seeded onboarding session"); + session.mode = "interactive"; + session.sandboxName = "my-assistant"; + session.metadata = { gatewayName: "nemoclaw-18080", fromDockerfile: null }; + onboardModule.onboardSession.saveSession(session); + onboardModule.registerIncompleteOnboardExitHandlerForSession( + onboardModule.onboardSession, + () => false, + ); +} + +const writePayload = (sandboxName, creationError, exitCode = 0) => { + const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); + fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ + sandboxName, + creationError, + exitCode, + deleted: _deleted, + sandboxCreated: createdSandbox.state.lifecycleState === "created", + sandboxId: createdSandbox.state.sandboxId, + sandboxListCalls, + killCalls: createCommand?.child?.killCalls ?? [], + groupKillCalls, + unrefCalls: createCommand?.child?.unrefCalls ?? 0, + stdoutDestroyCalls: createCommand?.child?.stdout.destroyCalls ?? 0, + stderrDestroyCalls: createCommand?.child?.stderr.destroyCalls ?? 0, + lifecycleObservationCommands, + registeredSandbox, + credentialReadCalls, + identityMismatchGetCalls, + mismatchedSandboxId, + routeReservationCalls, + checkpointReadCalls, + registryMutationCalls, + currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, + recoveryRegistryEntry: registry.getSandbox("my-assistant"), + savedSession: + cancelAfterCreate || + postCreateRunnerRefusal || + postCreateRegistrationRefusal || + postCreatePolicyChange + ? onboardModule.onboardSession.loadSession() + : null, + retainedRecoveryRecords: retainedRecovery.listRetainedSandboxRecoveryRecords(), + createCommand: createCommand?.command ?? null, + commandNames: commands.map((entry) => entry.command), + })); +}; +let finalCreationError = null; +if (${JSON.stringify( + expectedOutcome === "post-create-registration-recovery-retry" || + expectedOutcome === "post-create-runner-refusal", + )}) { + process.on("exit", (code) => writePayload(null, finalCreationError, code)); +} + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw-18080"; + if (recoveryReentry) { + if (recoveryReentry === "fresh-different-no-journal") { + try { + await onboardModule.onboard({ + fresh: true, + sandboxName: "replacement-sb", + deferProcessExit: true, + }); + writePayload(null, "recovery-only onboarding unexpectedly continued", 0); + } catch (error) { + writePayload( + null, + error instanceof Error ? error.message : String(error), + typeof error?.code === "number" ? error.code : 1, + ); + } + clearInterval(keepAlive); + return; + } + if (recoveryReentry === "fresh-different") { + const retainedNames = retainedRecovery + .listRetainedSandboxRecoveryRecords() + .map((record) => record.sandboxName); + const resolved = entryOptions.resolveDefaultRunEntryOptions( + { fresh: true, sandboxName: "replacement-sb" }, + onboardModule.onboardSession.loadSession(), + runner.validateName, + process.env, + retainedNames, + ); + onboardModule.onboardSession.clearSession(); + onboardModule.onboardSession.saveSession( + onboardModule.onboardSession.createSession({ + mode: resolved.nonInteractive ? "non-interactive" : "interactive", + sandboxName: resolved.requestedSandboxName, + metadata: { gatewayName: "nemoclaw-18080", fromDockerfile: null }, + }), + ); + writePayload("replacement-sb", null, 0); + clearInterval(keepAlive); + return; + } + try { + await onboardModule.onboard({ + resume: recoveryReentry === "explicit", + fresh: recoveryReentry === "fresh-same", + recreateSandbox: recoveryReentry === "recreate", + sandboxName: "my-assistant", + deferProcessExit: true, + }); + writePayload(null, "recovery-only onboarding unexpectedly continued", 0); + } catch (error) { + writePayload( + null, + error instanceof Error ? error.message : String(error), + typeof error?.code === "number" ? error.code : 1, + ); + } + clearInterval(keepAlive); + return; + } + const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, model, provider, null, null, null, selectedChannels, null, agent, null, null, null, []], + createFixture, + ); + createArgs[15] = { + ...createArgs[15], + ...(apfInterceptorRequested + ? { + apfInterceptorRequested: true, + deferSandboxEffectsUntilIdentityVerification: true, + } + : {}), + recreate: false, + toolDisclosure: "progressive", + observabilityEnabled: false, + ...(${JSON.stringify(expectedOutcome === "resolved-agent-refusal")} + ? { + resolved: { + policy: { options: { agentName: "hermes" } }, + }, + } + : {}), + }; + try { + const sandboxName = await createSandbox(...createArgs); + if (cancelAfterCreate) { + process.on("exit", (code) => writePayload(sandboxName, null, code)); + cancelPrompt = true; + if (cancellationSelector === "tier") { + await onboardModule.selectPolicyTier(); + } else if (cancellationSelector === "tier-presets") { + await onboardModule.selectTierPresetsAndAccess( + "balanced", + [{ name: "github", description: "GitHub" }], + ["github"], + ); + } else { + await onboardModule.presetsCheckboxSelector( + [{ name: "github", description: "GitHub" }], + ["github"], + ); + } + throw new Error("expected policy selection cancellation"); + } + writePayload(sandboxName, null); + } catch (error) { + if (cancelAfterCreate) throw error; + if (!apfInterceptorRequested) throw error; + finalCreationError = error instanceof Error ? error.message : String(error); + writePayload(null, finalCreationError); + } + clearInterval(keepAlive); +})().catch((error) => { + clearInterval(keepAlive); + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const childEnv = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: expectedOutcome.startsWith("cancel-after-create-") ? "" : "1", + NEMOCLAW_GATEWAY_PORT: "18080", + OPENSHELL_DRIVERS: "docker", + NEMOCLAW_MESSAGING_PLAN_B64: + expectedOutcome === "staged-messaging-refusal" + ? encodeMessagingPlan( + makeMessagingPlan({ sandboxName: "my-assistant", channels: ["telegram"] }), + ) + : "", + }; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: childEnv, + timeout: 30000, + }); + + const cancellationOutcome = expectedOutcome.startsWith("cancel-after-create-"); + assert.equal(result.status, cancellationOutcome ? 1 : 0, result.stderr); + assert.ok(fs.existsSync(payloadPath), result.stderr); + const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + const providerEffectCommands = payload.commandNames.filter((command: string) => + /(?:^|\s)provider (?:create|update|delete|profile import)\b|(?:^|\s)sandbox provider (?:attach|detach)\b/u.test( + command, + ), + ); + const providerExposureCommands = payload.commandNames.filter((command: string) => + /(?:^|\s)provider (?:create|update|profile import)\b|(?:^|\s)sandbox provider attach\b/u.test( + command, + ), + ); + const identityFingerprint = createHash("sha256").update(payload.sandboxId).digest("hex"); + const assertRecoveryTuple = (record: Record) => { + assert.equal(record.gatewayName, "nemoclaw-18080"); + assert.equal(record.gatewayPort, 18080); + assert.equal(record.sandboxIdentityFingerprint, identityFingerprint); + assert.equal( + record.lifecycleGeneration, + payload.recoveryRegistryEntry.lifecycleGeneration, + ); + }; + const assertProviderBackedApfRefusal = () => { + assert.match( + payload.creationError, + /supports providerless sandbox creation only.*No sandbox or provider was created/u, + ); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, false); + assert.equal(payload.createCommand, null); + assert.equal(payload.registeredSandbox, null); + assert.equal(payload.credentialReadCalls, 0); + assert.equal(payload.routeReservationCalls, 0); + assert.deepEqual(payload.registryMutationCalls, []); + assert.deepEqual(providerEffectCommands, []); + assert.equal( + payload.commandNames.some((command: string) => command.includes("sandbox create")), + false, + ); + }; + const assertUnsupportedAgentRefusal = () => { + assertProviderBackedApfRefusal(); + assert.deepEqual(payload.commandNames, []); + assert.equal(payload.sandboxListCalls, 0); + }; + const assertStagedMessagingRefusal = () => { + assert.match( + payload.creationError, + /supports providerless sandbox creation only.*No sandbox or provider was created/u, + ); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, false); + assert.equal(payload.createCommand, null); + assert.equal(payload.registeredSandbox, null); + assert.equal(payload.credentialReadCalls, 0); + assert.equal(payload.routeReservationCalls, 0); + assert.deepEqual(providerEffectCommands, []); + assert.equal( + payload.commandNames.some((command: string) => + /(?:^|\s)(?:docker build|policy (?:set|apply)|sandbox create)(?:\s|$)/u.test(command), + ), + false, + ); + }; + const assertSuccessfulCreation = () => { + assert.equal(payload.creationError, null, result.stderr); + assert.equal(payload.sandboxName, "my-assistant"); + assert.ok(payload.sandboxListCalls >= 2); + assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); + assert.deepEqual(payload.killCalls, []); + assert.equal(payload.unrefCalls, 0); + assert.equal(payload.stdoutDestroyCalls, 0); + assert.equal(payload.stderrDestroyCalls, 0); + assert.equal(payload.registeredSandbox.workload.kind, "managed-image"); + assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); + assert.equal( + payload.registeredSandbox.lifecycleLiveIdentityFingerprint, + createHash("sha256").update(payload.sandboxId).digest("hex"), + ); + assert.match( + payload.createCommand, + /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, + ); + const ownerScopedObservations = payload.lifecycleObservationCommands.filter( + (command: string) => command.includes("-g nemoclaw-18080"), + ); + assert.ok( + ownerScopedObservations.length >= 6, + "expected owner-scoped sandbox identity observations", + ); + assert.ok( + ownerScopedObservations.every( + (command: string) => + command.includes("sandbox get -g nemoclaw-18080 my-assistant") || + command.includes("sandbox list -g nemoclaw-18080"), + ), + `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, + ); + }; + const assertManagedProviderCreation = () => { + assertSuccessfulCreation(); + assert.equal("policyAuthority" in payload.registeredSandbox, false); + assert.equal("policyCreationReceipt" in payload.registeredSandbox, false); + assert.match(payload.createCommand, /--policy \S+/u); + assert.match(payload.createCommand, /--provider nvidia-prod/u); + }; + const assertProviderlessApfCreation = () => { + assertSuccessfulCreation(); + for (const field of [ + "appliedPolicies", + "policies", + "policyAuthority", + "policyCreationReceipt", + "policyTier", + ]) { + assert.equal(field in payload.registeredSandbox, false); + } + assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:=|\s)/u); + assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); + assert.equal(payload.credentialReadCalls, 0); + assert.deepEqual(providerExposureCommands, []); + const createIndex = payload.commandNames.findIndex((command: string) => + command.includes("sandbox create"), + ); + const deferredEffectIndexes = payload.commandNames + .map((command: string, index: number) => ({ command, index })) + .filter(({ command }: { command: string }) => + /provider (?:profile import|create)|sandbox provider attach/u.test(command), + ) + .map(({ index }: { index: number }) => index); + assert.ok(deferredEffectIndexes.every((index: number) => index > createIndex)); + }; + const assertIdentityMismatchRefusal = () => { + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.match(payload.creationError, /automatic sandbox cleanup was not safe/u); + assert.notEqual(payload.mismatchedSandboxId, payload.sandboxId); + assert.ok(payload.identityMismatchGetCalls >= 1); + assert.equal(payload.registeredSandbox, null); + assert.equal(payload.credentialReadCalls, 0); + assert.deepEqual(payload.registryMutationCalls, []); + assert.deepEqual(providerEffectCommands, []); + assert.equal( + payload.commandNames.some((command: string) => + /(?:^|\s)policy (?:set|apply)(?:\s|$)/u.test(command), + ), + false, + ); + }; + const assertCreateAttemptLabelReported = () => { + const match = payload.createCommand?.match( + /--label (ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62})/u, + ); + assert.ok(match?.[1], "expected the sandbox create-attempt label"); + assert.ok( + `${payload.creationError ?? ""}\n${result.stderr}`.includes( + `Create-attempt label: ${match[1]}`, + ), + "expected recovery output to report the exact create-attempt label", + ); + }; + const assertPostCreateRunnerRefusal = () => { + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /automatic sandbox cleanup was not safe/u); + assertCreateAttemptLabelReported(); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + assert.equal(payload.retainedRecoveryRecords.length, 1); + assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); + assert.ok(payload.checkpointReadCalls >= 6); + assert.equal( + payload.commandNames.filter((command: string) => command.includes("sandbox create")) + .length, + 1, + ); + }; + const assertPostCreateRegistrationRefusal = () => { + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /registry publication failed/u); + assertCreateAttemptLabelReported(); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + assert.equal(payload.retainedRecoveryRecords.length, 1); + const record = payload.retainedRecoveryRecords[0]; + assert.equal(record.sandboxName, "my-assistant"); + assert.equal(record.reason, "retained_after_sandbox_creation_failure"); + assertRecoveryTuple(record); + }; + const assertPostCreateRegistrationRecoveryReadbackFailure = () => { + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /recovery record could not be persisted/u); + assertCreateAttemptLabelReported(); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + assert.deepEqual(payload.retainedRecoveryRecords, []); + + const reentryCases = [ + { + mode: "fresh-different-no-journal", + message: /independent retained sandbox recovery record is unavailable/u, + }, + { + mode: "fresh-same", + message: /explicit sandbox name different from the retained sandbox/u, + }, + ] as const; + for (const { message, mode } of reentryCases) { + const reentry = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...childEnv, + NEMOCLAW_RECOVERY_REENTRY: mode, + }, + timeout: 30000, + }); + assert.equal(reentry.status, 0, reentry.stderr); + const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(reentryPayload.exitCode, 1); + assert.match(reentry.stderr, message); + assert.deepEqual(reentryPayload.commandNames, []); + assert.equal(reentryPayload.credentialReadCalls, 0); + assert.equal(reentryPayload.routeReservationCalls, 0); + assert.deepEqual(reentryPayload.registryMutationCalls, []); + assert.equal(reentryPayload.savedSession.sandboxName, "my-assistant"); + assert.deepEqual(reentryPayload.retainedRecoveryRecords, []); + } + }; + const assertPostCreateRegistrationRecoveryRetry = () => { + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /recovery record could not be persisted/u); + assertCreateAttemptLabelReported(); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal(payload.retainedRecoveryRecords.length, 1); + assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); + assert.equal( + payload.commandNames.filter((command: string) => command.includes("sandbox create")) + .length, + 1, + ); + }; + const assertPostCreatePolicyChange = () => { + assertSuccessfulCreation(); + assert.equal(payload.savedSession.status, "in_progress"); + assert.notEqual(payload.savedSession.status, "recovery_required"); + assert.deepEqual(payload.retainedRecoveryRecords, []); + }; + const assertCancellationRecovery = () => { + assert.equal(payload.exitCode, 1); + assert.equal(payload.sandboxName, "my-assistant"); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox.name, "my-assistant"); + assert.equal( + payload.currentRegistryEntry.lifecycleLiveIdentityFingerprint, + identityFingerprint, + ); + assert.equal(payload.currentRegistryEntry.name, "my-assistant"); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal(payload.savedSession.sandboxName, "my-assistant"); + assert.equal( + payload.savedSession.cancellationRecovery.reason, + "cancelled_after_sandbox_creation", + ); + assert.equal(payload.savedSession.cancellationRecovery.sandboxName, "my-assistant"); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + assert.equal( + payload.commandNames.some((command: string) => command.includes("sandbox delete")), + false, + ); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); + assert.match(result.stderr, /preserved incomplete sandbox 'my-assistant'/u); + assert.match(result.stderr, new RegExp(identityFingerprint, "u")); + assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); + assert.match(result.stderr, /Shared inference providers are gateway configuration/u); + assert.match(result.stderr, /not sandbox cleanup targets/u); + assert.match(result.stderr, /nemoclaw my-assistant destroy/u); + assert.match(result.stderr, /clear the matching recovery record/u); + assertCreateAttemptLabelReported(); + + const differentName = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...childEnv, + NEMOCLAW_RECOVERY_REENTRY: "fresh-different", + }, + timeout: 30000, + }); + assert.equal(differentName.status, 0, differentName.stderr); + const differentNamePayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(differentNamePayload.exitCode, 0); + assert.equal(differentNamePayload.savedSession.sandboxName, "replacement-sb"); + assert.deepEqual( + differentNamePayload.retainedRecoveryRecords, + payload.retainedRecoveryRecords, + ); + assert.deepEqual(differentNamePayload.commandNames, []); + assert.equal(differentNamePayload.credentialReadCalls, 0); + assert.equal(differentNamePayload.routeReservationCalls, 0); + assert.deepEqual(differentNamePayload.registryMutationCalls, []); + + const reentryCases = [ + { + mode: "automatic", + messages: [ + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, + ], + }, + { + mode: "explicit", + messages: [ + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, + ], + }, + { + mode: "recreate", + messages: [ + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, + ], + }, + { + mode: "fresh-same", + messages: [ + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, + ], + }, + ] as const; + for (const { messages, mode: reentryMode } of reentryCases) { + const reentry = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...childEnv, + NEMOCLAW_RECOVERY_REENTRY: reentryMode, + }, + timeout: 30000, + }); + assert.equal(reentry.status, 0, reentry.stderr); + const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(reentryPayload.exitCode, 1); + messages.forEach((message) => assert.match(reentry.stderr, message)); + assert.deepEqual(reentryPayload.commandNames, []); + assert.equal(reentryPayload.credentialReadCalls, 0); + assert.equal(reentryPayload.routeReservationCalls, 0); + assert.deepEqual(reentryPayload.registryMutationCalls, []); + assert.deepEqual(reentryPayload.currentRegistryEntry, payload.currentRegistryEntry); + assert.deepEqual(reentryPayload.savedSession, differentNamePayload.savedSession); + assert.deepEqual( + reentryPayload.retainedRecoveryRecords, + payload.retainedRecoveryRecords, + ); + } + }; + const assertions = { + "managed-provider": assertManagedProviderCreation, + "provider-refusal": assertProviderBackedApfRefusal, + "unsupported-agent-refusal": assertUnsupportedAgentRefusal, + "resolved-agent-refusal": assertUnsupportedAgentRefusal, + "providerless-apf": assertProviderlessApfCreation, + "identity-mismatch-refusal": assertIdentityMismatchRefusal, + "post-create-runner-refusal": assertPostCreateRunnerRefusal, + "post-create-registration-refusal": assertPostCreateRegistrationRefusal, + "post-create-registration-recovery-readback-failure": + assertPostCreateRegistrationRecoveryReadbackFailure, + "post-create-registration-recovery-retry": assertPostCreateRegistrationRecoveryRetry, + "post-create-policy-change": assertPostCreatePolicyChange, + "staged-messaging-refusal": assertStagedMessagingRefusal, + "cancel-after-create-tier": assertCancellationRecovery, + "cancel-after-create-tier-presets": assertCancellationRecovery, + "cancel-after-create-custom-presets": assertCancellationRecovery, + }; + assertions[expectedOutcome](); + }, + ); + }); +} diff --git a/test/onboarding/onboard-fresh-create-cancellation-identity.test.ts b/test/onboarding/onboard-fresh-create-cancellation-identity.test.ts new file mode 100644 index 00000000000..e67fdbf0444 --- /dev/null +++ b/test/onboarding/onboard-fresh-create-cancellation-identity.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerFreshCreateIdentityTests } from "../helpers/onboard-fresh-create-identity-suite"; + +registerFreshCreateIdentityTests("cancellation"); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts deleted file mode 100644 index 98d6516c449..00000000000 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ /dev/null @@ -1,1080 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { beforeEach, describe, it, vi } from "vitest"; -import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; -import { type CommandEntry, onboardScriptMocksPath } from "../helpers/onboard-split-context"; -import { encodeMessagingPlan, makeMessagingPlan } from "../helpers/messaging-plan-fixtures"; - -beforeEach(() => { - vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); - vi.stubEnv("NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE", "1"); - vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); -}); - -describe("fresh create identity", () => { - it.each([ - { - title: "binds ordinary providers at create time before managed registration (#9833)", - apfInterceptorRequested: false, - provider: "nvidia-prod", - model: "gpt-5.4", - agent: null, - expectedOutcome: "managed-provider" as const, - }, - { - title: "rejects provider-backed APF creation before sandbox or provider effects (#9833)", - apfInterceptorRequested: true, - provider: "nvidia-prod", - model: "gpt-5.4", - agent: null, - expectedOutcome: "provider-refusal" as const, - }, - { - title: "rejects a nondefault agent before credential reads or sandbox inspection (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: { name: "hermes" }, - expectedOutcome: "unsupported-agent-refusal" as const, - }, - { - title: - "rejects pre-resolved nondefault agent intent before credential reads or sandbox inspection (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "resolved-agent-refusal" as const, - }, - { - title: - "registers providerless APF only after identity, policy, and checkpoint verification (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "providerless-apf" as const, - }, - { - title: "rejects mismatched selector and get identities before later effects (#10463)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "identity-mismatch-refusal" as const, - }, - { - title: "retains recovery state when the create runner fails after verification (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "post-create-runner-refusal" as const, - }, - { - title: "retains recovery state when registry publication fails after create (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "post-create-registration-refusal" as const, - }, - { - title: "blocks every reentry when registry-failure recovery has no durable journal (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "post-create-registration-recovery-readback-failure" as const, - }, - { - title: "retries registry-failure recovery from the process-exit owner (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "post-create-registration-recovery-retry" as const, - }, - { - title: "accepts an external policy change after registration (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "post-create-policy-change" as const, - }, - { - title: "rejects staged messaging intent before any onboarding side effect (#9833)", - apfInterceptorRequested: true, - provider: null, - model: null, - agent: null, - expectedOutcome: "staged-messaging-refusal" as const, - }, - { - title: "makes a tier-cancelled created sandbox recovery-only (#9833)", - apfInterceptorRequested: false, - provider: "nvidia-prod", - model: "gpt-5.4", - agent: null, - expectedOutcome: "cancel-after-create-tier" as const, - }, - { - title: "makes a tier-preset-cancelled created sandbox recovery-only (#9833)", - apfInterceptorRequested: false, - provider: "nvidia-prod", - model: "gpt-5.4", - agent: null, - expectedOutcome: "cancel-after-create-tier-presets" as const, - }, - { - title: "makes a custom-preset-cancelled created sandbox recovery-only (#9833)", - apfInterceptorRequested: false, - provider: "nvidia-prod", - model: "gpt-5.4", - agent: null, - expectedOutcome: "cancel-after-create-custom-presets" as const, - }, - ])( - "$title", - { - timeout: 45000, - }, - async ({ agent, apfInterceptorRequested, expectedOutcome, model, provider }) => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "create-sandbox-ready-check.js"); - const payloadPath = path.join(tmpDir, "payload.json"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const recreateJournalPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "onboard-recreate-journal.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const entryOptionsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "entry-options.ts"), - ); - const retainedRecoveryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), - ); - const dockerExecPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "adapters", "docker", "exec.ts"), - ); - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` - const runner = require(${runnerPath}); - const fixtureMocks = require(${onboardScriptMocksPath}); - fixtureMocks.mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; -const registry = require(${registryPath}); -const recreateJournal = require(${recreateJournalPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const entryOptions = require(${entryOptionsPath}); -const retainedRecovery = require(${retainedRecoveryPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const dockerExec = require(${dockerExecPath}); -dockerExec.dockerSpawn = () => { - const child = new EventEmitter(); - process.nextTick(() => child.emit("close", 0)); - return child; -}; -const fs = require("node:fs"); - -const commands = []; -const lifecycleObservationCommands = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ - sandboxName: "my-assistant", - sandboxId: "sbx-fresh-create", - gatewayName: "nemoclaw-18080", -}); -const mismatchedSandboxId = createdSandbox.state.sandboxId + "-mismatch"; -let sandboxListCalls = 0; -let dockerPsCalls = 0; -let registeredSandbox = null; -let credentialReadCalls = 0; -let identityMismatchGetCalls = 0; -let routeReservationCalls = 0; -const keepAlive = setInterval(() => {}, 1000); -const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; -const agent = ${JSON.stringify(agent)}; -const model = ${JSON.stringify(model)}; -const provider = ${JSON.stringify(provider)}; -const selectedChannels = ${JSON.stringify(expectedOutcome === "provider-refusal" ? ["telegram"] : null)}; -const cancellationSelector = ${JSON.stringify( - expectedOutcome.startsWith("cancel-after-create-") - ? expectedOutcome.slice("cancel-after-create-".length) - : null, - )}; -const cancelAfterCreate = cancellationSelector !== null; -const recoveryReentry = process.env.NEMOCLAW_RECOVERY_REENTRY || ""; -const identityMismatchRefusal = ${JSON.stringify(expectedOutcome === "identity-mismatch-refusal")}; -const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-messaging-refusal")}; -const postCreateRunnerRefusal = ${JSON.stringify(expectedOutcome === "post-create-runner-refusal")}; -const postCreateRegistrationRefusal = ${JSON.stringify( - expectedOutcome === "post-create-registration-refusal" || - expectedOutcome === "post-create-registration-recovery-readback-failure" || - expectedOutcome === "post-create-registration-recovery-retry", - )}; -let recoveryJournalReadbackFailuresRemaining = ${JSON.stringify( - expectedOutcome === "post-create-registration-recovery-readback-failure" - ? 100 - : expectedOutcome === "post-create-registration-recovery-retry" || - expectedOutcome === "post-create-runner-refusal" - ? 1 - : 0, - )}; -const postCreatePolicyChange = ${JSON.stringify(expectedOutcome === "post-create-policy-change")}; -let cancelPrompt = false; -const originalGetCredential = credentials.getCredential; -credentials.getCredential = (...args) => { - credentialReadCalls += 1; - if (typeof args[0] !== "string") return null; - return originalGetCredential(...args); -}; -const originalReserveSandboxInferenceRoute = registry.reserveSandboxInferenceRoute; -registry.reserveSandboxInferenceRoute = (...args) => { - routeReservationCalls += 1; - return originalReserveSandboxInferenceRoute(...args); -}; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - _deleted = _deleted || cmd.includes("sandbox delete"); - commands.push({ command: cmd, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); - if (profileResult !== null) return profileResult; - const providerResult = require(${onboardScriptMocksPath}).mockNvidiaProviderGetRun(command, "nemoclaw-18080"); - if (providerResult !== null) return providerResult; - if (cmd.includes("sandbox delete") && createdSandbox.state.lifecycleState === "created") { - createdSandbox.delete(); - } - const sandboxResult = createdSandbox.run(command); - return sandboxResult ?? { status: 0 }; -}; - runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:18080"; - if (cmd.includes("policy get") && cmd.includes("--output json")) { - if (postCreatePolicyChange && registeredSandbox) { - throw new Error("final onboarding policy check failed"); - } - return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); - } - if (cmd.includes("sandbox get") || cmd.includes("sandbox list")) { - lifecycleObservationCommands.push(cmd); - } - if (cmd.includes("sandbox list") && !cmd.includes("--selector")) { - sandboxListCalls += 1; - createdSandbox.setPhase(sandboxListCalls >= 2 ? "Ready" : "Pending"); - } - const sandboxCapture = createdSandbox.capture(command); - if (sandboxCapture !== null) { - if ( - identityMismatchRefusal && - cmd.includes("sandbox get") && - sandboxCapture.includes("Id: " + createdSandbox.state.sandboxId) - ) { - identityMismatchGetCalls += 1; - return sandboxCapture.replace(createdSandbox.state.sandboxId, mismatchedSandboxId); - } - return sandboxCapture; - } - if (cmd.startsWith("docker ps -a --no-trunc ")) { - dockerPsCalls += 1; - if (dockerPsCalls === 1) return "a".repeat(64); - } - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; - const retainedRegistryEntry = recoveryReentry && fs.existsSync(${JSON.stringify(payloadPath)}) - ? JSON.parse(fs.readFileSync(${JSON.stringify(payloadPath)}, "utf8")).currentRegistryEntry - : null; - const registryMutationCalls = []; - let checkpointReadCalls = 0; - if (!recoveryReentry) { - const session = retainedRecovery.createSession({ - sessionId: "session-owner", - sandboxName: "my-assistant", - agent: agent?.name ?? "openclaw", - }); - retainedRecovery.saveSession(session); - registry.save({ - defaultSandbox: null, - sandboxes: { - "my-assistant": { - name: "my-assistant", - gatewayName: "nemoclaw-18080", - gatewayPort: 18080, - provider, - model, - endpointUrl: null, - endpointSource: null, - credentialEnv: null, - preferredInferenceApi: null, - pendingRouteReservation: true, - reservationSessionId: "session-owner", - }, - }, - }); - recreateJournal.openOnboardRecreateJournal({ - target: { - sandboxName: "my-assistant", - gatewayName: "nemoclaw-18080", - gatewayPort: 18080, - }, - agentName: agent?.name ?? "openclaw", - note: () => {}, - observe: () => ({ state: "missing", liveIdentityFingerprint: null }), - intent: { - agent: agent?.name ?? "openclaw", - fromDockerfile: null, - provider, - model, - preferredInferenceApi: null, - sandboxGpuConfig: null, - gatewayName: "nemoclaw-18080", - gatewayPort: 18080, - toolDisclosure: "progressive", - dcodeAutoApprovalMode: null, - observabilityEnabled: false, - }, - }); - } - const durableGetSandbox = registry.getSandbox.bind(registry); - const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - gatewayName: "nemoclaw-18080", - gatewayPort: 18080, - provider, - model, - sessionId: "session-owner", - apfInterceptorRequested, - getSandbox: (name) => retainedRegistryEntry ?? durableGetSandbox(name), - registerSandbox: (entry) => { - if (postCreateRegistrationRefusal) { - throw new Error("registry publication failed"); - } - registeredSandbox = entry; - registryMutationCalls.push({ operation: "register", name: entry.name }); - }, - updateSandbox: (name) => { registryMutationCalls.push({ operation: "update", name }); }, - setDefault: (name) => { registryMutationCalls.push({ operation: "set-default", name }); }, - removeSandbox: (name) => { registryMutationCalls.push({ operation: "remove", name }); }, - }); -if (postCreateRunnerRefusal) { - const requireCurrentCheckpoint = registry.requireCurrentPendingSandboxCreateIdentity; - registry.requireCurrentPendingSandboxCreateIdentity = (...args) => { - checkpointReadCalls += 1; - if (checkpointReadCalls === 6) { - throw new Error("post-verification create runner checkpoint failed"); - } - return requireCurrentCheckpoint(...args); - }; -} -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => { - if (cancelPrompt) { - throw Object.assign(new Error("Prompt interrupted"), { code: "SIGINT" }); - } - return ""; -}; - -const groupKillCalls = []; -const realProcessKill = process.kill.bind(process); -process.kill = (pid, signal) => { - if (pid < 0) { - groupKillCalls.push({ pid, signal }); - const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); - process.nextTick(() => createCommand.child.emit("close", signal === "SIGTERM" ? 0 : 1)); - return true; - } - return realProcessKill(pid, signal); -}; - -childProcess.spawn = (...args) => { - const command = [args[0], ...(Array.isArray(args[1]) ? args[1] : [])]; - createdSandbox.create(command); - if (_n(command).includes("sandbox create")) _deleted = false; - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - child.killCalls = []; - child.unrefCalls = 0; - child.stdout.destroyCalls = 0; - child.stderr.destroyCalls = 0; - child.stdout.destroy = () => { - child.stdout.destroyCalls += 1; - }; - child.stderr.destroy = () => { - child.stderr.destroyCalls += 1; - }; - child.unref = () => { - child.unrefCalls += 1; - }; - child.kill = (signal) => { - child.killCalls.push(signal); - process.nextTick(() => child.emit("close", signal === "SIGTERM" ? 0 : 1)); - return true; - }; - commands.push({ command: _n(command), env: args[2]?.env || null, child }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); - }); - return child; -}; - -const onboardModule = require(${onboardPath}); -const { createSandbox } = onboardModule; -if (recoveryJournalReadbackFailuresRemaining > 0) { - const renameSync = fs.renameSync.bind(fs); - fs.renameSync = (source, destination) => { - renameSync(source, destination); - if ( - recoveryJournalReadbackFailuresRemaining > 0 && - String(destination).endsWith("retained-sandbox-recovery.json") - ) { - recoveryJournalReadbackFailuresRemaining -= 1; - fs.writeFileSync( - destination, - JSON.stringify({ schemaVersion: 1, unresolved: [], resolutions: [] }), - ); - } - }; -} -if (cancelAfterCreate && !recoveryReentry) { - const session = onboardModule.onboardSession.loadSession(); - if (!session) throw new Error("missing seeded onboarding session"); - session.mode = "interactive"; - session.sandboxName = "my-assistant"; - session.metadata = { gatewayName: "nemoclaw-18080", fromDockerfile: null }; - onboardModule.onboardSession.saveSession(session); - onboardModule.registerIncompleteOnboardExitHandlerForSession( - onboardModule.onboardSession, - () => false, - ); -} - -const writePayload = (sandboxName, creationError, exitCode = 0) => { - const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); - fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ - sandboxName, - creationError, - exitCode, - deleted: _deleted, - sandboxCreated: createdSandbox.state.lifecycleState === "created", - sandboxId: createdSandbox.state.sandboxId, - sandboxListCalls, - killCalls: createCommand?.child?.killCalls ?? [], - groupKillCalls, - unrefCalls: createCommand?.child?.unrefCalls ?? 0, - stdoutDestroyCalls: createCommand?.child?.stdout.destroyCalls ?? 0, - stderrDestroyCalls: createCommand?.child?.stderr.destroyCalls ?? 0, - lifecycleObservationCommands, - registeredSandbox, - credentialReadCalls, - identityMismatchGetCalls, - mismatchedSandboxId, - routeReservationCalls, - checkpointReadCalls, - registryMutationCalls, - currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, - recoveryRegistryEntry: registry.getSandbox("my-assistant"), - savedSession: - cancelAfterCreate || - postCreateRunnerRefusal || - postCreateRegistrationRefusal || - postCreatePolicyChange - ? onboardModule.onboardSession.loadSession() - : null, - retainedRecoveryRecords: retainedRecovery.listRetainedSandboxRecoveryRecords(), - createCommand: createCommand?.command ?? null, - commandNames: commands.map((entry) => entry.command), - })); -}; -let finalCreationError = null; -if (${JSON.stringify( - expectedOutcome === "post-create-registration-recovery-retry" || - expectedOutcome === "post-create-runner-refusal", - )}) { - process.on("exit", (code) => writePayload(null, finalCreationError, code)); -} - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw-18080"; - if (recoveryReentry) { - if (recoveryReentry === "fresh-different-no-journal") { - try { - await onboardModule.onboard({ - fresh: true, - sandboxName: "replacement-sb", - deferProcessExit: true, - }); - writePayload(null, "recovery-only onboarding unexpectedly continued", 0); - } catch (error) { - writePayload( - null, - error instanceof Error ? error.message : String(error), - typeof error?.code === "number" ? error.code : 1, - ); - } - clearInterval(keepAlive); - return; - } - if (recoveryReentry === "fresh-different") { - const retainedNames = retainedRecovery - .listRetainedSandboxRecoveryRecords() - .map((record) => record.sandboxName); - const resolved = entryOptions.resolveDefaultRunEntryOptions( - { fresh: true, sandboxName: "replacement-sb" }, - onboardModule.onboardSession.loadSession(), - runner.validateName, - process.env, - retainedNames, - ); - onboardModule.onboardSession.clearSession(); - onboardModule.onboardSession.saveSession( - onboardModule.onboardSession.createSession({ - mode: resolved.nonInteractive ? "non-interactive" : "interactive", - sandboxName: resolved.requestedSandboxName, - metadata: { gatewayName: "nemoclaw-18080", fromDockerfile: null }, - }), - ); - writePayload("replacement-sb", null, 0); - clearInterval(keepAlive); - return; - } - try { - await onboardModule.onboard({ - resume: recoveryReentry === "explicit", - fresh: recoveryReentry === "fresh-same", - recreateSandbox: recoveryReentry === "recreate", - sandboxName: "my-assistant", - deferProcessExit: true, - }); - writePayload(null, "recovery-only onboarding unexpectedly continued", 0); - } catch (error) { - writePayload( - null, - error instanceof Error ? error.message : String(error), - typeof error?.code === "number" ? error.code : 1, - ); - } - clearInterval(keepAlive); - return; - } - const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( - [null, model, provider, null, null, null, selectedChannels, null, agent, null, null, null, []], - createFixture, - ); - createArgs[15] = { - ...createArgs[15], - ...(apfInterceptorRequested - ? { - apfInterceptorRequested: true, - deferSandboxEffectsUntilIdentityVerification: true, - } - : {}), - recreate: false, - toolDisclosure: "progressive", - observabilityEnabled: false, - ...(${JSON.stringify(expectedOutcome === "resolved-agent-refusal")} - ? { - resolved: { - policy: { options: { agentName: "hermes" } }, - }, - } - : {}), - }; - try { - const sandboxName = await createSandbox(...createArgs); - if (cancelAfterCreate) { - process.on("exit", (code) => writePayload(sandboxName, null, code)); - cancelPrompt = true; - if (cancellationSelector === "tier") { - await onboardModule.selectPolicyTier(); - } else if (cancellationSelector === "tier-presets") { - await onboardModule.selectTierPresetsAndAccess( - "balanced", - [{ name: "github", description: "GitHub" }], - ["github"], - ); - } else { - await onboardModule.presetsCheckboxSelector( - [{ name: "github", description: "GitHub" }], - ["github"], - ); - } - throw new Error("expected policy selection cancellation"); - } - writePayload(sandboxName, null); - } catch (error) { - if (cancelAfterCreate) throw error; - if (!apfInterceptorRequested) throw error; - finalCreationError = error instanceof Error ? error.message : String(error); - writePayload(null, finalCreationError); - } - clearInterval(keepAlive); -})().catch((error) => { - clearInterval(keepAlive); - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const childEnv = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: expectedOutcome.startsWith("cancel-after-create-") ? "" : "1", - NEMOCLAW_GATEWAY_PORT: "18080", - OPENSHELL_DRIVERS: "docker", - NEMOCLAW_MESSAGING_PLAN_B64: - expectedOutcome === "staged-messaging-refusal" - ? encodeMessagingPlan( - makeMessagingPlan({ sandboxName: "my-assistant", channels: ["telegram"] }), - ) - : "", - }; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: childEnv, - timeout: 30000, - }); - - const cancellationOutcome = expectedOutcome.startsWith("cancel-after-create-"); - assert.equal(result.status, cancellationOutcome ? 1 : 0, result.stderr); - assert.ok(fs.existsSync(payloadPath), result.stderr); - const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); - const providerEffectCommands = payload.commandNames.filter((command: string) => - /(?:^|\s)provider (?:create|update|delete|profile import)\b|(?:^|\s)sandbox provider (?:attach|detach)\b/u.test( - command, - ), - ); - const providerExposureCommands = payload.commandNames.filter((command: string) => - /(?:^|\s)provider (?:create|update|profile import)\b|(?:^|\s)sandbox provider attach\b/u.test( - command, - ), - ); - const identityFingerprint = createHash("sha256").update(payload.sandboxId).digest("hex"); - const assertRecoveryTuple = (record: Record) => { - assert.equal(record.gatewayName, "nemoclaw-18080"); - assert.equal(record.gatewayPort, 18080); - assert.equal(record.sandboxIdentityFingerprint, identityFingerprint); - assert.equal(record.lifecycleGeneration, payload.recoveryRegistryEntry.lifecycleGeneration); - }; - const assertProviderBackedApfRefusal = () => { - assert.match( - payload.creationError, - /supports providerless sandbox creation only.*No sandbox or provider was created/u, - ); - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, false); - assert.equal(payload.createCommand, null); - assert.equal(payload.registeredSandbox, null); - assert.equal(payload.credentialReadCalls, 0); - assert.equal(payload.routeReservationCalls, 0); - assert.deepEqual(payload.registryMutationCalls, []); - assert.deepEqual(providerEffectCommands, []); - assert.equal( - payload.commandNames.some((command: string) => command.includes("sandbox create")), - false, - ); - }; - const assertUnsupportedAgentRefusal = () => { - assertProviderBackedApfRefusal(); - assert.deepEqual(payload.commandNames, []); - assert.equal(payload.sandboxListCalls, 0); - }; - const assertStagedMessagingRefusal = () => { - assert.match( - payload.creationError, - /supports providerless sandbox creation only.*No sandbox or provider was created/u, - ); - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, false); - assert.equal(payload.createCommand, null); - assert.equal(payload.registeredSandbox, null); - assert.equal(payload.credentialReadCalls, 0); - assert.equal(payload.routeReservationCalls, 0); - assert.deepEqual(providerEffectCommands, []); - assert.equal( - payload.commandNames.some((command: string) => - /(?:^|\s)(?:docker build|policy (?:set|apply)|sandbox create)(?:\s|$)/u.test(command), - ), - false, - ); - }; - const assertSuccessfulCreation = () => { - assert.equal(payload.creationError, null, result.stderr); - assert.equal(payload.sandboxName, "my-assistant"); - assert.ok(payload.sandboxListCalls >= 2); - assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); - assert.deepEqual(payload.killCalls, []); - assert.equal(payload.unrefCalls, 0); - assert.equal(payload.stdoutDestroyCalls, 0); - assert.equal(payload.stderrDestroyCalls, 0); - assert.equal(payload.registeredSandbox.workload.kind, "managed-image"); - assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); - assert.equal( - payload.registeredSandbox.lifecycleLiveIdentityFingerprint, - createHash("sha256").update(payload.sandboxId).digest("hex"), - ); - assert.match( - payload.createCommand, - /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, - ); - const ownerScopedObservations = payload.lifecycleObservationCommands.filter( - (command: string) => command.includes("-g nemoclaw-18080"), - ); - assert.ok( - ownerScopedObservations.length >= 6, - "expected owner-scoped sandbox identity observations", - ); - assert.ok( - ownerScopedObservations.every( - (command: string) => - command.includes("sandbox get -g nemoclaw-18080 my-assistant") || - command.includes("sandbox list -g nemoclaw-18080"), - ), - `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, - ); - }; - const assertManagedProviderCreation = () => { - assertSuccessfulCreation(); - assert.equal("policyAuthority" in payload.registeredSandbox, false); - assert.equal("policyCreationReceipt" in payload.registeredSandbox, false); - assert.match(payload.createCommand, /--policy \S+/u); - assert.match(payload.createCommand, /--provider nvidia-prod/u); - }; - const assertProviderlessApfCreation = () => { - assertSuccessfulCreation(); - for (const field of [ - "appliedPolicies", - "policies", - "policyAuthority", - "policyCreationReceipt", - "policyTier", - ]) { - assert.equal(field in payload.registeredSandbox, false); - } - assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:=|\s)/u); - assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); - assert.equal(payload.credentialReadCalls, 0); - assert.deepEqual(providerExposureCommands, []); - const createIndex = payload.commandNames.findIndex((command: string) => - command.includes("sandbox create"), - ); - const deferredEffectIndexes = payload.commandNames - .map((command: string, index: number) => ({ command, index })) - .filter(({ command }: { command: string }) => - /provider (?:profile import|create)|sandbox provider attach/u.test(command), - ) - .map(({ index }: { index: number }) => index); - assert.ok(deferredEffectIndexes.every((index: number) => index > createIndex)); - }; - const assertIdentityMismatchRefusal = () => { - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, true); - assert.equal(payload.deleted, false); - assert.match(payload.creationError, /automatic sandbox cleanup was not safe/u); - assert.notEqual(payload.mismatchedSandboxId, payload.sandboxId); - assert.ok(payload.identityMismatchGetCalls >= 1); - assert.equal(payload.registeredSandbox, null); - assert.equal(payload.credentialReadCalls, 0); - assert.deepEqual(payload.registryMutationCalls, []); - assert.deepEqual(providerEffectCommands, []); - assert.equal( - payload.commandNames.some((command: string) => - /(?:^|\s)policy (?:set|apply)(?:\s|$)/u.test(command), - ), - false, - ); - }; - const assertCreateAttemptLabelReported = () => { - const match = payload.createCommand?.match( - /--label (ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62})/u, - ); - assert.ok(match?.[1], "expected the sandbox create-attempt label"); - assert.ok( - `${payload.creationError ?? ""}\n${result.stderr}`.includes( - `Create-attempt label: ${match[1]}`, - ), - "expected recovery output to report the exact create-attempt label", - ); - }; - const assertPostCreateRunnerRefusal = () => { - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, true); - assert.equal(payload.deleted, false); - assert.equal(payload.registeredSandbox, null); - assert.match(payload.creationError, /automatic sandbox cleanup was not safe/u); - assertCreateAttemptLabelReported(); - assert.equal(payload.savedSession.status, "recovery_required"); - assert.equal(payload.savedSession.resumable, false); - assert.equal( - payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, - identityFingerprint, - ); - assert.equal(payload.retainedRecoveryRecords.length, 1); - assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); - assertRecoveryTuple(payload.retainedRecoveryRecords[0]); - assert.ok(payload.checkpointReadCalls >= 6); - assert.equal( - payload.commandNames.filter((command: string) => command.includes("sandbox create")) - .length, - 1, - ); - }; - const assertPostCreateRegistrationRefusal = () => { - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, true); - assert.equal(payload.deleted, false); - assert.equal(payload.registeredSandbox, null); - assert.match(payload.creationError, /registry publication failed/u); - assertCreateAttemptLabelReported(); - assert.equal(payload.savedSession.status, "recovery_required"); - assert.equal(payload.savedSession.resumable, false); - assert.equal( - payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, - identityFingerprint, - ); - assert.equal(payload.retainedRecoveryRecords.length, 1); - const record = payload.retainedRecoveryRecords[0]; - assert.equal(record.sandboxName, "my-assistant"); - assert.equal(record.reason, "retained_after_sandbox_creation_failure"); - assertRecoveryTuple(record); - }; - const assertPostCreateRegistrationRecoveryReadbackFailure = () => { - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, true); - assert.equal(payload.deleted, false); - assert.equal(payload.registeredSandbox, null); - assert.match(payload.creationError, /recovery record could not be persisted/u); - assertCreateAttemptLabelReported(); - assert.equal(payload.savedSession.status, "recovery_required"); - assert.equal(payload.savedSession.resumable, false); - assert.equal( - payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, - identityFingerprint, - ); - assert.deepEqual(payload.retainedRecoveryRecords, []); - - const reentryCases = [ - { - mode: "fresh-different-no-journal", - message: /independent retained sandbox recovery record is unavailable/u, - }, - { - mode: "fresh-same", - message: /explicit sandbox name different from the retained sandbox/u, - }, - ] as const; - for (const { message, mode } of reentryCases) { - const reentry = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...childEnv, - NEMOCLAW_RECOVERY_REENTRY: mode, - }, - timeout: 30000, - }); - assert.equal(reentry.status, 0, reentry.stderr); - const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); - assert.equal(reentryPayload.exitCode, 1); - assert.match(reentry.stderr, message); - assert.deepEqual(reentryPayload.commandNames, []); - assert.equal(reentryPayload.credentialReadCalls, 0); - assert.equal(reentryPayload.routeReservationCalls, 0); - assert.deepEqual(reentryPayload.registryMutationCalls, []); - assert.equal(reentryPayload.savedSession.sandboxName, "my-assistant"); - assert.deepEqual(reentryPayload.retainedRecoveryRecords, []); - } - }; - const assertPostCreateRegistrationRecoveryRetry = () => { - assert.equal(payload.sandboxName, null); - assert.equal(payload.sandboxCreated, true); - assert.equal(payload.deleted, false); - assert.equal(payload.registeredSandbox, null); - assert.match(payload.creationError, /recovery record could not be persisted/u); - assertCreateAttemptLabelReported(); - assert.equal(payload.savedSession.status, "recovery_required"); - assert.equal(payload.savedSession.resumable, false); - assert.equal(payload.retainedRecoveryRecords.length, 1); - assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); - assertRecoveryTuple(payload.retainedRecoveryRecords[0]); - assert.equal( - payload.commandNames.filter((command: string) => command.includes("sandbox create")) - .length, - 1, - ); - }; - const assertPostCreatePolicyChange = () => { - assertSuccessfulCreation(); - assert.equal(payload.savedSession.status, "in_progress"); - assert.notEqual(payload.savedSession.status, "recovery_required"); - assert.deepEqual(payload.retainedRecoveryRecords, []); - }; - const assertCancellationRecovery = () => { - assert.equal(payload.exitCode, 1); - assert.equal(payload.sandboxName, "my-assistant"); - assert.equal(payload.deleted, false); - assert.equal(payload.registeredSandbox.name, "my-assistant"); - assert.equal( - payload.currentRegistryEntry.lifecycleLiveIdentityFingerprint, - identityFingerprint, - ); - assert.equal(payload.currentRegistryEntry.name, "my-assistant"); - assert.equal(payload.savedSession.status, "recovery_required"); - assert.equal(payload.savedSession.resumable, false); - assert.equal(payload.savedSession.sandboxName, "my-assistant"); - assert.equal( - payload.savedSession.cancellationRecovery.reason, - "cancelled_after_sandbox_creation", - ); - assert.equal(payload.savedSession.cancellationRecovery.sandboxName, "my-assistant"); - assert.equal( - payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, - identityFingerprint, - ); - assert.equal( - payload.commandNames.some((command: string) => command.includes("sandbox delete")), - false, - ); - assertRecoveryTuple(payload.retainedRecoveryRecords[0]); - assert.match(result.stderr, /preserved incomplete sandbox 'my-assistant'/u); - assert.match(result.stderr, new RegExp(identityFingerprint, "u")); - assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); - assert.match(result.stderr, /Shared inference providers are gateway configuration/u); - assert.match(result.stderr, /not sandbox cleanup targets/u); - assert.match(result.stderr, /nemoclaw my-assistant destroy/u); - assert.match(result.stderr, /clear the matching recovery record/u); - assertCreateAttemptLabelReported(); - - const differentName = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...childEnv, - NEMOCLAW_RECOVERY_REENTRY: "fresh-different", - }, - timeout: 30000, - }); - assert.equal(differentName.status, 0, differentName.stderr); - const differentNamePayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); - assert.equal(differentNamePayload.exitCode, 0); - assert.equal(differentNamePayload.savedSession.sandboxName, "replacement-sb"); - assert.deepEqual( - differentNamePayload.retainedRecoveryRecords, - payload.retainedRecoveryRecords, - ); - assert.deepEqual(differentNamePayload.commandNames, []); - assert.equal(differentNamePayload.credentialReadCalls, 0); - assert.equal(differentNamePayload.routeReservationCalls, 0); - assert.deepEqual(differentNamePayload.registryMutationCalls, []); - - const reentryCases = [ - { - mode: "automatic", - messages: [ - /cannot use retained sandbox 'my-assistant'/u, - /same-name fresh onboarding remain disabled/u, - ], - }, - { - mode: "explicit", - messages: [ - /cannot use retained sandbox 'my-assistant'/u, - /same-name fresh onboarding remain disabled/u, - ], - }, - { - mode: "recreate", - messages: [ - /cannot use retained sandbox 'my-assistant'/u, - /same-name fresh onboarding remain disabled/u, - ], - }, - { - mode: "fresh-same", - messages: [ - /cannot use retained sandbox 'my-assistant'/u, - /same-name fresh onboarding remain disabled/u, - ], - }, - ] as const; - for (const { messages, mode: reentryMode } of reentryCases) { - const reentry = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...childEnv, - NEMOCLAW_RECOVERY_REENTRY: reentryMode, - }, - timeout: 30000, - }); - assert.equal(reentry.status, 0, reentry.stderr); - const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); - assert.equal(reentryPayload.exitCode, 1); - messages.forEach((message) => assert.match(reentry.stderr, message)); - assert.deepEqual(reentryPayload.commandNames, []); - assert.equal(reentryPayload.credentialReadCalls, 0); - assert.equal(reentryPayload.routeReservationCalls, 0); - assert.deepEqual(reentryPayload.registryMutationCalls, []); - assert.deepEqual(reentryPayload.currentRegistryEntry, payload.currentRegistryEntry); - assert.deepEqual(reentryPayload.savedSession, differentNamePayload.savedSession); - assert.deepEqual(reentryPayload.retainedRecoveryRecords, payload.retainedRecoveryRecords); - } - }; - const assertions = { - "managed-provider": assertManagedProviderCreation, - "provider-refusal": assertProviderBackedApfRefusal, - "unsupported-agent-refusal": assertUnsupportedAgentRefusal, - "resolved-agent-refusal": assertUnsupportedAgentRefusal, - "providerless-apf": assertProviderlessApfCreation, - "identity-mismatch-refusal": assertIdentityMismatchRefusal, - "post-create-runner-refusal": assertPostCreateRunnerRefusal, - "post-create-registration-refusal": assertPostCreateRegistrationRefusal, - "post-create-registration-recovery-readback-failure": - assertPostCreateRegistrationRecoveryReadbackFailure, - "post-create-registration-recovery-retry": assertPostCreateRegistrationRecoveryRetry, - "post-create-policy-change": assertPostCreatePolicyChange, - "staged-messaging-refusal": assertStagedMessagingRefusal, - "cancel-after-create-tier": assertCancellationRecovery, - "cancel-after-create-tier-presets": assertCancellationRecovery, - "cancel-after-create-custom-presets": assertCancellationRecovery, - }; - assertions[expectedOutcome](); - }, - ); -}); diff --git a/test/onboarding/onboard-fresh-create-provider-identity.test.ts b/test/onboarding/onboard-fresh-create-provider-identity.test.ts new file mode 100644 index 00000000000..448d9da514b --- /dev/null +++ b/test/onboarding/onboard-fresh-create-provider-identity.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerFreshCreateIdentityTests } from "../helpers/onboard-fresh-create-identity-suite"; + +registerFreshCreateIdentityTests("provider"); diff --git a/test/onboarding/onboard-fresh-create-recovery-identity.test.ts b/test/onboarding/onboard-fresh-create-recovery-identity.test.ts new file mode 100644 index 00000000000..4e0bed4e2a6 --- /dev/null +++ b/test/onboarding/onboard-fresh-create-recovery-identity.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerFreshCreateIdentityTests } from "../helpers/onboard-fresh-create-identity-suite"; + +registerFreshCreateIdentityTests("recovery"); diff --git a/test/repository/cli-coverage-sequencer.test.ts b/test/repository/cli-coverage-sequencer.test.ts index aac8c8a67bc..6f9644a9938 100644 --- a/test/repository/cli-coverage-sequencer.test.ts +++ b/test/repository/cli-coverage-sequencer.test.ts @@ -115,29 +115,29 @@ describe("stable CLI coverage sharding", () => { ); expect(Object.fromEntries(owners)).toEqual({ - "cli:src/lib/example.test.ts": 6, - "e2e-support:test/e2e/support/example.test.ts": 1, + "cli:src/lib/example.test.ts": 4, + "e2e-support:test/e2e/support/example.test.ts": 8, "integration:test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts": 2, - "integration:test/credentials/local-credential-helper-fields.test.ts": 5, - "integration:test/regular-0.test.ts": 5, + "integration:test/credentials/local-credential-helper-fields.test.ts": 6, + "integration:test/regular-0.test.ts": 6, }); }); - it("keeps the current test roster balanced across the twelve CI shards (#6237)", () => { - const shards = assignStableShards(currentCliCoverageEntries(), 12); + it("keeps the current test roster balanced across the ten CI shards (#6237)", () => { + const shards = assignStableShards(currentCliCoverageEntries(), 10); const weights = shards.map((shard) => shard.totalWeightMs); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; - expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.05); + expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.06); }); - it("balances the serialized integration lane across the twelve CI shards (#6237)", () => { + it("balances the serialized integration lane across the ten CI shards (#6237)", () => { const integrationEntries = currentCliCoverageEntries().filter((entry) => entry.key.startsWith("integration:"), ); expect(integrationEntries.length).toBeGreaterThan(0); - const weights = assignStableShards(integrationEntries, 12).map( + const weights = assignStableShards(integrationEntries, 10).map( (shard) => shard.totalWeightMs, ); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; @@ -182,22 +182,34 @@ describe("stable CLI coverage sharding", () => { expect(cliTestTimingHints.defaultDurationMs).toBe(5_000); expect(cliTestTimingHints.sources).toEqual([ { - runId: 32538808045, - artifactId: 9467034647, - headSha: "9577b175338d6cf1ead335452ade76470f1593a9", - recordedAt: "2026-08-22T00:40:44Z", + runId: 33895428444, + artifactId: 9946097324, + headSha: "58732cfd84267842e6d8caa08ee9c7eeb01bf41a", + recordedAt: "2026-09-04T16:46:06Z", }, { - runId: 32541609216, - artifactId: 9467388387, - headSha: "1080ecce4fd4d0366e546b3e92a25c3ec158af61", - recordedAt: "2026-08-22T01:03:19Z", + runId: 33898651121, + artifactId: 9947254151, + headSha: "d99d1dc579b26be7b240cf50d26b210b7a8d1b1d", + recordedAt: "2026-09-04T17:20:15Z", }, { - runId: 32542347175, - artifactId: 9467589302, - headSha: "bb686324dd2ce19f3708c6900b3d22110e198662", - recordedAt: "2026-08-22T01:16:45Z", + runId: 33917968257, + artifactId: 9954274692, + headSha: "37cedc99007a776d4ea4591fcc74eb14bdcc0ef9", + recordedAt: "2026-09-04T21:00:44Z", + }, + { + runId: 33921927346, + artifactId: 9955756307, + headSha: "94bb868ea0f3e98533b06196def74b7559a1d1d4", + recordedAt: "2026-09-04T21:52:12Z", + }, + { + runId: 33930230351, + artifactId: 9958570313, + headSha: "5b74336486a8492ad87dd7eaba3d1bc06abbadf6", + recordedAt: "2026-09-04T23:53:49Z", }, ]); expect(files).toEqual([...files].sort()); From 383dfa381a1d9947292475f0ae3d214cfb7e7420 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 4 Sep 2026 21:36:11 -0700 Subject: [PATCH 02/20] perf(ci): refine hosted shard balance Signed-off-by: Prekshi Vyas --- test/helpers/cli-coverage-sequencer.ts | 4 ++-- test/helpers/vitest-watch-triggers.ts | 4 +++- test/repository/cli-coverage-sequencer.test.ts | 2 +- test/repository/vitest-watch-triggers.test.ts | 4 +++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index 6e18167424b..6ecef5d403d 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -41,9 +41,9 @@ const cliCoverageProjects = new Set(["cli", "integration", "e2e-support"]); // ordinary roster changes preserve ownership between profile refreshes. // Integration coverage is serialized, so it needs an independent salt instead // of relying on combined weight from the parallel CLI and E2E-support lanes. -const stableShardSalt = "5276"; +const stableShardSalt = "43695"; const integrationShardSalt = "45487"; -const e2eSupportShardSalt = "9870"; +const e2eSupportShardSalt = "41846"; // Only measured outliers are stored; new and ordinary files share the // conservative fallback used to estimate each stable shard's load. const timingHintsUrl = new URL("../../ci/cli-test-timing-hints.json", import.meta.url); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index ea73d442abc..5818ed6b884 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -55,7 +55,9 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ "test/helpers/onboard-created-sandbox-fixture.test.ts", "test/onboarding/onboard-custom-dockerfile.test.ts", "test/onboarding/onboard-extra-provider-reconciliation.test.ts", - "test/onboarding/onboard-fresh-create-identity.test.ts", + "test/onboarding/onboard-fresh-create-provider-identity.test.ts", + "test/onboarding/onboard-fresh-create-recovery-identity.test.ts", + "test/onboarding/onboard-fresh-create-cancellation-identity.test.ts", "test/onboarding/onboard-installer-restore-intent.test.ts", "test/onboarding/onboard-managed-image-buildless-e2e.test.ts", "test/onboarding/onboard-mcp-observability-redirect.test.ts", diff --git a/test/repository/cli-coverage-sequencer.test.ts b/test/repository/cli-coverage-sequencer.test.ts index 6f9644a9938..4240b9b1aae 100644 --- a/test/repository/cli-coverage-sequencer.test.ts +++ b/test/repository/cli-coverage-sequencer.test.ts @@ -115,7 +115,7 @@ describe("stable CLI coverage sharding", () => { ); expect(Object.fromEntries(owners)).toEqual({ - "cli:src/lib/example.test.ts": 4, + "cli:src/lib/example.test.ts": 1, "e2e-support:test/e2e/support/example.test.ts": 8, "integration:test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts": 2, "integration:test/credentials/local-credential-helper-fields.test.ts": 6, diff --git a/test/repository/vitest-watch-triggers.test.ts b/test/repository/vitest-watch-triggers.test.ts index 5813c3fa6fe..2b976119ea6 100644 --- a/test/repository/vitest-watch-triggers.test.ts +++ b/test/repository/vitest-watch-triggers.test.ts @@ -123,7 +123,9 @@ describe("Vitest opaque-input watch triggers", () => { "test/helpers/onboard-created-sandbox-fixture.test.ts", "test/onboarding/onboard-custom-dockerfile.test.ts", "test/onboarding/onboard-extra-provider-reconciliation.test.ts", - "test/onboarding/onboard-fresh-create-identity.test.ts", + "test/onboarding/onboard-fresh-create-provider-identity.test.ts", + "test/onboarding/onboard-fresh-create-recovery-identity.test.ts", + "test/onboarding/onboard-fresh-create-cancellation-identity.test.ts", "test/onboarding/onboard-installer-restore-intent.test.ts", "test/onboarding/onboard-managed-image-buildless-e2e.test.ts", "test/onboarding/onboard-mcp-observability-redirect.test.ts", From 0574153b5fd72cc5188d65eb5231b862138e813b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 4 Sep 2026 21:59:52 -0700 Subject: [PATCH 03/20] perf(test): isolate gateway runtime unit dependency Signed-off-by: Prekshi Vyas --- src/lib/onboard/gateway-host-runtime.test.ts | 8 +++++++- src/lib/onboard/gateway-host-runtime.ts | 11 ++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index 94639413704..a7a1edb7adc 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -576,11 +576,17 @@ describe("gateway host runtime attachment probe", () => { it("reads the authoritative gateway port lazily, not at construction (#6576)", () => { let port = 8080; - const runtime = createGatewayHostRuntime(createDeps({ gatewayPort: () => port })); + const getGatewayStartNetworkEnv = vi.fn((gatewayPort: number) => ({ + OPENSHELL_SERVER_PORT: String(gatewayPort), + })); + const runtime = createGatewayHostRuntime( + createDeps({ gatewayPort: () => port, getGatewayStartNetworkEnv }), + ); port = 9443; expect(runtime.getGatewayStartEnv()).toMatchObject({ OPENSHELL_SERVER_PORT: "9443" }); + expect(getGatewayStartNetworkEnv).toHaveBeenCalledWith(9443); }, 15_000); it("registers and selects the exact declared endpoint without prior gateway metadata (#6576)", async () => { diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 49cdd794fa7..b3330efb24a 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -56,6 +56,8 @@ export interface GatewayHostRuntimeDeps { */ gatewayPort(): number; gatewayName(): string; + /** Optional narrow resolver for callers that already own the gateway network boundary. */ + getGatewayStartNetworkEnv?(gatewayPort: number): Record; /** * Unfiltered listener enumeration. An externally supervised gateway is an * ordinary systemd-run executable with no Docker-driver env markers, so the @@ -545,9 +547,12 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH // reads NEMOCLAW_GATEWAY_BIND_ADDRESS at load, and callers that reload // onboarding with a different environment drop it from the require cache. // A hoisted binding would pin this module to the stale first instance. - const { getGatewayStartNetworkEnv } = - require("./docker-driver-gateway-env") as typeof import("./docker-driver-gateway-env"); - const gatewayEnv = getGatewayStartNetworkEnv(deps.gatewayPort()); + const gatewayPort = deps.gatewayPort(); + const gatewayEnv = deps.getGatewayStartNetworkEnv + ? deps.getGatewayStartNetworkEnv(gatewayPort) + : ( + require("./docker-driver-gateway-env") as typeof import("./docker-driver-gateway-env") + ).getGatewayStartNetworkEnv(gatewayPort); const openshellVersion = deps.getInstalledOpenshellVersion(); if (openshellVersion) { const stableGatewayImage = `ghcr.io/nvidia/openshell/cluster:${openshellVersion}`; From 195c46b56ecae0349263252a84481a63488fd146 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 00:51:42 -0700 Subject: [PATCH 04/20] test(ci): close shard review gaps Signed-off-by: Prekshi Vyas --- .../pr-workflow-contract.test.ts | 16 ++++++++++++- .../onboard-fresh-create-identity-suite.ts | 23 +++++++++++-------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 2ba3fb4f57a..df8aa5c4eca 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -486,7 +486,21 @@ describe("pull request and main workflow contracts", () => { ["pull_request", prWorkflow], ["main", mainWorkflow], ] as const)("keeps the %s CLI coverage shard budget aligned", (_workflowName, workflow) => { - expect(workflow.jobs["cli-test-shards"]?.["timeout-minutes"]).toBe(cliShardTimeoutMinutes); + const shardJob = workflow.jobs["cli-test-shards"]; + const mergeJob = workflow.jobs["cli-tests"]; + const expectedShards = Array.from({ length: Number(cliShardCount) }, (_, index) => index + 1); + + expect({ + mergeShardCount: requiredWorkflowStep(mergeJob, "Merge CLI coverage").with?.["shard-count"], + shardCount: requiredWorkflowStep(shardJob, "Run CLI coverage shard").with?.["shard-count"], + shards: shardJob.strategy?.matrix?.shard, + timeoutMinutes: shardJob["timeout-minutes"], + }).toEqual({ + mergeShardCount: cliShardCount, + shardCount: cliShardCount, + shards: expectedShards, + timeoutMinutes: cliShardTimeoutMinutes, + }); }); // source-shape-contract: security -- Credential-free workflow structure prevents pull request code from receiving Hugging Face or checkout credentials diff --git a/test/helpers/onboard-fresh-create-identity-suite.ts b/test/helpers/onboard-fresh-create-identity-suite.ts index c0f914d6ec6..fdd97cb7595 100644 --- a/test/helpers/onboard-fresh-create-identity-suite.ts +++ b/test/helpers/onboard-fresh-create-identity-suite.ts @@ -162,11 +162,18 @@ const freshCreateIdentityCases = [ export type FreshCreateIdentityGroup = (typeof freshCreateIdentityCases)[number]["group"]; export function registerFreshCreateIdentityTests(group: FreshCreateIdentityGroup): void { + const childProcessTimeoutMs = 30_000; + const caseTimeoutMs = + group === "cancellation" + ? childProcessTimeoutMs * 6 + 15_000 + : group === "recovery" + ? childProcessTimeoutMs * 3 + 15_000 + : 45_000; describe(`fresh create identity (${group})`, () => { it.each(freshCreateIdentityCases.filter((testCase) => testCase.group === group))( "$title", { - timeout: 45000, + timeout: caseTimeoutMs, }, async ({ agent, apfInterceptorRequested, expectedOutcome, model, provider }) => { const repoRoot = path.join(import.meta.dirname, "../.."); @@ -335,7 +342,6 @@ runner.run = (command, opts = {}) => { ? JSON.parse(fs.readFileSync(${JSON.stringify(payloadPath)}, "utf8")).currentRegistryEntry : null; const registryMutationCalls = []; - let checkpointReadCalls = 0; if (!recoveryReentry) { const session = retainedRecovery.createSession({ sessionId: "session-owner", @@ -409,8 +415,7 @@ runner.run = (command, opts = {}) => { if (postCreateRunnerRefusal) { const requireCurrentCheckpoint = registry.requireCurrentPendingSandboxCreateIdentity; registry.requireCurrentPendingSandboxCreateIdentity = (...args) => { - checkpointReadCalls += 1; - if (checkpointReadCalls === 6) { + if (commands.some((entry) => entry.command.includes("sandbox create"))) { throw new Error("post-verification create runner checkpoint failed"); } return requireCurrentCheckpoint(...args); @@ -523,7 +528,6 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { identityMismatchGetCalls, mismatchedSandboxId, routeReservationCalls, - checkpointReadCalls, registryMutationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, recoveryRegistryEntry: registry.getSandbox("my-assistant"), @@ -688,7 +692,7 @@ if (${JSON.stringify( cwd: repoRoot, encoding: "utf-8", env: childEnv, - timeout: 30000, + timeout: childProcessTimeoutMs, }); const cancellationOutcome = expectedOutcome.startsWith("cancel-after-create-"); @@ -871,7 +875,6 @@ if (${JSON.stringify( assert.equal(payload.retainedRecoveryRecords.length, 1); assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); assertRecoveryTuple(payload.retainedRecoveryRecords[0]); - assert.ok(payload.checkpointReadCalls >= 6); assert.equal( payload.commandNames.filter((command: string) => command.includes("sandbox create")) .length, @@ -930,7 +933,7 @@ if (${JSON.stringify( ...childEnv, NEMOCLAW_RECOVERY_REENTRY: mode, }, - timeout: 30000, + timeout: childProcessTimeoutMs, }); assert.equal(reentry.status, 0, reentry.stderr); const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); @@ -1011,7 +1014,7 @@ if (${JSON.stringify( ...childEnv, NEMOCLAW_RECOVERY_REENTRY: "fresh-different", }, - timeout: 30000, + timeout: childProcessTimeoutMs, }); assert.equal(differentName.status, 0, differentName.stderr); const differentNamePayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); @@ -1064,7 +1067,7 @@ if (${JSON.stringify( ...childEnv, NEMOCLAW_RECOVERY_REENTRY: reentryMode, }, - timeout: 30000, + timeout: childProcessTimeoutMs, }); assert.equal(reentry.status, 0, reentry.stderr); const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); From ee6ac6334dbe97d639abdce1f7c00ebdf3d033f4 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 00:57:01 -0700 Subject: [PATCH 05/20] perf(ci): trim repeated plugin dependency work Signed-off-by: Prekshi Vyas --- .../actions/ci-build-typecheck/action.yaml | 4 +- .../actions/ci-cli-coverage-merge/action.yaml | 2 +- .../actions/ci-cli-coverage-shard/action.yaml | 2 +- .github/actions/ci-install-dependencies.sh | 24 ++++++- ci/cli-test-timing-hints.json | 1 - ci/source-shape-test-budget.json | 5 ++ .../pr-workflow-contract.test.ts | 37 +++++++--- .../ci-install-dependencies.test.ts | 69 +++++++++++++++---- test/repository/plugin-vitest-project.test.ts | 44 +++++++----- 9 files changed, 140 insertions(+), 48 deletions(-) diff --git a/.github/actions/ci-build-typecheck/action.yaml b/.github/actions/ci-build-typecheck/action.yaml index 869125ed57f..f2e02de4bbe 100644 --- a/.github/actions/ci-build-typecheck/action.yaml +++ b/.github/actions/ci-build-typecheck/action.yaml @@ -17,9 +17,9 @@ runs: shell: bash run: npm run typecheck:cli - - name: Typecheck plugin + - name: Typecheck plugin tests shell: bash - run: npm --prefix nemoclaw run typecheck + run: npx tsc --noEmit -p nemoclaw/tsconfig.test.json - name: Typecheck JavaScript config shell: bash diff --git a/.github/actions/ci-cli-coverage-merge/action.yaml b/.github/actions/ci-cli-coverage-merge/action.yaml index 6e8bb427c81..4bb3b111c4c 100644 --- a/.github/actions/ci-cli-coverage-merge/action.yaml +++ b/.github/actions/ci-cli-coverage-merge/action.yaml @@ -41,7 +41,7 @@ runs: shell: bash env: NODE_AUTH_TOKEN: ${{ github.event_name == 'push' && github.token || '' }} - run: bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh" + run: bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh" none - name: Download compiled CLI artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/actions/ci-cli-coverage-shard/action.yaml b/.github/actions/ci-cli-coverage-shard/action.yaml index ea642e0d58d..e107a936316 100644 --- a/.github/actions/ci-cli-coverage-shard/action.yaml +++ b/.github/actions/ci-cli-coverage-shard/action.yaml @@ -93,7 +93,7 @@ runs: shell: bash env: NODE_AUTH_TOKEN: ${{ github.event_name == 'push' && github.token || '' }} - run: bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh" + run: bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh" production - name: Validate changed live E2E mock parity if: ${{ inputs.shard == '1' }} diff --git a/.github/actions/ci-install-dependencies.sh b/.github/actions/ci-install-dependencies.sh index d0748c5c37c..c6c7e5bc610 100755 --- a/.github/actions/ci-install-dependencies.sh +++ b/.github/actions/ci-install-dependencies.sh @@ -4,6 +4,25 @@ set -euo pipefail +if [ "$#" -gt 1 ]; then + echo "Usage: ci-install-dependencies.sh [full|production|none]" >&2 + exit 1 +fi + +plugin_install_mode="${1:-full}" +plugin_install_args=(--prefix nemoclaw ci) +case "$plugin_install_mode" in + full) ;; + production) + plugin_install_args+=(--omit=dev) + ;; + none) ;; + *) + echo "Unsupported plugin dependency install mode: $plugin_install_mode" >&2 + exit 1 + ;; +esac + candidate_npmrc="$(find . -path './.git' -prune -o -name .npmrc -print -quit)" if [ -n "$candidate_npmrc" ]; then echo "Candidate repository npm configuration is not allowed during trusted dependency installation." >&2 @@ -57,4 +76,7 @@ if [ "$package_mode" = "registry" ] && [ -n "${NODE_AUTH_TOKEN:-}" ]; then fi npm ci --ignore-scripts --prefer-offline --no-audit --no-fund --cache "$npm_cache" -npm --prefix nemoclaw ci --ignore-scripts --prefer-offline --no-audit --no-fund --cache "$npm_cache" +if [ "$plugin_install_mode" != "none" ]; then + npm "${plugin_install_args[@]}" \ + --ignore-scripts --prefer-offline --no-audit --no-fund --cache "$npm_cache" +fi diff --git a/ci/cli-test-timing-hints.json b/ci/cli-test-timing-hints.json index f45abd98e27..7ae28191749 100644 --- a/ci/cli-test-timing-hints.json +++ b/ci/cli-test-timing-hints.json @@ -169,7 +169,6 @@ "test/onboarding/onboard.test.ts": 7766, "test/repository/e2e-assertion-census.test.ts": 6151, "test/repository/layer-import-boundaries.test.ts": 9165, - "test/repository/plugin-vitest-project.test.ts": 6683, "test/repository/source-architecture.test.ts": 15693, "test/repository/test-registration-boundary.test.ts": 11385, "test/runtime/gateway/gateway-serving-watchdog.test.ts": 12216, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index a0bdef95a75..2a4490c60e1 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -181,6 +181,11 @@ "test": "provides the package token only to trusted main dependency installation", "category": "security" }, + { + "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", + "test": "keeps %s plugin test typechecking after the trusted production build", + "category": "security" + }, { "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", "test": "passes only the base-packaged SDK archive to pull request dependency jobs", diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index df8aa5c4eca..6a511e0f832 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -466,9 +466,7 @@ describe("pull request and main workflow contracts", () => { ) as TypeScriptConfig; const sharedActions = { staticChecks: readYaml(".github/actions/ci-static-checks/action.yaml"), - compileArtifacts: readYaml( - ".github/actions/ci-compile-artifacts/action.yaml", - ), + compileArtifacts: readYaml(".github/actions/ci-compile-artifacts/action.yaml"), buildTypecheck: readYaml(".github/actions/ci-build-typecheck/action.yaml"), cliCoverageShard: readYaml( ".github/actions/ci-cli-coverage-shard/action.yaml", @@ -507,10 +505,7 @@ describe("pull request and main workflow contracts", () => { it("verifies changed Hugging Face catalog references without credentials", () => { const job = prWorkflow.jobs["hugging-face-models"]; const filterStep = prWorkflow.jobs.changes.steps?.find((step) => step.id === "filter"); - const filters = YAML.parse(String(filterStep?.with?.filters ?? "")) as Record< - string, - string[] - >; + const filters = YAML.parse(String(filterStep?.with?.filters ?? "")) as Record; const huggingFaceModelFilters = filters.hugging_face_models ?? []; expect( @@ -584,11 +579,33 @@ describe("pull request and main workflow contracts", () => { NODE_AUTH_TOKEN: "${{ github.event_name == 'push' && github.token || '' }}", })), ); - expect(actions.map((action) => requiredStep(action, "Install dependencies").run)).toEqual( - actions.map(() => 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh"'), - ); + expect(actions.map((action) => requiredStep(action, "Install dependencies").run)).toEqual([ + 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh"', + 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh"', + 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh" none', + 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh"', + 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh" production', + 'bash "$GITHUB_ACTION_PATH/../ci-install-dependencies.sh"', + ]); }); + // source-shape-contract: security -- The trusted split must retain test-config coverage after compiling candidate production code + it.each([ + ["pull request", prWorkflow], + ["main", mainWorkflow], + ] as const)( + "keeps %s plugin test typechecking after the trusted production build", + (_name, workflow) => { + expect([workflow.jobs["build-typecheck"].needs].flat()).toContain("compile-artifacts"); + expect(requiredStep(sharedActions.buildTypecheck, "Typecheck plugin tests").run).toBe( + "npx tsc --noEmit -p nemoclaw/tsconfig.test.json", + ); + expect(stepRuns(sharedActions.buildTypecheck)).not.toContain( + "npm --prefix nemoclaw run typecheck", + ); + }, + ); + // source-shape-contract: security -- The PR workflow must select an exact base-controlled package run before publishing its archive internally it("passes only the base-packaged SDK archive to pull request dependency jobs", () => { const packageJob = prWorkflow.jobs["openshell-sdk-package"]; diff --git a/test/repository/ci-install-dependencies.test.ts b/test/repository/ci-install-dependencies.test.ts index c6e7e718937..4dc5985bd58 100644 --- a/test/repository/ci-install-dependencies.test.ts +++ b/test/repository/ci-install-dependencies.test.ts @@ -42,6 +42,22 @@ function makeFixture(): { root: string; trace: string; path: string } { return { root, trace, path: `${bin}:${process.env.PATH || ""}` }; } +function runInstaller(fixture: ReturnType, args: string[] = []) { + return spawnSync("bash", [installer, ...args], { + cwd: fixture.root, + encoding: "utf8", + env: { + ...process.env, + GITHUB_ACTION_PATH: compositeActionPath, + GITHUB_EVENT_NAME: "pull_request", + NPM_CONFIG_CACHE: join(fixture.root, "npm-cache"), + NPM_TRACE: fixture.trace, + PATH: fixture.path, + RUNNER_TEMP: join(fixture.root, "runner-temp"), + }, + }); +} + afterEach(() => { for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); }); @@ -49,20 +65,7 @@ afterEach(() => { describe("shared CI dependency installer", () => { it("installs from a composite-action path without lifecycle scripts", () => { const fixture = makeFixture(); - - const result = spawnSync("bash", [installer], { - cwd: fixture.root, - encoding: "utf8", - env: { - ...process.env, - GITHUB_ACTION_PATH: compositeActionPath, - GITHUB_EVENT_NAME: "pull_request", - NPM_CONFIG_CACHE: join(fixture.root, "npm-cache"), - NPM_TRACE: fixture.trace, - PATH: fixture.path, - RUNNER_TEMP: join(fixture.root, "runner-temp"), - }, - }); + const result = runInstaller(fixture); expect(result.status, result.stderr).toBe(0); expect(readFileSync(fixture.trace, "utf8").trim().split("\n")).toEqual([ @@ -71,6 +74,44 @@ describe("shared CI dependency installer", () => { ]); }); + it("can install only plugin production dependencies", () => { + const fixture = makeFixture(); + const result = runInstaller(fixture, ["production"]); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(fixture.trace, "utf8").trim().split("\n")).toEqual([ + `ci --ignore-scripts --prefer-offline --no-audit --no-fund --cache ${join(fixture.root, "npm-cache")}`, + `--prefix nemoclaw ci --omit=dev --ignore-scripts --prefer-offline --no-audit --no-fund --cache ${join(fixture.root, "npm-cache")}`, + ]); + }); + + it("can skip plugin dependency installation", () => { + const fixture = makeFixture(); + const result = runInstaller(fixture, ["none"]); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(fixture.trace, "utf8").trim()).toBe( + `ci --ignore-scripts --prefer-offline --no-audit --no-fund --cache ${join(fixture.root, "npm-cache")}`, + ); + }); + + it.each([ + [["invalid"], "Unsupported plugin dependency install mode: invalid\n"], + [["production", "extra"], "Usage: ci-install-dependencies.sh [full|production|none]\n"], + ] as const)("rejects unsupported install arguments before npm runs [case %#]", (args, error) => { + const fixture = makeFixture(); + + const result = spawnSync("bash", [installer, ...args], { + cwd: fixture.root, + encoding: "utf8", + env: { ...process.env, NPM_TRACE: fixture.trace, PATH: fixture.path }, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(error); + expect(existsSync(fixture.trace)).toBe(false); + }); + it("rejects candidate npm configuration before npm receives the package token", () => { const fixture = makeFixture(); writeFileSync( diff --git a/test/repository/plugin-vitest-project.test.ts b/test/repository/plugin-vitest-project.test.ts index 23d7e6098d8..f99b9926e22 100644 --- a/test/repository/plugin-vitest-project.test.ts +++ b/test/repository/plugin-vitest-project.test.ts @@ -9,17 +9,27 @@ import { describe, expect, it } from "vitest"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const rootRequire = createRequire(path.join(repositoryRoot, "package.json")); -const pluginRequire = createRequire(path.join(repositoryRoot, "nemoclaw", "package.json")); -const pluginTypeScript = pluginRequire.resolve("typescript/bin/tsc"); +const rootTypeScript = rootRequire.resolve("typescript/bin/tsc"); -function installedVersion(requireFromPackage: NodeJS.Require, packageName: string): string { - return (requireFromPackage(`${packageName}/package.json`) as { version: string }).version; +type NpmDependencyTree = { + dependencies?: Record; + version?: string; +}; + +function lockedDependencyTree(prefix?: string): NpmDependencyTree { + const prefixArgs = prefix ? ["--prefix", prefix] : []; + return JSON.parse( + execFileSync("npm", [...prefixArgs, "ls", "--package-lock-only", "--json", "vitest", "vite"], { + cwd: repositoryRoot, + encoding: "utf8", + }), + ) as NpmDependencyTree; } function listedTypeScriptFiles(configPath: string): string[] { return execFileSync( process.execPath, - [pluginTypeScript, "--noEmit", "-p", configPath, "--listFilesOnly"], + [rootTypeScript, "--noEmit", "-p", configPath, "--listFilesOnly"], { cwd: repositoryRoot, encoding: "utf8" }, ) .trim() @@ -28,27 +38,25 @@ function listedTypeScriptFiles(configPath: string): string[] { } describe("plugin Vitest project contract", () => { - it.each(["vitest", "vite"] as const)( - "keeps standalone plugin dependencies on the root Vitest toolchain [case %#]", - (packageName) => { - expect(installedVersion(pluginRequire, packageName), packageName).toBe( - installedVersion(rootRequire, packageName), - ); - }, - ); + it("keeps the standalone plugin lock on the root Vitest toolchain", () => { + const rootTree = lockedDependencyTree(); + const pluginTree = lockedDependencyTree("nemoclaw"); + + expect(pluginTree.dependencies?.vitest?.version, "vitest").toBe( + rootTree.dependencies?.vitest?.version, + ); + expect(pluginTree.dependencies?.vitest?.dependencies?.vite?.version, "vite").toBe( + rootTree.dependencies?.vitest?.dependencies?.vite?.version, + ); + }); it("typechecks plugin production and test sources without emitting tests", () => { const productionFiles = listedTypeScriptFiles("nemoclaw/tsconfig.json"); const testFiles = listedTypeScriptFiles("nemoclaw/tsconfig.test.json"); - const typecheckOutput = execFileSync("npm", ["--prefix", "nemoclaw", "run", "typecheck"], { - cwd: repositoryRoot, - encoding: "utf8", - }); expect(productionFiles.some((file) => file.endsWith(".test.ts"))).toBe(false); expect(testFiles).toContain(path.join(repositoryRoot, "nemoclaw", "src", "register.test.ts")); expect(testFiles).toContain(path.join(repositoryRoot, "nemoclaw", "vitest.config.ts")); expect(testFiles).toContain(path.join(repositoryRoot, "nemoclaw", "vitest.project.ts")); - expect(typecheckOutput).toContain("tsc --noEmit -p tsconfig.test.json"); }); }); From 4df200771016e65ed2cb1b6929085f68ac1b0da2 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 01:25:00 -0700 Subject: [PATCH 06/20] test(ci): pin plugin compiler ownership Signed-off-by: Prekshi Vyas --- .github/actions/ci-build-typecheck/action.yaml | 2 +- .../pull-requests/pr-workflow-contract.test.ts | 2 +- test/repository/plugin-vitest-project.test.ts | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/actions/ci-build-typecheck/action.yaml b/.github/actions/ci-build-typecheck/action.yaml index f2e02de4bbe..a696c771231 100644 --- a/.github/actions/ci-build-typecheck/action.yaml +++ b/.github/actions/ci-build-typecheck/action.yaml @@ -19,7 +19,7 @@ runs: - name: Typecheck plugin tests shell: bash - run: npx tsc --noEmit -p nemoclaw/tsconfig.test.json + run: npm --prefix nemoclaw exec -- tsc --noEmit -p nemoclaw/tsconfig.test.json - name: Typecheck JavaScript config shell: bash diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 6a511e0f832..05fef4529c8 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -598,7 +598,7 @@ describe("pull request and main workflow contracts", () => { (_name, workflow) => { expect([workflow.jobs["build-typecheck"].needs].flat()).toContain("compile-artifacts"); expect(requiredStep(sharedActions.buildTypecheck, "Typecheck plugin tests").run).toBe( - "npx tsc --noEmit -p nemoclaw/tsconfig.test.json", + "npm --prefix nemoclaw exec -- tsc --noEmit -p nemoclaw/tsconfig.test.json", ); expect(stepRuns(sharedActions.buildTypecheck)).not.toContain( "npm --prefix nemoclaw run typecheck", diff --git a/test/repository/plugin-vitest-project.test.ts b/test/repository/plugin-vitest-project.test.ts index f99b9926e22..6f8e5547161 100644 --- a/test/repository/plugin-vitest-project.test.ts +++ b/test/repository/plugin-vitest-project.test.ts @@ -19,10 +19,14 @@ type NpmDependencyTree = { function lockedDependencyTree(prefix?: string): NpmDependencyTree { const prefixArgs = prefix ? ["--prefix", prefix] : []; return JSON.parse( - execFileSync("npm", [...prefixArgs, "ls", "--package-lock-only", "--json", "vitest", "vite"], { - cwd: repositoryRoot, - encoding: "utf8", - }), + execFileSync( + "npm", + [...prefixArgs, "ls", "--package-lock-only", "--json", "typescript", "vitest", "vite"], + { + cwd: repositoryRoot, + encoding: "utf8", + }, + ), ) as NpmDependencyTree; } @@ -38,7 +42,7 @@ function listedTypeScriptFiles(configPath: string): string[] { } describe("plugin Vitest project contract", () => { - it("keeps the standalone plugin lock on the root Vitest toolchain", () => { + it("keeps the standalone plugin lock on the root test toolchain", () => { const rootTree = lockedDependencyTree(); const pluginTree = lockedDependencyTree("nemoclaw"); @@ -48,6 +52,9 @@ describe("plugin Vitest project contract", () => { expect(pluginTree.dependencies?.vitest?.dependencies?.vite?.version, "vite").toBe( rootTree.dependencies?.vitest?.dependencies?.vite?.version, ); + expect(pluginTree.dependencies?.typescript?.version, "typescript").toBe( + rootTree.dependencies?.typescript?.version, + ); }); it("typechecks plugin production and test sources without emitting tests", () => { From f52d79fee5d2a1b82a9ffc1143feca5c5d8f140e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 01:49:42 -0700 Subject: [PATCH 07/20] test(ci): require locked plugin tool versions Signed-off-by: Prekshi Vyas --- test/repository/plugin-vitest-project.test.ts | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/test/repository/plugin-vitest-project.test.ts b/test/repository/plugin-vitest-project.test.ts index 6f8e5547161..6d458e46703 100644 --- a/test/repository/plugin-vitest-project.test.ts +++ b/test/repository/plugin-vitest-project.test.ts @@ -30,6 +30,12 @@ function lockedDependencyTree(prefix?: string): NpmDependencyTree { ) as NpmDependencyTree; } +function requiredLockedVersion(version: string | undefined, dependency: string): string { + expect(version, `${dependency} lockfile version`).toBeTypeOf("string"); + expect(version, `${dependency} lockfile version`).not.toBe(""); + return version as string; +} + function listedTypeScriptFiles(configPath: string): string[] { return execFileSync( process.execPath, @@ -46,15 +52,23 @@ describe("plugin Vitest project contract", () => { const rootTree = lockedDependencyTree(); const pluginTree = lockedDependencyTree("nemoclaw"); - expect(pluginTree.dependencies?.vitest?.version, "vitest").toBe( - rootTree.dependencies?.vitest?.version, - ); - expect(pluginTree.dependencies?.vitest?.dependencies?.vite?.version, "vite").toBe( - rootTree.dependencies?.vitest?.dependencies?.vite?.version, + expect(requiredLockedVersion(pluginTree.dependencies?.vitest?.version, "plugin vitest")).toBe( + requiredLockedVersion(rootTree.dependencies?.vitest?.version, "root vitest"), ); - expect(pluginTree.dependencies?.typescript?.version, "typescript").toBe( - rootTree.dependencies?.typescript?.version, + expect( + requiredLockedVersion( + pluginTree.dependencies?.vitest?.dependencies?.vite?.version, + "plugin vite", + ), + ).toBe( + requiredLockedVersion( + rootTree.dependencies?.vitest?.dependencies?.vite?.version, + "root vite", + ), ); + expect( + requiredLockedVersion(pluginTree.dependencies?.typescript?.version, "plugin typescript"), + ).toBe(requiredLockedVersion(rootTree.dependencies?.typescript?.version, "root typescript")); }); it("typechecks plugin production and test sources without emitting tests", () => { From a92f3f1a778da5bb2371ebccfdf559a1d4494b1a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 02:05:09 -0700 Subject: [PATCH 08/20] test(ci): stabilize shard timing contracts Signed-off-by: Prekshi Vyas --- ci/source-shape-test-budget.json | 5 ++ src/lib/onboard/sandbox-create-plan.test.ts | 2 +- test/repository/plugin-vitest-project.test.ts | 56 +++++++------------ tools/e2e/base-image-publication.mts | 13 ++++- 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 2a4490c60e1..dfec3e9e8df 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -186,6 +186,11 @@ "test": "keeps %s plugin test typechecking after the trusted production build", "category": "security" }, + { + "file": "test/repository/plugin-vitest-project.test.ts", + "test": "keeps standalone plugin %s locked to the root test toolchain", + "category": "compatibility" + }, { "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", "test": "passes only the base-packaged SDK archive to pull request dependency jobs", diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 113f44cc18f..5ed764a6a28 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -189,7 +189,7 @@ function expectCredentialBindingFailure({ expect(upsertProviders).not.toHaveBeenCalled(); } -describe("prepareSandboxCreatePolicy", () => { +describe("prepareSandboxCreatePolicy", testTimeoutOptions(15_000), () => { it("passes the sandbox name so credential-binding presets can materialize", () => { const intent = resolveSandboxCreateIntent({ basePolicyPath: "/repo/policy.yaml", diff --git a/test/repository/plugin-vitest-project.test.ts b/test/repository/plugin-vitest-project.test.ts index 6d458e46703..26dce33921f 100644 --- a/test/repository/plugin-vitest-project.test.ts +++ b/test/repository/plugin-vitest-project.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; @@ -11,23 +12,16 @@ const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const rootRequire = createRequire(path.join(repositoryRoot, "package.json")); const rootTypeScript = rootRequire.resolve("typescript/bin/tsc"); -type NpmDependencyTree = { - dependencies?: Record; - version?: string; +type PackageLock = { + packages?: Record; }; -function lockedDependencyTree(prefix?: string): NpmDependencyTree { - const prefixArgs = prefix ? ["--prefix", prefix] : []; - return JSON.parse( - execFileSync( - "npm", - [...prefixArgs, "ls", "--package-lock-only", "--json", "typescript", "vitest", "vite"], - { - cwd: repositoryRoot, - encoding: "utf8", - }, - ), - ) as NpmDependencyTree; +function readPackageLock(relativePath: string): PackageLock { + return JSON.parse(readFileSync(path.join(repositoryRoot, relativePath), "utf8")) as PackageLock; +} + +function lockedVersion(lock: PackageLock, dependency: string): string | undefined { + return lock.packages?.[`node_modules/${dependency}`]?.version; } function requiredLockedVersion(version: string | undefined, dependency: string): string { @@ -48,28 +42,18 @@ function listedTypeScriptFiles(configPath: string): string[] { } describe("plugin Vitest project contract", () => { - it("keeps the standalone plugin lock on the root test toolchain", () => { - const rootTree = lockedDependencyTree(); - const pluginTree = lockedDependencyTree("nemoclaw"); + // source-shape-contract: compatibility -- Root-driven plugin test tooling requires lockstep versions without installing plugin development dependencies + it.each(["typescript", "vite", "vitest"] as const)( + "keeps standalone plugin %s locked to the root test toolchain", + (dependency) => { + const rootLock = readPackageLock("package-lock.json"); + const pluginLock = readPackageLock("nemoclaw/package-lock.json"); - expect(requiredLockedVersion(pluginTree.dependencies?.vitest?.version, "plugin vitest")).toBe( - requiredLockedVersion(rootTree.dependencies?.vitest?.version, "root vitest"), - ); - expect( - requiredLockedVersion( - pluginTree.dependencies?.vitest?.dependencies?.vite?.version, - "plugin vite", - ), - ).toBe( - requiredLockedVersion( - rootTree.dependencies?.vitest?.dependencies?.vite?.version, - "root vite", - ), - ); - expect( - requiredLockedVersion(pluginTree.dependencies?.typescript?.version, "plugin typescript"), - ).toBe(requiredLockedVersion(rootTree.dependencies?.typescript?.version, "root typescript")); - }); + expect( + requiredLockedVersion(lockedVersion(pluginLock, dependency), `plugin ${dependency}`), + ).toBe(requiredLockedVersion(lockedVersion(rootLock, dependency), `root ${dependency}`)); + }, + ); it("typechecks plugin production and test sources without emitting tests", () => { const productionFiles = listedTypeScriptFiles("nemoclaw/tsconfig.json"); diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index d010fd6a718..a78333b08ec 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -907,8 +907,14 @@ export async function githubRequest( for (let attempt = 1; attempt <= attempts; attempt += 1) { let response: Response; + let requestSignal: AbortSignal | null = null; + let requestUsedRemainingBudget = false; try { const remainingMs = remainingBudget(); + requestUsedRemainingBudget = remainingMs !== undefined && remainingMs <= timeoutMs; + requestSignal = AbortSignal.timeout( + remainingMs === undefined ? timeoutMs : Math.min(timeoutMs, remainingMs), + ); response = await fetchImpl(`${API_ROOT}${path}`, { headers: { Accept: "application/vnd.github+json", @@ -916,11 +922,12 @@ export async function githubRequest( "User-Agent": "NemoClaw-base-image-publication-gate", "X-GitHub-Api-Version": "2022-11-28", }, - signal: AbortSignal.timeout( - remainingMs === undefined ? timeoutMs : Math.min(timeoutMs, remainingMs), - ), + signal: requestSignal, }); } catch { + if (requestUsedRemainingBudget && requestSignal?.aborted) { + throw new Error("GitHub API request exceeded its time budget"); + } remainingBudget(); if (attempt === attempts) { throw new Error(`GitHub API request failed after ${attempts} attempts`); From ef526443d11a68ac1eb20e8a84fbb1d451d2168e Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 8 Sep 2026 13:38:54 -0700 Subject: [PATCH 09/20] perf(ci): split slow tests across coverage shards Signed-off-by: Charan Jagwani --- .../actions/ci-cli-coverage-merge/action.yaml | 4 + ci/cli-test-timing-hints.json | 24 +- ci/test-file-size-budget.json | 1 - .../checks/report-cli-test-timing-drift.mts | 193 ++ .../classify-ci-failure-artifacts.test.ts | 6 + ...lassify-ci-failure-log-diagnostics.test.ts | 6 + ...ssify-ci-failure-process-lifecycle.test.ts | 6 + test/automation/classify-ci-failure.test.ts | 1141 ---------- .../pr-workflow-contract.test.ts | 9 +- test/e2e/mock-parity.json | 18 +- .../launch-agent-turn-failure-paths.test.ts | 6 + .../launch-agent-turn-readiness.test.ts | 6 + ...launch-agent-turn-session-evidence.test.ts | 6 + test/e2e/support/launch-agent-turn-suite.ts | 1543 +++++++++++++ .../support/launch-agent-turn-termios.test.ts | 6 + test/e2e/support/launch-agent-turn.test.ts | 1499 ------------ test/helpers/classify-ci-failure-suite.ts | 1164 ++++++++++ .../onboard-fresh-create-identity-suite.ts | 12 +- test/helpers/onboard-messaging-suite.ts | 2020 +++++++++++++++++ .../onboard-sandbox-build-suite.ts} | 723 +++--- test/helpers/vitest-watch-triggers.ts | 10 +- ...te-recovery-registration-identity.test.ts} | 2 +- ...sh-create-recovery-resume-identity.test.ts | 6 + ...oard-messaging-image-configuration.test.ts | 6 + .../onboard-messaging-provider.test.ts | 6 + .../onboard-messaging-recreation.test.ts | 6 + test/onboarding/onboard-messaging.test.ts | 1971 ---------------- .../onboard-sandbox-build-base.test.ts | 6 + .../onboard-sandbox-build-dashboard.test.ts | 6 + .../repository/cli-coverage-sequencer.test.ts | 6 + test/repository/cli-test-timing-drift.test.ts | 79 + test/repository/vitest-watch-triggers.test.ts | 10 +- 32 files changed, 5519 insertions(+), 4988 deletions(-) create mode 100755 scripts/checks/report-cli-test-timing-drift.mts create mode 100644 test/automation/classify-ci-failure-artifacts.test.ts create mode 100644 test/automation/classify-ci-failure-log-diagnostics.test.ts create mode 100644 test/automation/classify-ci-failure-process-lifecycle.test.ts delete mode 100644 test/automation/classify-ci-failure.test.ts create mode 100644 test/e2e/support/launch-agent-turn-failure-paths.test.ts create mode 100644 test/e2e/support/launch-agent-turn-readiness.test.ts create mode 100644 test/e2e/support/launch-agent-turn-session-evidence.test.ts create mode 100644 test/e2e/support/launch-agent-turn-suite.ts create mode 100644 test/e2e/support/launch-agent-turn-termios.test.ts delete mode 100644 test/e2e/support/launch-agent-turn.test.ts create mode 100644 test/helpers/classify-ci-failure-suite.ts create mode 100644 test/helpers/onboard-messaging-suite.ts rename test/{onboarding/onboard-sandbox-build.test.ts => helpers/onboard-sandbox-build-suite.ts} (53%) rename test/onboarding/{onboard-fresh-create-recovery-identity.test.ts => onboard-fresh-create-recovery-registration-identity.test.ts} (82%) create mode 100644 test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts create mode 100644 test/onboarding/onboard-messaging-image-configuration.test.ts create mode 100644 test/onboarding/onboard-messaging-provider.test.ts create mode 100644 test/onboarding/onboard-messaging-recreation.test.ts delete mode 100644 test/onboarding/onboard-messaging.test.ts create mode 100644 test/onboarding/onboard-sandbox-build-base.test.ts create mode 100644 test/onboarding/onboard-sandbox-build-dashboard.test.ts create mode 100644 test/repository/cli-test-timing-drift.test.ts diff --git a/.github/actions/ci-cli-coverage-merge/action.yaml b/.github/actions/ci-cli-coverage-merge/action.yaml index 4bb3b111c4c..7ac21fca139 100644 --- a/.github/actions/ci-cli-coverage-merge/action.yaml +++ b/.github/actions/ci-cli-coverage-merge/action.yaml @@ -99,6 +99,10 @@ runs: --coverage.exclude="test/**/*.ts" npx tsx scripts/check-coverage-ratchet.mts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json "CLI coverage" + - name: Report CLI timing drift + shell: bash + run: npx tsx scripts/checks/report-cli-test-timing-drift.mts coverage/cli/vitest-results.json ci/cli-test-timing-hints.json + - name: Upload CLI coverage report if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} uses: actions/upload-code-coverage@abb5995db9e0199b0e2bb9dbd136fce4cb1ec4d3 # v1 diff --git a/ci/cli-test-timing-hints.json b/ci/cli-test-timing-hints.json index 7ae28191749..f432b5bd7cf 100644 --- a/ci/cli-test-timing-hints.json +++ b/ci/cli-test-timing-hints.json @@ -31,6 +31,12 @@ "artifactId": 9958570313, "headSha": "5b74336486a8492ad87dd7eaba3d1bc06abbadf6", "recordedAt": "2026-09-04T23:53:49Z" + }, + { + "runId": 34013605812, + "artifactId": 9983530478, + "headSha": "be1452092af48f439531502870e826215b0d76f9", + "recordedAt": "2026-09-06T05:43:35Z" } ], "files": { @@ -74,7 +80,9 @@ "test/agents/openclaw/openclaw-device-self-approval-patch.test.ts": 8453, "test/agents/openclaw/openclaw-integrity-pin-base.test.ts": 10604, "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 13566, - "test/automation/classify-ci-failure.test.ts": 32936, + "test/automation/classify-ci-failure-artifacts.test.ts": 12274, + "test/automation/classify-ci-failure-log-diagnostics.test.ts": 8884, + "test/automation/classify-ci-failure-process-lifecycle.test.ts": 9105, "test/automation/pull-requests/analyze-pr-value-stream.test.ts": 10435, "test/automation/releases/release-latest-tag.test.ts": 17720, "test/automation/releases/reviewed-npm-audit-workflow.test.ts": 6952, @@ -115,7 +123,9 @@ "test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts": 6449, "test/e2e/support/e2e-semantic-phase-check.test.ts": 6122, "test/e2e/support/hermes-workflow-boundary.test.ts": 5155, - "test/e2e/support/launch-agent-turn.test.ts": 50714, + "test/e2e/support/launch-agent-turn-failure-paths.test.ts": 16363, + "test/e2e/support/launch-agent-turn-readiness.test.ts": 21380, + "test/e2e/support/launch-agent-turn-termios.test.ts": 10965, "test/e2e/support/managed-image-protected-runtime-workflow.test.ts": 5213, "test/e2e/support/mcp-workflow-boundary.test.ts": 8713, "test/e2e/support/portable-profile-systemctl-shim.test.ts": 8548, @@ -144,14 +154,17 @@ "test/onboarding/onboard-extra-provider-reconciliation.test.ts": 7915, "test/onboarding/onboard-fresh-create-cancellation-identity.test.ts": 49272, "test/onboarding/onboard-fresh-create-provider-identity.test.ts": 30497, - "test/onboarding/onboard-fresh-create-recovery-identity.test.ts": 44995, + "test/onboarding/onboard-fresh-create-recovery-registration-identity.test.ts": 27050, + "test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts": 17404, "test/onboarding/onboard-fsm-live-slices.test.ts": 29983, "test/onboarding/onboard-hermes-portable-provider-publication.test.ts": 7087, "test/onboarding/onboard-inference-reconciliation.test.ts": 12166, "test/onboarding/onboard-installer-restore-intent.test.ts": 11131, "test/onboarding/onboard-lifecycle.test.ts": 7372, "test/onboarding/onboard-managed-image-buildless-e2e.test.ts": 14585, - "test/onboarding/onboard-messaging.test.ts": 63014, + "test/onboarding/onboard-messaging-image-configuration.test.ts": 23003, + "test/onboarding/onboard-messaging-provider.test.ts": 19363, + "test/onboarding/onboard-messaging-recreation.test.ts": 18771, "test/onboarding/onboard-nim-memory-selection.test.ts": 5815, "test/onboarding/onboard-ollama-autostart.test.ts": 19183, "test/onboarding/onboard-ollama-context-floor.test.ts": 6122, @@ -160,7 +173,8 @@ "test/onboarding/onboard-provider-key-bridge.test.ts": 8723, "test/onboarding/onboard-remote-recreate-credential-reuse.test.ts": 5997, "test/onboarding/onboard-reservation-recreate.test.ts": 40804, - "test/onboarding/onboard-sandbox-build.test.ts": 23820, + "test/onboarding/onboard-sandbox-build-base.test.ts": 13428, + "test/onboarding/onboard-sandbox-build-dashboard.test.ts": 9564, "test/onboarding/onboard-sandbox-name.test.ts": 5063, "test/onboarding/onboard-sandbox-recreation.test.ts": 40625, "test/onboarding/onboard-selection-vllm.test.ts": 11283, diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 50644fdf2b7..efaeb8a39d2 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -8,7 +8,6 @@ "test/generation/generate-openclaw-config.test.ts": 1898, "test/installer-integration/install-preflight.test.ts": 3025, "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 4377, - "test/onboarding/onboard-messaging.test.ts": 1971, "test/onboarding/onboard-selection.test.ts": 4133 } } diff --git a/scripts/checks/report-cli-test-timing-drift.mts b/scripts/checks/report-cli-test-timing-drift.mts new file mode 100755 index 00000000000..74780305921 --- /dev/null +++ b/scripts/checks/report-cli-test-timing-drift.mts @@ -0,0 +1,193 @@ +#!/usr/bin/env -S npx tsx +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { appendFileSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +type TimingHints = { + defaultDurationMs: number; + files: Record; +}; + +type VitestFileResult = { + name?: unknown; + startTime?: unknown; + endTime?: unknown; +}; + +type VitestReport = { + testResults?: unknown; +}; + +export type TimingDrift = { + file: string; + hintMs?: number; + kind: "faster" | "slower" | "unprofiled"; + observedMs: number; +}; + +const MINIMUM_DELTA_MS = 10_000; +const HINT_RATIO = 2; +const UNPROFILED_RATIO = 3; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeReportPath(name: string, repoRoot: string): string | undefined { + const absoluteName = path.isAbsolute(name) ? path.resolve(name) : path.resolve(repoRoot, name); + const relativeName = path.relative(repoRoot, absoluteName).split(path.sep).join("/"); + if (relativeName === "" || relativeName === ".." || relativeName.startsWith("../")) { + return undefined; + } + return relativeName; +} + +function parseHints(value: unknown): TimingHints { + if (!isRecord(value) || !Number.isSafeInteger(value.defaultDurationMs)) { + throw new Error("Invalid CLI timing hints"); + } + if (!isRecord(value.files)) { + throw new Error("Invalid CLI timing hint file map"); + } + + const defaultDurationMs = Number(value.defaultDurationMs); + if (defaultDurationMs <= 0) { + throw new Error("Invalid CLI timing hint default duration"); + } + + const files: Record = {}; + for (const [file, duration] of Object.entries(value.files)) { + if (!Number.isSafeInteger(duration) || Number(duration) <= defaultDurationMs) { + throw new Error(`Invalid CLI timing hint: ${file}`); + } + files[file] = Number(duration); + } + return { defaultDurationMs, files }; +} + +function parseFileResults(value: VitestReport, repoRoot: string): Map { + if (!Array.isArray(value.testResults)) { + throw new Error("Invalid Vitest timing report"); + } + + const durations = new Map(); + for (const candidate of value.testResults) { + if (!isRecord(candidate)) continue; + const { name, startTime, endTime } = candidate as VitestFileResult; + if ( + typeof name !== "string" || + typeof startTime !== "number" || + !Number.isFinite(startTime) || + typeof endTime !== "number" || + !Number.isFinite(endTime) || + endTime < startTime + ) { + continue; + } + const file = normalizeReportPath(name, repoRoot); + if (!file) continue; + durations.set(file, Math.max(durations.get(file) ?? 0, Math.round(endTime - startTime))); + } + return durations; +} + +export function findCliTestTimingDrift( + reportValue: VitestReport, + hintsValue: unknown, + repoRoot: string, +): TimingDrift[] { + const hints = parseHints(hintsValue); + const durations = parseFileResults(reportValue, path.resolve(repoRoot)); + const drift: TimingDrift[] = []; + + for (const [file, observedMs] of durations) { + const hintMs = hints.files[file]; + if (hintMs === undefined) { + if (observedMs >= hints.defaultDurationMs * UNPROFILED_RATIO) { + drift.push({ file, kind: "unprofiled", observedMs }); + } + continue; + } + + const deltaMs = Math.abs(observedMs - hintMs); + if (deltaMs < MINIMUM_DELTA_MS) continue; + if (observedMs >= hintMs * HINT_RATIO) { + drift.push({ file, hintMs, kind: "slower", observedMs }); + } else if (hintMs >= observedMs * HINT_RATIO) { + drift.push({ file, hintMs, kind: "faster", observedMs }); + } + } + + return drift.sort( + (left, right) => right.observedMs - left.observedMs || left.file.localeCompare(right.file), + ); +} + +function escapeWorkflowCommand(value: string): string { + return value.replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A"); +} + +function formatDuration(milliseconds: number): string { + return `${(milliseconds / 1_000).toFixed(1)}s`; +} + +function driftMessage(item: TimingDrift): string { + if (item.kind === "unprofiled") { + return `Unprofiled test file took ${formatDuration(item.observedMs)}; refresh CLI timing hints`; + } + return `Test file ran ${item.kind} than its timing hint: observed ${formatDuration(item.observedMs)}, hint ${formatDuration(item.hintMs!)}`; +} + +export function formatCliTestTimingDriftSummary(drift: readonly TimingDrift[]): string { + if (drift.length === 0) { + return "### CLI test timing drift\n\nNo material timing-hint drift detected.\n"; + } + const rows = drift + .slice(0, 20) + .map( + (item) => + `| \`${item.file}\` | ${item.kind} | ${formatDuration(item.observedMs)} | ${item.hintMs === undefined ? "—" : formatDuration(item.hintMs)} |`, + ) + .join("\n"); + return `### CLI test timing drift\n\n${drift.length} file(s) need a timing-hint refresh. This report is advisory.\n\n| File | Drift | Observed | Hint |\n| --- | --- | ---: | ---: |\n${rows}\n`; +} + +function loadJson(file: string): unknown { + try { + return JSON.parse(readFileSync(path.resolve(file), "utf8")); + } catch (cause) { + throw new Error(`Failed to load ${file}`, { cause }); + } +} + +function main(): void { + const [reportFile, hintsFile] = process.argv.slice(2); + if (!reportFile || !hintsFile) { + throw new Error( + "Usage: report-cli-test-timing-drift.mts ", + ); + } + + const drift = findCliTestTimingDrift( + loadJson(reportFile) as VitestReport, + loadJson(hintsFile), + process.cwd(), + ); + const summary = formatCliTestTimingDriftSummary(drift); + process.stdout.write(summary); + for (const item of drift) { + process.stdout.write( + `::warning file=${escapeWorkflowCommand(item.file)},title=CLI test timing drift::${escapeWorkflowCommand(driftMessage(item))}\n`, + ); + } + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } +} + +if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? "")) { + main(); +} diff --git a/test/automation/classify-ci-failure-artifacts.test.ts b/test/automation/classify-ci-failure-artifacts.test.ts new file mode 100644 index 00000000000..f4664b41987 --- /dev/null +++ b/test/automation/classify-ci-failure-artifacts.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerClassifyCiFailureTests } from "../helpers/classify-ci-failure-suite"; + +registerClassifyCiFailureTests("artifacts"); diff --git a/test/automation/classify-ci-failure-log-diagnostics.test.ts b/test/automation/classify-ci-failure-log-diagnostics.test.ts new file mode 100644 index 00000000000..ff57124ccd6 --- /dev/null +++ b/test/automation/classify-ci-failure-log-diagnostics.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerClassifyCiFailureTests } from "../helpers/classify-ci-failure-suite"; + +registerClassifyCiFailureTests("logs"); diff --git a/test/automation/classify-ci-failure-process-lifecycle.test.ts b/test/automation/classify-ci-failure-process-lifecycle.test.ts new file mode 100644 index 00000000000..1642ddc3b69 --- /dev/null +++ b/test/automation/classify-ci-failure-process-lifecycle.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerClassifyCiFailureTests } from "../helpers/classify-ci-failure-suite"; + +registerClassifyCiFailureTests("process-lifecycle"); diff --git a/test/automation/classify-ci-failure.test.ts b/test/automation/classify-ci-failure.test.ts deleted file mode 100644 index 5dcbe0cbbe7..00000000000 --- a/test/automation/classify-ci-failure.test.ts +++ /dev/null @@ -1,1141 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -import { spawn, spawnSync } from "node:child_process"; -import { - chmodSync, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; -import { afterEach, describe, expect, test, vi } from "vitest"; - -import { - execute, - type GhResolverFilesystem, - resolveProductionGhExecutableForTest, -} from "../../.agents/skills/nemoclaw-maintainer-classify-ci-failure/scripts/classify-ci-failure.mts"; -import { artifactZip, artifactZipEntryDataOffset } from "../helpers/artifact-zip"; -const script = resolve( - ".agents/skills/nemoclaw-maintainer-classify-ci-failure/scripts/classify-ci-failure.mts", -); -const roots: string[] = []; -const uid = process.getuid?.() ?? "unknown"; -const REDACTION_CASES = [ - [ - "Slack bot token", - ["xoxb", "123456789012", "123456789012", "abcdefghijklmnopqrstuvwx"].join("-"), - ["xoxb", "123456789012", "123456789012", "abcdefghijklmnopqrstuvwx"].join("-"), - ], - [ - "Slack app token", - ["xapp", "1", "A1234567890", "1234567890123", "abcdefghijklmnopqrstuvwx"].join("-"), - ["xapp", "1", "A1234567890", "1234567890123", "abcdefghijklmnopqrstuvwx"].join("-"), - ], - [ - "OpenAI API key", - "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", - "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", - ], - [ - "OpenAI project API key", - "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", - "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", - ], - [ - "NVIDIA API key", - "nvapi-abcdefghijklmnopqrstuvwxyz0123456789", - "nvapi-abcdefghijklmnopqrstuvwxyz0123456789", - ], - [ - "NVIDIA Cloud Functions key", - "nvcf-abcdefghijklmnopqrstuvwxyz0123456789", - "nvcf-abcdefghijklmnopqrstuvwxyz0123456789", - ], - [ - "npm access token", - "npm_abcdefghijklmnopqrstuvwxyz0123456789", - "npm_abcdefghijklmnopqrstuvwxyz0123456789", - ], - ["quoted JSON secret", '"client_secret": "json-secret-value"', "json-secret-value"], - ["quoted JSON token", '"refresh-token": "json-token-value"', "json-token-value"], - ["quoted JSON password", '"password": "json-password-value"', "json-password-value"], - ["quoted JSON API key", '"api-key": "json-api-key-value"', "json-api-key-value"], - [ - "quoted JSON authorization", - '"authorization": "Basic json-authorization-value"', - "json-authorization-value", - ], -] as const; -afterEach(() => { - for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); -}); -function fixture(log: string, result?: Record, archive?: Buffer) { - const root = mkdtempSync(join(tmpdir(), "classify-ci-test-")); - roots.push(root); - const bin = join(root, "bin"); - mkdirSync(bin, { recursive: true }); - const zip = join(root, "artifact.zip"); - writeFileSync( - zip, - archive ?? - artifactZip([{ name: "sample.result.json", contents: JSON.stringify(result ?? {}) }]), - ); - const gh = join(bin, "gh"); - const fake = [ - `#!${process.execPath}`, - "const fs=require('fs');", - "const {spawn}=require('node:child_process');", - "const block=()=>{ if(process.env.EXIT_PROMPTLY_ON_TERM) { fs.writeFileSync(process.env.BLOCK_WRAPPER_MARKER,String(process.ppid)); fs.writeFileSync(process.env.BLOCK_COMMAND_MARKER,String(process.pid)); process.once('SIGTERM',()=>{ fs.appendFileSync(process.env.SIGNAL_LOG,'SIGTERM\\n'); process.exit(0); }); } if(process.env.BLOCK_DESCENDANT_MARKER) { process.on('SIGINT',()=>{}); process.on('SIGTERM',()=>{}); } if(process.env.BLOCK_GROUP_MARKER) fs.writeFileSync(process.env.BLOCK_GROUP_MARKER,String(process.pid)); if(process.env.BLOCK_DESCENDANT_MARKER) { const child=spawn(process.execPath,['-e',\"process.on('SIGINT',()=>{}); process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)\"],{stdio:process.env.EXIT_GROUP_LEADER ? ['ignore',process.stdout,process.stderr] : 'ignore'}); fs.writeFileSync(process.env.BLOCK_DESCENDANT_MARKER,String(child.pid)); if(process.env.EXIT_GROUP_LEADER) { fs.writeFileSync(process.env.BLOCK_MARKER,'ready'); process.exit(0); } } fs.writeFileSync(process.env.BLOCK_MARKER,'ready'); setInterval(()=>{},1000); };", - "const a=process.argv.slice(2).join(' ');", - "fs.appendFileSync(process.env.GH_CALLS,a+'\\n');", - "fs.appendFileSync(process.env.GH_ENV,JSON.stringify({GH_HOST:process.env.GH_HOST,GH_ENTERPRISE_TOKEN:process.env.GH_ENTERPRISE_TOKEN,GH_TOKEN:process.env.GH_TOKEN,GITHUB_TOKEN:process.env.GITHUB_TOKEN})+'\\n');", - "if(a==='api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123') { if(process.env.BLOCK_METADATA) { block(); } else if(process.env.FAIL_METADATA) { console.error(process.env.FAIL_METADATA); process.exit(8); } else console.log(JSON.stringify({id:123,run_id:456,name:process.env.JOB_NAME||'CLI tests',status:'completed',conclusion:'failure',html_url:process.env.JOB_URL||'https://example.test/job'})); }", - "else if(a==='api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123/logs') { if(process.env.BLOCK_LOG) { block(); } else if(process.env.FAIL_LOG) { console.error(process.env.FAIL_LOG); process.exit(8); } else process.stdout.write('discarded\\n'.repeat(Number(process.env.LOG_PREFIX_LINES||0))+process.env.TEST_LOG); }", - "else if(a==='api --hostname github.com --include repos/NVIDIA/NemoClaw/actions/runs/456/artifacts?per_page=100&page=1') { if(process.env.FAIL_INVENTORY) { console.error(process.env.FAIL_INVENTORY); process.exit(8); } const artifacts=process.env.DUPLICATE_ARTIFACTS ? [{id:789,name:'results',size_in_bytes:Number(process.env.ZIP_SIZE)},{id:790,name:'results',size_in_bytes:Number(process.env.ZIP_SIZE)}] : [{id:789,name:'results',size_in_bytes:Number(process.env.ZIP_SIZE)}]; process.stdout.write('HTTP/2 200\\r\\n\\r\\n'+JSON.stringify({total_count:artifacts.length,artifacts})); }", - "else if(a==='api --hostname github.com repos/NVIDIA/NemoClaw/actions/artifacts/789/zip') { if(process.env.BLOCK_ARTIFACT) { block(); } else if(process.env.STREAM_BYTES) process.stdout.write(Buffer.alloc(Number(process.env.STREAM_BYTES))); else process.stdout.write(fs.readFileSync(process.env.ZIP_PATH)); }", - "else { console.error('unexpected '+a); process.exit(9); }", - ].join("\n"); - writeFileSync(gh, fake); - chmodSync(gh, 0o755); - const dd = join(bin, "dd"); - writeFileSync( - dd, - [ - `#!${process.execPath}`, - "const {spawnSync}=require('node:child_process');", - "if(process.env.FAIL_PROBE_DD && process.argv.includes('count=1')) process.exit(7);", - "const result=spawnSync('/usr/bin/dd',process.argv.slice(2),{stdio:'inherit'});", - "process.exit(result.status ?? 1);", - ].join("\n"), - ); - chmodSync(dd, 0o755); - const wc = join(bin, "wc"); - writeFileSync( - wc, - [ - `#!${process.execPath}`, - "const {spawnSync}=require('node:child_process');", - "if(process.env.FAIL_PROBE_WC && process.argv.includes('-c')) process.exit(7);", - "const result=spawnSync('/usr/bin/wc',process.argv.slice(2),{stdio:'inherit'});", - "process.exit(result.status ?? 1);", - ].join("\n"), - ); - chmodSync(wc, 0o755); - const env: NodeJS.ProcessEnv = { - ...process.env, - PATH: bin + ":" + process.env.PATH, - TMPDIR: root, - GH_CALLS: join(root, "calls"), - GH_ENV: join(root, "environment"), - TEST_DD: dd, - TEST_GH: gh, - TEST_WC: wc, - TEST_LOG: log, - LOG_PREFIX_LINES: "0", - ZIP_PATH: zip, - ZIP_SIZE: String(statSync(zip).size), - }; - return { root, env }; -} -const classifierArgs = (extra: string[] = []) => [ - "--experimental-strip-types", - "--no-warnings", - script, - "--job-id", - "123", - ...extra, -]; -function importedClassifierArgs(env: NodeJS.ProcessEnv, extra: string[]): string[] { - const optionValue = (name: string): string | undefined => { - const index = extra.indexOf(name); - return index < 0 ? undefined : extra[index + 1]; - }; - const repo = optionValue("--repo"); - const artifactName = optionValue("--artifact-name"); - const maxLines = optionValue("--max-lines"); - const clipMode = optionValue("--clip-mode"); - const input: Record = { - workdir: process.cwd(), - jobId: "123", - ...(repo === undefined ? {} : { repo }), - ...(artifactName === undefined ? {} : { artifactName }), - ...(maxLines === undefined ? {} : { maxLines: Number(maxLines) }), - ...(clipMode === undefined ? {} : { clipMode }), - }; - return [ - "--experimental-strip-types", - "--no-warnings", - "--input-type=module", - "-e", - [ - `import { classifyCiFailureWithRuntimeForTest } from ${JSON.stringify(new URL("file://" + script).href)};`, - `const input = ${JSON.stringify(input)};`, - "const environment = { ...process.env };", - "const executables = { bash: '/usr/bin/bash', dd: process.env.TEST_DD || '/usr/bin/dd', gh: process.env.TEST_GH, stat: '/usr/bin/stat', tail: '/usr/bin/tail', wc: process.env.TEST_WC || '/usr/bin/wc' };", - "const timeouts = { metadataMs: process.env.TEST_METADATA_TIMEOUT_MS ? Number(process.env.TEST_METADATA_TIMEOUT_MS) : undefined, logMs: process.env.TEST_LOG_TIMEOUT_MS ? Number(process.env.TEST_LOG_TIMEOUT_MS) : undefined, artifactMs: process.env.TEST_ARTIFACT_TIMEOUT_MS ? Number(process.env.TEST_ARTIFACT_TIMEOUT_MS) : undefined };", - "void classifyCiFailureWithRuntimeForTest(input, { executables, environment, timeouts }).then((value) => console.log(JSON.stringify(value, null, 2))).catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; });", - ].join("\n"), - ]; -} -function run(env: NodeJS.ProcessEnv, extra: string[] = []) { - const useCli = - extra.some((value) => value === "--unknown") || - extra.some((value) => value === "--repo") || - extra.some((value) => value === "0" || value === "1.5" || value === "501"); - return spawnSync( - process.execPath, - useCli ? classifierArgs(extra) : importedClassifierArgs(env, extra), - { - encoding: "utf8", - env, - }, - ); -} -function classifierTemporaryDirectories(prefix: "nemoclaw-ci-log." | "nemoclaw-ci-classify.") { - const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; - return existsSync(temporaryRoot) - ? readdirSync(temporaryRoot).filter((name) => name.startsWith(prefix)) - : []; -} -async function waitForFile(path: string): Promise { - await vi.waitFor(() => expect(existsSync(path)).toBe(true), { timeout: 2_000, interval: 10 }); -} -type FakePath = { - type: "directory" | "file" | "symlink"; - mode?: number; - uid?: number; - realpath?: string; -}; -function resolverFilesystem(entries: Record): GhResolverFilesystem { - const missing = (path: string): never => { - const error = new Error("missing " + path) as NodeJS.ErrnoException; - error.code = "ENOENT"; - throw error; - }; - const entry = (path: string): FakePath => entries[path] ?? missing(path); - return { - lstat: (path) => { - const value = entry(path); - return { - isDirectory: () => value.type === "directory", - isFile: () => value.type === "file", - isSymbolicLink: () => value.type === "symlink", - mode: value.mode ?? 0o755, - uid: value.uid ?? 0, - }; - }, - realpath: (path) => entry(path).realpath ?? path, - access: (path) => { - entry(path); - }, - }; -} -function trustedDirectories(home = "/home/tester"): Record { - return { - "/": { type: "directory" }, - "/usr": { type: "directory" }, - "/usr/bin": { type: "directory" }, - "/usr/local": { type: "directory" }, - "/usr/local/bin": { type: "directory" }, - "/home": { type: "directory" }, - [home]: { type: "directory", uid: 1000, mode: 0o750 }, - [join(home, ".local")]: { type: "directory", uid: 1000, mode: 0o700 }, - [join(home, ".local", "bin")]: { type: "directory", uid: 1000 }, - }; -} - -describe("GitHub CLI production resolver", () => { - test("selects a safe user-local executable without consulting PATH", () => { - const home = "/home/tester"; - const gh = join(home, ".local", "bin", "gh"); - const filesystem = resolverFilesystem({ - ...trustedDirectories(home), - [gh]: { type: "file", uid: 1000 }, - "/attacker/gh": { type: "file", uid: 1000 }, - }); - expect( - resolveProductionGhExecutableForTest({ HOME: home, PATH: "/attacker" }, 1000, filesystem), - ).toBe(gh); - }); - - test("skips a writable system candidate and selects a safe user-local executable", () => { - const home = "/home/tester"; - const gh = join(home, ".local", "bin", "gh"); - const filesystem = resolverFilesystem({ - ...trustedDirectories(home), - "/usr/bin/gh": { type: "file", mode: 0o777 }, - [gh]: { type: "file", uid: 1000 }, - }); - expect(resolveProductionGhExecutableForTest({ HOME: home }, 1000, filesystem)).toBe(gh); - }); - - test.each<[string, Record]>([ - [ - "writable path component", - { "/home/tester/.local": { type: "directory", uid: 1000, mode: 0o770 } }, - ], - ["foreign owner", { "/home/tester/.local/bin/gh": { type: "file", uid: 2000 } }], - [ - "symlink candidate", - { - "/home/tester/.local/bin/gh": { - type: "symlink", - uid: 1000, - realpath: "/attacker/gh", - }, - "/attacker/gh": { type: "file", uid: 1000 }, - }, - ], - ])("rejects a user-local executable with a %s", (_case, overrides) => { - const home = "/home/tester"; - const gh = join(home, ".local", "bin", "gh"); - const filesystem = resolverFilesystem({ - ...trustedDirectories(home), - [gh]: { type: "file", uid: 1000 }, - ...overrides, - }); - expect(() => - resolveProductionGhExecutableForTest({ HOME: home, PATH: "/attacker" }, 1000, filesystem), - ).toThrow("Could not find a trusted GitHub CLI executable"); - }); -}); - -describe.skipIf(process.platform !== "linux")("CI failure classifier process", () => { - test("redacts credentials from classified diagnostic output", () => { - const secrets = [ - "Authorization: Bearer full authorization value with spaces", - "> X-API-Key: x-header-secret", - "request: Api-Key: api-header-secret", - "cOoKiE: log-cookie-secret=plain", - "> sEt-CoOkIe: log-set-cookie-secret=prefixed; Secure", - "< Set-Cookie: log-response-cookie-secret=prefixed; HttpOnly", - "< Cookie: log-response-request-cookie-secret=prefixed", - "< Authorization: Bearer log-response-auth-secret", - "< API-Key: log-response-api-key-secret", - "request: COOKIE: log-request-cookie-secret=prefixed", - "unrelated diagnostic text", - "https://user:password@example.test/path", - "AWS_ACCESS_KEY_ID=AKIAEXAMPLEVALUE", - "BUILD_TOKEN=token-value", - "CLIENT_SECRET: secret-value", - "DB_PASSWORD=password-value", - "SERVICE_API_KEY=api-key-value", - "ghp_alpha gho_beta ghu_gamma ghs_delta ghr_epsilon github_pat_zeta", - ].join("\n"); - const known = fixture(`${secrets}\nAssertionError: expected true`); - const r = run(known.env); - expect(r.status).toBe(0); - expect(r.stdout).not.toMatch( - /full authorization|x-header-secret|api-header-secret|log-cookie-secret|log-set-cookie-secret|log-response-cookie-secret|log-response-request-cookie-secret|log-response-auth-secret|log-response-api-key-secret|log-request-cookie-secret|user:password|AKIAEXAMPLE|token-value|secret-value|password-value|api-key-value|ghp_alpha|gho_beta|ghu_gamma|ghs_delta|ghr_epsilon|github_pat_zeta/, - ); - expect(r.stdout).toContain("> X-API-Key: [REDACTED]"); - expect(r.stdout).toContain("request: Api-Key: [REDACTED]"); - expect(r.stdout).toContain("cOoKiE: [REDACTED]"); - expect(r.stdout).toContain("> sEt-CoOkIe: [REDACTED]"); - expect(r.stdout).toContain("< Set-Cookie: [REDACTED]"); - expect(r.stdout).toContain("< Cookie: [REDACTED]"); - expect(r.stdout).toContain("< Authorization: [REDACTED]"); - expect(r.stdout).toContain("< API-Key: [REDACTED]"); - expect(r.stdout).toContain("request: COOKIE: [REDACTED]"); - expect(r.stdout).toContain("unrelated diagnostic text"); - expect(r.stdout.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(13); - }); - test("does not pass executable lookup or startup injection to classifier subprocesses", () => { - const item = fixture("AssertionError: expected true"); - const attackerBin = join(item.root, "attacker-bin"); - const executableMarker = join(item.root, "attacker-executable-ran"); - const bashMarker = join(item.root, "bash-env-ran"); - const nodeMarker = join(item.root, "node-options-ran"); - mkdirSync(attackerBin); - const bashExecutable = join(attackerBin, "bash"); - const ghExecutable = join(attackerBin, "gh"); - writeFileSync( - bashExecutable, - `#!${process.execPath}\nrequire("node:fs").appendFileSync(${JSON.stringify(executableMarker)}, "bash\n");`, - ); - writeFileSync( - ghExecutable, - `#!${process.execPath}\nrequire("node:fs").appendFileSync(${JSON.stringify(executableMarker)}, "gh\n");`, - ); - chmodSync(bashExecutable, 0o755); - chmodSync(ghExecutable, 0o755); - const bashHook = join(item.root, "bash-env"); - writeFileSync(bashHook, `printf injected >> ${JSON.stringify(bashMarker)}`); - const nodeHook = join(item.root, "node-options.mjs"); - writeFileSync( - nodeHook, - `import { appendFileSync } from "node:fs"; appendFileSync(${JSON.stringify(nodeMarker)}, ${JSON.stringify("loaded\n")});`, - ); - item.env.PATH = attackerBin; - item.env.BASH_ENV = bashHook; - item.env.ENV = bashHook; - item.env.NODE_OPTIONS = `--import=${nodeHook}`; - const result = run(item.env); - expect(result.status, result.stderr).toBe(0); - expect(JSON.parse(result.stdout).result).toBe("classified"); - expect(existsSync(executableMarker)).toBe(false); - expect(existsSync(bashMarker)).toBe(false); - expect(readFileSync(nodeMarker, "utf8").trim().split("\n")).toEqual(["loaded"]); - }); - - test("binds GitHub API reads to github.com without forwarding enterprise host credentials", () => { - const item = fixture("AssertionError: expected true"); - item.env.GH_HOST = "attacker.invalid"; - item.env.GH_ENTERPRISE_TOKEN = "enterprise-secret"; - item.env.GH_TOKEN = "github-token"; - item.env.GITHUB_TOKEN = "github-actions-token"; - - const result = run(item.env, ["--artifact-name", "results"]); - - expect(result.status, result.stderr).toBe(0); - expect(readFileSync(item.env.GH_CALLS!, "utf8").trim().split("\n")).toEqual([ - "api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123", - "api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123/logs", - "api --hostname github.com --include repos/NVIDIA/NemoClaw/actions/runs/456/artifacts?per_page=100&page=1", - "api --hostname github.com repos/NVIDIA/NemoClaw/actions/artifacts/789/zip", - ]); - const environments = readFileSync(item.env.GH_ENV!, "utf8") - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - expect(environments).toHaveLength(4); - expect(environments).toEqual( - environments.map(() => ({ GH_TOKEN: "github-token", GITHUB_TOKEN: "github-actions-token" })), - ); - }); - - test("classifies an AssertionError as a test failure", () => { - const item = fixture("AssertionError: expected true"); - const result = run(item.env); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.result).toBe("classified"); - expect(value.categories).toContain("test-failure"); - }); - test("returns unclassified when output has no failure signature", () => { - const item = fixture("ordinary unrelated output"); - const result = run(item.env); - expect(result.status, result.stderr).toBe(0); - expect(JSON.parse(result.stdout).result).toBe("unclassified"); - }); - test.each(REDACTION_CASES)( - "redacts a standalone %s from returned process logs", - (_name, secret, exposed) => { - const item = fixture( - ["diagnostic before", secret, "AssertionError: expected true", "diagnostic after"].join( - "\n", - ), - ); - const result = run(item.env); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("[REDACTED]"); - expect(result.stdout).not.toContain(exposed ?? secret); - }, - ); - test("redacts signed URL credentials in selected logs without changing other query fields", () => { - const signedUrl = - "https://storage.test/object?keep=visible&X-Amz-Credential=aws-credential&X-Amz-Signature=aws-signature&X-Amz-Security-Token=aws-session&X-Goog-Credential=google-credential&X-Goog-Signature=google-signature&sig=azure-signature&access_token=oauth-token&token=common-token&tail=retained#fragment"; - const item = fixture(`${signedUrl}\nAssertionError: expected true`); - const result = run(item.env); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.log.stdout).toContain( - "?keep=visible&X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&X-Amz-Security-Token=[REDACTED]&X-Goog-Credential=[REDACTED]&X-Goog-Signature=[REDACTED]&sig=[REDACTED]&access_token=[REDACTED]&token=[REDACTED]&tail=retained#fragment", - ); - expect(result.stdout).not.toMatch( - /aws-credential|aws-signature|aws-session|google-credential|google-signature|azure-signature|oauth-token|common-token/, - ); - }); - test("redacts artifact errors and omits ignored command data", () => { - const item = fixture("ordinary output", { - exitCode: 1, - error: - "> X-API-Key: artifact-error-key\n< Set-Cookie: artifact-response-cookie-secret=opaque; HttpOnly\nrequest: Set-Cookie: artifact-error-cookie=opaque; HttpOnly\nunrelated artifact error\ndownload failed https://storage.test/object?X-Amz-Credential=error-credential&X-Amz-Signature=error-signature&X-Amz-Security-Token=error-session&keep=error-visible", - command: - "request: Api-Key: artifact-command-key\n> COOKIE: artifact-command-cookie=opaque\nunrelated artifact command\ncurl 'https://storage.test/object?X-Goog-Credential=command-credential&X-Goog-Signature=command-google-signature&sig=command-signature&access_token=command-access&token=command-token&keep=command-visible'", - }); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const failure = JSON.parse(result.stdout).artifact.failures[0]; - expect(failure.error).toContain("> X-API-Key: [REDACTED]"); - expect(failure.error).toContain("< Set-Cookie: [REDACTED]"); - expect(failure.error).toContain("request: Set-Cookie: [REDACTED]"); - expect(failure.error).toContain("unrelated artifact error"); - expect(failure).not.toHaveProperty("command"); - expect(failure.error).toContain( - "?X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&X-Amz-Security-Token=[REDACTED]&keep=error-visible", - ); - expect(result.stdout).not.toMatch( - /artifact-error-key|artifact-response-cookie-secret|artifact-error-cookie|artifact-command-key|artifact-command-cookie|error-credential|error-signature|error-session|command-credential|command-google-signature|command-signature|command-access|command-token/, - ); - }); - test("redacts and bounds dynamic GitHub job metadata", () => { - const item = fixture("ordinary output"); - const nameSecret = "metadata-name-secret"; - const urlSecret = "metadata-url-secret"; - item.env.JOB_NAME = `${"n".repeat(600)} BUILD_TOKEN=${nameSecret}`; - item.env.JOB_URL = `https://example.test/job/${"u".repeat(2100)}?keep=visible&token=${urlSecret}`; - const result = run(item.env); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.job.name).toContain("[REDACTED]"); - expect(value.job.name.length).toBeLessThanOrEqual(500); - expect(value.job.url).toContain("token=[REDACTED]"); - expect(value.job.url.length).toBeLessThanOrEqual(2000); - expect(result.stdout).not.toContain(nameSecret); - expect(result.stdout).not.toContain(urlSecret); - }); - test("rejects an alternate valid repository before invoking GitHub", () => { - const item = fixture("unused"); - const r = spawnSync( - process.execPath, - importedClassifierArgs(item.env, ["--repo", "NVIDIA/Other"]), - { - encoding: "utf8", - env: item.env, - }, - ); - expect(r.status).toBe(1); - expect(r.stderr).toContain("repo must be NVIDIA/NemoClaw"); - expect(readdirSync(item.root)).not.toContain("calls"); - }); - test("bounds streamed logs before filtering", () => { - const item = fixture("AssertionError: retained tail"); - item.env.LOG_PREFIX_LINES = "500000"; - const r = run(item.env); - expect(r.status, r.stderr).toBe(0); - const value = JSON.parse(r.stdout); - expect(value.log.truncated).toBe(true); - expect(value.log.truncationReasons).toContain("source-log-bounded-before-filtering"); - expect(value.log.stdout).toContain("retained tail"); - expect(value.log.stdout.length).toBeLessThan(40000); - const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; - expect( - existsSync(temporaryRoot) - ? readdirSync(temporaryRoot).filter((name) => name.startsWith("nemoclaw-ci-log.")) - : [], - ).toEqual([]); - }); - test.each([ - [["--unknown", "value"], "Unknown option --unknown"], - [["--repo", "NVIDIA/NemoClaw"], "Unknown option --repo"], - [["--max-lines", "0"], "--max-lines must be an integer from 1 through 500"], - [["--max-lines", "1.5"], "--max-lines must be an integer from 1 through 500"], - [["--max-lines", "501"], "--max-lines must be an integer from 1 through 500"], - ])("rejects invalid flags before invoking GitHub", (extra, message) => { - const item = fixture("unused"); - const r = run(item.env, extra); - expect(r.status).toBe(1); - expect(r.stderr).toContain(message); - expect(readdirSync(item.root)).not.toContain("calls"); - }); - test.each([ - ["authentication", "authentication required BUILD_TOKEN=log-access-secret"], - ["authorization", "HTTP 403 resource not accessible BUILD_TOKEN=log-access-secret"], - ])("fails log acquisition with bounded, redacted %s recovery guidance", (_kind, failure) => { - const item = fixture("unused"); - item.env.FAIL_LOG = failure; - const result = run(item.env); - expect(result.status).not.toBe(0); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("GitHub access failed. Run gh auth status"); - expect(result.stderr).toContain("ask the user to correct authentication, authorization, SSO"); - expect(result.stderr).toContain("BUILD_TOKEN=[REDACTED]"); - expect(result.stderr).not.toContain("log-access-secret"); - expect(result.stderr.length).toBeLessThanOrEqual(2001); - expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); - }); - - test.each([ - [ - "job metadata", - "FAIL_METADATA", - [], - "authentication required BUILD_TOKEN=metadata-access-secret", - ], - [ - "artifact inventory", - "FAIL_INVENTORY", - ["--artifact-name", "results"], - "HTTP 403 resource not accessible BUILD_TOKEN=inventory-access-secret", - ], - ] as const)( - "fails %s access with bounded, redacted recovery guidance", - (_kind, variable, extra, failure) => { - const item = fixture("ordinary output"); - item.env[variable] = failure; - const result = run(item.env, [...extra]); - expect(result.status).not.toBe(0); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("GitHub access failed. Run gh auth status"); - expect(result.stderr).toContain( - "ask the user to correct authentication, authorization, SSO, or token scope", - ); - expect(result.stderr).toContain("BUILD_TOKEN=[REDACTED]"); - expect(result.stderr).not.toMatch(/metadata-access-secret|inventory-access-secret/u); - expect(result.stderr.length).toBeLessThanOrEqual(2001); - }, - ); - - test.each([ - [ - "job metadata", - "BLOCK_METADATA", - "TEST_METADATA_TIMEOUT_MS", - [], - "GitHub job metadata read timed out after 30 seconds", - ], - [ - "job log", - "BLOCK_LOG", - "TEST_LOG_TIMEOUT_MS", - [], - "GitHub job log read timed out after 60 seconds", - ], - [ - "artifact", - "BLOCK_ARTIFACT", - "TEST_ARTIFACT_TIMEOUT_MS", - ["--artifact-name", "results"], - "GitHub artifact read timed out after 60 seconds", - ], - ] as const)( - "reports a bounded silent %s timeout and removes temporary data", - (_operation, block, timeoutVariable, extra, diagnostic) => { - const item = fixture("ordinary output"); - item.env[block] = "1"; - item.env.BLOCK_MARKER = join(item.root, "blocked"); - item.env[timeoutVariable] = "50"; - const result = run(item.env, [...extra]); - expect(result.status).not.toBe(0); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain(diagnostic); - expect(result.stderr).toContain( - "Check GitHub availability, then retry CI failure classification.", - ); - expect(result.stderr).not.toContain("repos/NVIDIA/NemoClaw/actions"); - expect(result.stderr).not.toContain("--hostname"); - expect(result.stderr.length).toBeLessThanOrEqual(2001); - expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }, - ); - - test("preserves the primary log failure and directly removes its temporary directory", () => { - const item = fixture("AssertionError: retained tail"); - const credential = "cleanup-path-secret"; - const tempRoot = join(item.root, `BUILD_TOKEN=${credential}`); - mkdirSync(tempRoot, { recursive: true }); - item.env.TMPDIR = tempRoot; - item.env.FAIL_LOG = "primary log download failure"; - const result = run(item.env); - expect(result.status).not.toBe(0); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("primary log download failure"); - expect(result.stderr).not.toContain(credential); - expect(result.stderr).not.toContain("cleanup-secret"); - expect(result.stderr.length).toBeLessThanOrEqual(2001); - expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); - }); - test.each([ - ["malformed", Buffer.from("not a zip")], - [ - "symlink", - (() => { - const archive = artifactZip([{ name: "sample.result.json", contents: "{}" }]); - archive.writeUInt32LE(0xa0000000, archive.readUInt32LE(archive.length - 6) + 38); - return archive; - })(), - ], - ["traversal", artifactZip([{ name: "../sample.result.json", contents: "{}" }])], - ["option-like", artifactZip([{ name: "-sample.result.json", contents: "{}" }])], - [ - "duplicate", - artifactZip([ - { name: "sample.result.json", contents: "{}" }, - { name: "sample.result.json", contents: "{}" }, - ]), - ], - ])("rejects a %s artifact ZIP", (_name, archive) => { - const item = fixture("SIGKILL", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); - }); - test("allows an artifact stream at the exact compressed-size limit", () => { - const item = fixture("SIGKILL"); - item.env.STREAM_BYTES = "25000000"; - item.env.ZIP_SIZE = "25000000"; - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); - expect(result.stderr).not.toContain("compressed stream exceeds"); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }); - - test("rejects one byte beyond the compressed-size limit", () => { - const item = fixture("SIGKILL"); - item.env.STREAM_BYTES = "25000001"; - item.env.ZIP_SIZE = "25000000"; - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "Selected artifact compressed stream exceeds the 25,000,000-byte limit", - ); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }); - - test("rejects the artifact stream when the extra-byte dd probe fails", () => { - const item = fixture("SIGKILL", { exitCode: 1 }); - item.env.FAIL_PROBE_DD = "1"; - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("Could not download selected artifact"); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }); - - test("rejects the artifact stream when the extra-byte wc probe fails", () => { - const item = fixture("SIGKILL", { exitCode: 1 }); - item.env.FAIL_PROBE_WC = "1"; - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("Could not download selected artifact"); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }); - - test("rejects artifact metadata that differs from downloaded bytes", () => { - const item = fixture("SIGKILL"); - item.env.ZIP_SIZE = String(Number(item.env.ZIP_SIZE) + 1); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Could not download selected artifact"); - }); - test("rejects a corrupted unrelated entry beside a valid result", () => { - const archive = artifactZip( - [ - { name: "sample.result.json", contents: JSON.stringify({ exitCode: 1 }) }, - { name: "diagnostics/unrelated.txt", contents: "unrelated" }, - ], - 8, - ); - archive[artifactZipEntryDataOffset(archive, 1)] ^= 0xff; - const item = fixture("SIGKILL", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); - }); - - test("rejects an artifact whose declared expanded size exceeds the aggregate bound", () => { - const archive = artifactZip([{ name: "sample.result.json", contents: "{}" }]); - const centralOffset = archive.readUInt32LE(archive.length - 6); - archive.writeUInt32LE(100_000_001, centralOffset + 24); - const item = fixture("SIGKILL", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); - }); - test("rejects a result entry over the per-file expanded limit", () => { - const archive = artifactZip( - [{ name: "large.result.json", contents: "x".repeat(1_000_001) }], - 8, - ); - const item = fixture("SIGKILL", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("exceeds the 1,000,000-byte limit"); - }); - test("preserves an artifact validation failure and directly removes its temporary directory", () => { - const item = fixture("SIGKILL", undefined, Buffer.from("not a zip")); - const credential = "artifact-cleanup-path-secret"; - const tempRoot = join(item.root, `BUILD_TOKEN=${credential}`); - mkdirSync(tempRoot, { recursive: true }); - item.env.TMPDIR = tempRoot; - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); - expect(result.stderr).not.toContain("cleanup-secret"); - expect(result.stderr).not.toContain(credential); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }); - test("reads a real ZIP artifact and removes private temporary directories", () => { - const item = fixture("SIGKILL", { - exitCode: 137, - signal: "SIGKILL", - }); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.artifact.artifactId).toBe(789); - expect(value.artifact.filesRead).toBe(1); - expect(value.artifact.failures[0].signal).toBe("SIGKILL"); - expect(result.stdout).not.toContain("artifact-secret"); - const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; - expect( - existsSync(temporaryRoot) - ? readdirSync(temporaryRoot).filter((name) => name.startsWith("nemoclaw-ci-")) - : [], - ).toEqual([]); - }); - test.each([ - { - name: "nonzero exit code", - artifactResult: { exitCode: 23 }, - category: "process-exit-code", - nextAction: "Inspect the captured command and its producing step", - }, - { - name: "timeout", - artifactResult: { exitCode: null, timedOut: true }, - category: "process-timeout", - nextAction: "Inspect the captured command and surrounding resource evidence", - }, - { - name: "reported error", - artifactResult: { exitCode: null, error: "runner could not start" }, - category: "artifact-reported-error", - nextAction: "Inspect the reported command error and its producing step", - }, - ])( - "classifies an artifact-reported $name without a known log signature", - ({ artifactResult, category, nextAction }) => { - const item = fixture("ordinary output", artifactResult); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.result).toBe("classified"); - expect(value.categories).toContain(category); - expect(value.nextActions).toContainEqual(expect.stringContaining(nextAction)); - }, - ); - - test.each([ - { - name: "signal", - artifactResult: { exitCode: 137, signal: "SIGKILL" }, - category: "process-signal", - diagnostic: "ended with SIGKILL", - }, - { - name: "timeout", - artifactResult: { timedOut: true }, - category: "process-timeout", - diagnostic: "exceeded its time limit", - }, - { - name: "reported error", - artifactResult: { error: "runner could not start" }, - category: "artifact-reported-error", - diagnostic: "reported: runner could not start", - }, - ])( - "selects a later $name beyond capped failure evidence", - ({ artifactResult, category, diagnostic }) => { - const secret = "winner-path-secret"; - const earlier = Array.from({ length: 20 }, (_, index) => ({ - name: `earlier-${index}.result.json`, - contents: JSON.stringify({ exitCode: 23 }), - })); - const winnerPath = `BUILD_TOKEN=${secret} ${"x".repeat(1100)}.result.json`; - const archive = artifactZip([ - ...earlier, - { name: winnerPath, contents: JSON.stringify(artifactResult) }, - ]); - const item = fixture("ordinary output", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - const finding = value.findings.find((entry: { type: string }) => entry.type === category); - expect(value.categories).toEqual([category]); - expect(value.artifact.failures).toHaveLength(20); - expect(value.artifact.failuresTruncated).toBe(true); - expect(value.artifact.failures).not.toContainEqual( - expect.objectContaining({ path: expect.stringContaining("[REDACTED]") }), - ); - expect(finding.detail).toContain("BUILD_TOKEN=[REDACTED]"); - expect(finding.detail).toContain(diagnostic); - expect(finding.detail.length).toBeLessThanOrEqual(1100); - expect(result.stdout).not.toContain(secret); - }, - ); - - test("reports bounded malformed result evidence while retaining valid artifact failures", () => { - const pathSecret = "malformed-path-secret"; - const malformed = Array.from({ length: 25 }, (_, index) => ({ - name: `results/TOKEN=${pathSecret}-${index}.result.json`, - contents: '{"exitCode":', - })); - const archive = artifactZip([ - { name: `${"a".repeat(1_100)}.result.json`, contents: "null" }, - ...malformed, - { name: "valid.result.json", contents: JSON.stringify({ exitCode: 1 }) }, - ]); - const item = fixture("ordinary output", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.artifact.filesRead).toBe(27); - expect(value.artifact.malformedResultCount).toBe(26); - expect(value.artifact.malformedResultPaths).toHaveLength(20); - expect(value.artifact.malformedResultPaths.every((path: string) => path.length <= 1_000)).toBe( - true, - ); - expect(value.artifact.malformedResultPathsTruncated).toBe(true); - expect( - value.artifact.malformedResultPaths.some((path: string) => path.includes("[REDACTED]")), - ).toBe(true); - expect(value.artifact.failures).toHaveLength(1); - expect(value.artifact.failures[0].exitCode).toBe(1); - expect(result.stdout).not.toContain(pathSecret); - }); - - test("reports malformed evidence when every selected artifact result is malformed", () => { - const archive = artifactZip([ - { name: "truncated.result.json", contents: '{"exitCode":' }, - { name: "non-object.result.json", contents: JSON.stringify("invalid result") }, - ]); - const item = fixture("ordinary output", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.result).toBe("unclassified"); - expect(value.artifact).toMatchObject({ - filesRead: 2, - malformedResultCount: 2, - malformedResultPaths: ["truncated.result.json", "non-object.result.json"], - malformedResultPathsTruncated: false, - failures: [], - }); - }); - - test("ignores non-failure and command-only records and retains a true timeout", () => { - const archive = artifactZip([ - { - name: "string-false.result.json", - contents: JSON.stringify({ exitCode: 0, timedOut: "false" }), - }, - { - name: "command-only.result.json", - contents: JSON.stringify({ command: "npm test" }), - }, - { - name: "boolean-true.result.json", - contents: JSON.stringify({ exitCode: 0, timedOut: true }), - }, - ]); - const item = fixture("ordinary output", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const failures = JSON.parse(result.stdout).artifact.failures; - expect(failures).toHaveLength(1); - expect(failures[0]).toMatchObject({ path: "boolean-true.result.json", timedOut: true }); - }); - - test("rejects ambiguous same-name artifacts before downloading a ZIP", () => { - const item = fixture("SIGKILL"); - item.env.DUPLICATE_ARTIFACTS = "1"; - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status).toBe(1); - expect(result.stderr).toContain("Artifact results is ambiguous for run 456"); - expect(result.stderr).toContain("789, 790"); - const calls = readFileSync(item.env.GH_CALLS!, "utf8"); - expect(calls).not.toContain("actions/artifacts/789/zip"); - expect(calls).not.toContain("actions/artifacts/790/zip"); - }); - - test("does not return or use a credential-shaped artifact signal", () => { - const secret = "artifact-signal-secret"; - const item = fixture("ordinary output", { - exitCode: 1, - signal: `BUILD_TOKEN=${secret}`, - }); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.artifact.failures[0].signal).toBeNull(); - expect(value.findings).not.toContainEqual(expect.objectContaining({ type: "process-signal" })); - expect(result.stdout).not.toContain(secret); - }); - - test("retains a leader for an ignored-stdio process-group member until timeout", async () => { - const root = mkdtempSync(join(tmpdir(), "classify-ci-timeout-")); - roots.push(root); - const marker = join(root, "descendant-pid"); - const program = [ - "const {spawn}=require('node:child_process');", - "const child=spawn(process.execPath,['-e',\"process.title='member ) name'; process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)\"],{stdio:'ignore'});", - "require('node:fs').writeFileSync(process.argv[1],String(child.pid));", - ].join(""); - const started = Date.now(); - const pending = execute(process.execPath, ["-e", program, marker], root, 200); - await waitForFile(marker); - const descendantPid = Number(readFileSync(marker, "utf8")); - expect(() => process.kill(descendantPid, 0)).not.toThrow(); - const result = await pending; - expect(Date.now() - started).toBeGreaterThanOrEqual(200); - expect(result.exitCode).not.toBe(0); - expect(result.timedOut).toBe(true); - expect(() => process.kill(descendantPid, 0)).toThrow(); - }); - - test("drains a process group whose command exits promptly on SIGTERM", async () => { - const item = fixture("AssertionError: retained tail"); - const marker = join(item.root, "blocked"); - const signals = join(item.root, "process-group-signals"); - const wrapperMarker = join(item.root, "wrapper-pid"); - const commandMarker = join(item.root, "command-pid"); - item.env.BLOCK_LOG = "1"; - item.env.BLOCK_MARKER = marker; - item.env.EXIT_PROMPTLY_ON_TERM = "1"; - item.env.SIGNAL_LOG = signals; - item.env.BLOCK_WRAPPER_MARKER = wrapperMarker; - item.env.BLOCK_COMMAND_MARKER = commandMarker; - const child = spawn(process.execPath, importedClassifierArgs(item.env, []), { - env: item.env, - stdio: ["ignore", "pipe", "pipe"], - }); - await Promise.all([ - waitForFile(marker), - waitForFile(wrapperMarker), - waitForFile(commandMarker), - ]); - const wrapperPid = Number(readFileSync(wrapperMarker, "utf8")); - const commandPid = Number(readFileSync(commandMarker, "utf8")); - child.kill("SIGTERM"); - const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolve) => - child.once("close", (code, closeSignal) => resolve({ code, signal: closeSignal })), - ); - expect(result).toEqual({ code: 143, signal: null }); - expect(readFileSync(signals, "utf8").trim().split("\n")).toEqual(["SIGTERM"]); - expect(() => process.kill(wrapperPid, 0)).toThrow(); - expect(() => process.kill(commandPid, 0)).toThrow(); - expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); - expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); - }); - - test("reports a redacted fixed-root removal command when cancellation cleanup fails", async () => { - const item = fixture("AssertionError: retained tail"); - const marker = join(item.root, "blocked"); - const instrument = join(item.root, "fail-cancellation-cleanup.mjs"); - writeFileSync( - instrument, - [ - 'import fs from "node:fs";', - 'import { syncBuiltinESMExports } from "node:module";', - 'import path from "node:path";', - "const realRmSync = fs.rmSync.bind(fs);", - "fs.rmSync = (target, options) =>", - " path.basename(String(target)).startsWith('nemoclaw-ci-log.')", - " ? (() => { throw new Error('BUILD_TOKEN=cancellation-cleanup-secret'); })()", - " : realRmSync(target, options);", - "syncBuiltinESMExports();", - ].join("\n"), - ); - item.env.BLOCK_LOG = "1"; - item.env.BLOCK_MARKER = marker; - item.env.NODE_OPTIONS = `--import=${instrument}`; - const child = spawn(process.execPath, importedClassifierArgs(item.env, []), { - env: item.env, - stdio: ["ignore", "pipe", "pipe"], - }); - let stderr = ""; - child.stderr.setEncoding("utf8"); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - await waitForFile(marker); - child.kill("SIGTERM"); - const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolve) => - child.once("close", (code, closeSignal) => resolve({ code, signal: closeSignal })), - ); - expect(result).toEqual({ code: 143, signal: null }); - const generatedName = stderr.match( - /Cancellation cleanup failure for (nemoclaw-ci-log\.[A-Za-z0-9]{6})/, - )?.[1]; - expect(generatedName).toBeDefined(); - expect(stderr).toContain( - `Remove it directly with: rm -rf -- /tmp/nemoclaw-ci-classifier-${uid}/${generatedName}`, - ); - expect(stderr).toContain("BUILD_TOKEN=[REDACTED]"); - expect(stderr).not.toContain("cancellation-cleanup-secret"); - expect(stderr.length).toBeLessThanOrEqual(2001); - rmSync(`/tmp/nemoclaw-ci-classifier-${uid}/${generatedName}`, { recursive: true, force: true }); - }); - - test.each([ - ["metadata", "BLOCK_METADATA", [], "SIGTERM", 143, false], - ["log", "BLOCK_LOG", [], "SIGINT", 130, true], - ["log", "BLOCK_LOG", [], "SIGHUP", 129, true], - ["artifact", "BLOCK_ARTIFACT", ["--artifact-name", "results"], "SIGTERM", 143, true], - ] as const)( - "kills the detached group and its ignoring descendant during %s cancellation", - async (_kind, block, extra, signal, exitCode, exitGroupLeader) => { - const item = fixture("AssertionError: retained tail"); - const marker = join(item.root, "blocked"); - const groupMarker = join(item.root, "group-pid"); - const descendantMarker = join(item.root, "descendant-pid"); - item.env[block] = "1"; - item.env.BLOCK_MARKER = marker; - item.env.BLOCK_GROUP_MARKER = groupMarker; - item.env.BLOCK_DESCENDANT_MARKER = descendantMarker; - Object.assign(item.env, exitGroupLeader ? { EXIT_GROUP_LEADER: "1" } : {}); - const child = spawn(process.execPath, importedClassifierArgs(item.env, [...extra]), { - env: item.env, - stdio: ["ignore", "pipe", "pipe"], - }); - let stderr = ""; - child.stderr.setEncoding("utf8"); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - await Promise.all([ - waitForFile(marker), - waitForFile(groupMarker), - waitForFile(descendantMarker), - ]); - const groupPid = Number(readFileSync(groupMarker, "utf8")); - const descendantPid = Number(readFileSync(descendantMarker, "utf8")); - child.kill(signal); - child.kill(signal); - const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolve) => - child.once("close", (code, closeSignal) => resolve({ code, signal: closeSignal })), - ); - expect(result).toEqual({ code: exitCode, signal: null }); - expect(() => process.kill(groupPid, 0)).toThrow(); - expect(() => process.kill(descendantPid, 0)).toThrow(); - expect(stderr.length).toBeLessThanOrEqual(2000); - const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; - const remaining = existsSync(temporaryRoot) - ? readdirSync(temporaryRoot).filter((name) => name.startsWith("nemoclaw-ci-")) - : []; - expect(remaining).toEqual([]); - }, - ); - - test("redacts credential assignments in returned artifact paths", () => { - const secret = "artifact-path-secret"; - const archive = artifactZip([ - { - name: `BUILD_TOKEN=${secret}.result.json`, - contents: JSON.stringify({ exitCode: 1 }), - }, - ]); - const item = fixture("SIGKILL", undefined, archive); - const result = run(item.env, ["--artifact-name", "results"]); - expect(result.status, result.stderr).toBe(0); - const value = JSON.parse(result.stdout); - expect(value.artifact.failures[0].path).toContain("[REDACTED]"); - expect(result.stdout).not.toContain(secret); - }); -}); diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 04748d3dabc..c194f58b57f 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -1099,6 +1099,11 @@ describe("pull request and main workflow contracts", () => { step: "Merge CLI coverage", stem: "scripts/check-coverage-ratchet", }, + { + action: sharedActions.cliCoverageMerge, + step: "Report CLI timing drift", + stem: "scripts/checks/report-cli-test-timing-drift", + }, { action: sharedActions.pluginCoverage, step: "Run plugin coverage", @@ -1129,7 +1134,7 @@ describe("pull request and main workflow contracts", () => { const fakeBin = join(temp, "bin"); mkdirSync(fakeBin); mkdirSync(join(temp, "dist")); - mkdirSync(join(temp, "scripts")); + mkdirSync(join(temp, "scripts", "checks"), { recursive: true }); writeFileSync(join(temp, "dist", ["nemoclaw", "js"].join(".")), "built\n"); writeFileSync(join(fakeBin, "node"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); writeFileSync(join(fakeBin, "npm"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); @@ -1138,7 +1143,7 @@ describe("pull request and main workflow contracts", () => { [ "#!/usr/bin/env bash", "set -euo pipefail", - 'if [ "${1:-}" = "tsx" ] && [[ "${2:-}" == scripts/check-* ]]; then', + 'if [ "${1:-}" = "tsx" ] && [[ "${2:-}" == scripts/* ]]; then', ' test "${2}" = "${EXPECTED_ENTRYPOINT}"', ' test -f "${2}"', "fi", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index d92394d2c2d..7ff566d0c89 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -161,7 +161,10 @@ "fast": [ "test/e2e/support/hosted-inference.test.ts", "test/e2e/support/full-e2e-inference-probe.test.ts", - "test/e2e/support/launch-agent-turn.test.ts", + "test/e2e/support/launch-agent-turn-failure-paths.test.ts", + "test/e2e/support/launch-agent-turn-readiness.test.ts", + "test/e2e/support/launch-agent-turn-session-evidence.test.ts", + "test/e2e/support/launch-agent-turn-termios.test.ts", "test/e2e/support/openclaw-agent-output.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", @@ -175,7 +178,10 @@ "src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts", "src/lib/actions/sandbox/launch-readiness.test.ts", "src/lib/state/launch-readiness-lease.test.ts", - "test/e2e/support/launch-agent-turn.test.ts", + "test/e2e/support/launch-agent-turn-failure-paths.test.ts", + "test/e2e/support/launch-agent-turn-readiness.test.ts", + "test/e2e/support/launch-agent-turn-session-evidence.test.ts", + "test/e2e/support/launch-agent-turn-termios.test.ts", "test/e2e-runtime/launch-readiness-forward-observation.test.ts" ] }, @@ -652,7 +658,9 @@ "src/lib/onboard/providers.test.ts", "test/channels/channels-add-bridge-lifecycle.test.ts", "test/e2e/support/messaging-providers-runtime-proofs.test.ts", - "test/onboarding/onboard-messaging.test.ts", + "test/onboarding/onboard-messaging-image-configuration.test.ts", + "test/onboarding/onboard-messaging-provider.test.ts", + "test/onboarding/onboard-messaging-recreation.test.ts", "test/runtime/messaging/messaging-build-applier.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", @@ -733,7 +741,9 @@ "src/lib/onboard/sandbox-create-plan.test.ts", "test/e2e/support/openclaw-discord-pairing-helpers.test.ts", "test/e2e/support/messaging-providers-runtime-proofs.test.ts", - "test/onboarding/onboard-messaging.test.ts", + "test/onboarding/onboard-messaging-image-configuration.test.ts", + "test/onboarding/onboard-messaging-provider.test.ts", + "test/onboarding/onboard-messaging-recreation.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] diff --git a/test/e2e/support/launch-agent-turn-failure-paths.test.ts b/test/e2e/support/launch-agent-turn-failure-paths.test.ts new file mode 100644 index 00000000000..ee541a0bfd6 --- /dev/null +++ b/test/e2e/support/launch-agent-turn-failure-paths.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerLaunchAgentTurnTests } from "./launch-agent-turn-suite"; + +registerLaunchAgentTurnTests("failure-paths"); diff --git a/test/e2e/support/launch-agent-turn-readiness.test.ts b/test/e2e/support/launch-agent-turn-readiness.test.ts new file mode 100644 index 00000000000..95fe5ce77bb --- /dev/null +++ b/test/e2e/support/launch-agent-turn-readiness.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerLaunchAgentTurnTests } from "./launch-agent-turn-suite"; + +registerLaunchAgentTurnTests("readiness"); diff --git a/test/e2e/support/launch-agent-turn-session-evidence.test.ts b/test/e2e/support/launch-agent-turn-session-evidence.test.ts new file mode 100644 index 00000000000..226a2583f44 --- /dev/null +++ b/test/e2e/support/launch-agent-turn-session-evidence.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerLaunchAgentTurnTests } from "./launch-agent-turn-suite"; + +registerLaunchAgentTurnTests("evidence"); diff --git a/test/e2e/support/launch-agent-turn-suite.ts b/test/e2e/support/launch-agent-turn-suite.ts new file mode 100644 index 00000000000..e8f8443316c --- /dev/null +++ b/test/e2e/support/launch-agent-turn-suite.ts @@ -0,0 +1,1543 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + appendFileSync, + chmodSync, + closeSync, + constants, + existsSync, + fstatSync, + fsyncSync, + ftruncateSync, + mkdtempSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + rmSync, + type Stats, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { createServer } from "node:net"; +import { join, resolve } from "node:path"; + +import { expect, it, vi } from "vitest"; +import { + isSubprocessEnvNameAllowed, + SUBPROCESS_ENV_ALLOWED_NAMES, + SUBPROCESS_ENV_ALLOWED_PREFIXES, +} from "../../../src/lib/subprocess-env"; +import { testTimeout } from "../../helpers/timeouts"; +import { + LAUNCH_TURN_SCRIPT, + OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, + OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + runOpenClawLaunchSession, + runOpenClawLaunchReadinessLeaseTurns, +} from "../live/launch-agent-turn.ts"; +const PROCESS_EXIT_WAIT = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); +type SessionRecords = Record; +type FixtureMode = + | "cleanup-failure" + | "delayed-input-attachment" + | "delayed-recording" + | "delayed-tui-ready" + | "input-mode-timeout" + | "invalid-order" + | "late-extra" + | "nonzero" + | "nonzero-pty-cleanup-failure" + | "pty-cleanup-failure" + | "pty-cleanup-unknown-entry" + | "pty-socket-invalid" + | "pty-socket-permission" + | "pty-response-identity" + | "pty-socket-timeout" + | "pty-path-unreadable" + | "pty-termios-unavailable" + | "recording-timeout" + | "restored-canonical-timeout" + | "valid"; + +function message(role: "assistant" | "user", content = "nonempty"): string { + return JSON.stringify({ + message: { content: [{ text: content, type: "text" }], role }, + type: "message", + }); +} + +function emptyMessage(role: "assistant" | "user"): string { + return JSON.stringify({ message: { content: [], role }, type: "message" }); +} + +function writeSessionRecords( + root: string, + sessions: SessionRecords, + append: boolean, + finalNewline = true, +): void { + for (const [sessionId, records] of Object.entries(sessions)) { + const filePath = join(root, `${sessionId}.jsonl`); + const body = records.length > 0 ? `${records.join("\n")}${finalNewline ? "\n" : ""}` : ""; + const writeRecords = append ? appendFileSync : writeFileSync; + writeRecords(filePath, body); + } +} + +function withOwnedFixtureFile( + filePath: string, + flags: number, + action: (descriptor: number, stats: Stats) => T, +): T { + const descriptor = openSync(filePath, flags | constants.O_NOFOLLOW, 0o600); + try { + const stats = fstatSync(descriptor); + expect([stats.isFile(), stats.uid, stats.mode & 0o777, stats.nlink]).toEqual([ + true, + process.getuid?.(), + 0o600, + 1, + ]); + return action(descriptor, stats); + } finally { + closeSync(descriptor); + } +} + +function runEvidenceFixture(input: { + after: SessionRecords; + afterFinalNewline?: boolean; + before?: SessionRecords; + expectedTurns: number; +}) { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-evidence-")); + const runId = randomUUID().replaceAll("-", ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + const sessionRoot = join(fixtureRoot, "sessions"); + mkdirSync(sessionRoot); + try { + writeSessionRecords(sessionRoot, input.before ?? {}, false); + const baseline = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "baseline", + sessionRoot, + baselinePath, + "", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + writeSessionRecords(sessionRoot, input.after, true, input.afterFinalNewline ?? true); + const qualification = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "qualify", + sessionRoot, + baselinePath, + String(input.expectedTurns), + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + const baselineFile = withOwnedFixtureFile( + baselinePath, + constants.O_RDONLY, + (descriptor, stats) => ({ body: readFileSync(descriptor, "utf8"), stats }), + ); + return { + baseline, + baselineKeys: Object.keys(JSON.parse(baselineFile.body)).sort(), + baselineMode: baselineFile.stats.mode & 0o777, + baselineNlink: baselineFile.stats.nlink, + baselineUid: baselineFile.stats.uid, + qualification, + }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(baselinePath, { force: true }); + rmSync(`${baselinePath}.tmp`, { force: true }); + rmSync(ptyMonitorRoot, { force: true, recursive: true }); + } +} + +function runBaselineMutationFixture(mutation: "invalid" | "removed" | "rewritten" | "truncated") { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-baseline-")); + const runId = randomUUID().replaceAll("-", ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + const sessionRoot = join(fixtureRoot, "sessions"); + const sessionPath = join(sessionRoot, "session-a.jsonl"); + mkdirSync(sessionRoot); + writeSessionRecords(sessionRoot, { "session-a": [message("user"), message("assistant")] }, false); + try { + const baseline = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "baseline", + sessionRoot, + baselinePath, + "", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + const applyMutation: Record void> = { + invalid: () => + withOwnedFixtureFile(baselinePath, constants.O_WRONLY, (descriptor) => { + ftruncateSync(descriptor, 0); + writeFileSync(descriptor, "{}"); + fsyncSync(descriptor); + }), + removed: () => rmSync(sessionPath), + rewritten: () => + writeFileSync( + sessionPath, + readFileSync(sessionPath, "utf8").replace("nonempty", "changed!"), + ), + truncated: () => writeFileSync(sessionPath, ""), + }; + applyMutation[mutation](); + const qualification = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "qualify", + sessionRoot, + baselinePath, + "1", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + return { baseline, qualification }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(baselinePath, { force: true }); + rmSync(`${baselinePath}.tmp`, { force: true }); + rmSync(ptyMonitorRoot, { force: true, recursive: true }); + } +} + +export type LaunchAgentTurnGroup = "evidence" | "failure-paths" | "readiness" | "termios"; + +export function registerLaunchAgentTurnTests(group: LaunchAgentTurnGroup): void { + if (group === "evidence") { + it("reports a residual PTY monitor socket without removing it (#9384)", async () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-monitor-cleanup-")); + const runId = randomUUID().replaceAll("-", ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + const socketPath = join(ptyMonitorRoot, "pty-input-mode.sock"); + const server = createServer(); + mkdirSync(ptyMonitorRoot, { mode: 0o700 }); + chmodSync(ptyMonitorRoot, 0o700); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, resolve); + }); + chmodSync(socketPath, 0o600); + try { + const cleanup = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "cleanup-pty", + fixtureRoot, + baselinePath, + "", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + + expect(cleanup.status).toBe(2); + expect(cleanup.stderr).toContain('"reason":"pty_monitor_socket_still_present"'); + expect(existsSync(socketPath)).toBe(true); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(ptyMonitorRoot, { force: true, recursive: true }); + } + }); + } + + function runLaunchSessionFixture( + mode: FixtureMode, + terminalCopy: "absent" | "ansi" | "reordered", + ) { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-turn-")); + const canonicalRestoredMarker = join(fixtureRoot, "canonical-restored"); + const earlyInputMarker = join(fixtureRoot, "early-input"); + const fakeLaunch = join(fixtureRoot, "openclaw"); + const fakeOpenshell = join(fixtureRoot, "openshell"); + const fakeStty = join(fixtureRoot, "stty"); + const monitorPidPath = join(fixtureRoot, "monitor-pid"); + const sessionRoot = join(fixtureRoot, "sessions"); + const tuiPidsPath = join(fixtureRoot, "tui-pids"); + const ttyMarker = join(fixtureRoot, "tty-observed"); + const openshellCallsRoot = join(fixtureRoot, "openshell-calls"); + const pendingQualificationMarker = join(fixtureRoot, "pending-qualification-observed"); + const ptyPathUnreadableMarker = join(fixtureRoot, "pty-path-unreadable"); + const ptySocketReceiptPath = join(fixtureRoot, "pty-socket-receipt.json"); + const runId = randomUUID().replaceAll("-", ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + mkdirSync(sessionRoot); + mkdirSync(openshellCallsRoot, { mode: 0o700 }); + writeFileSync( + join(fixtureRoot, ".bash_profile"), + 'export PATH="$NEMOCLAW_FIXTURE_BIN_ROOT:$PATH"\n', + ); + try { + writeFileSync( + join(fixtureRoot, "sleep"), + '#!/bin/bash\n[[ "$1:$NEMOCLAW_FIXTURE_MODE" =~ ^0\.05:pty-(socket-(invalid|permission)|response-identity)$ ]] || exec /usr/bin/sleep "$@"\n', + { mode: 0o755 }, + ); + writeFileSync( + fakeStty, + String.raw`#!/bin/bash +marker=${JSON.stringify(ttyMarker)} +if [[ ! -e "$marker" ]]; then + exec /usr/bin/stty "$@" +fi +printf 'fixture stty denied\n' >&2 +exit 1 +`, + ); + writeFileSync( + fakeLaunch, + String.raw`#!/usr/bin/env node +const childProcess = require("node:child_process"); +const fs = require("node:fs"); +const net = require("node:net"); +const readline = require("node:readline"); + +const mode = process.env.NEMOCLAW_FIXTURE_MODE; +if (process.argv[2] !== "tui") { + const allowedNames = new Set(${JSON.stringify(SUBPROCESS_ENV_ALLOWED_NAMES)}); + const allowedPrefixes = ${JSON.stringify(SUBPROCESS_ENV_ALLOWED_PREFIXES)}; + const subprocessEnv = Object.fromEntries( + Object.entries(process.env).filter( + ([name, value]) => + value !== undefined && + (allowedNames.has(name) || + allowedPrefixes.some((prefix) => name.startsWith(prefix)) || + name.startsWith("NEMOCLAW_FIXTURE_")), + ), + ); + const args = [ + "sandbox", + "exec", + "--name", + process.argv[3], + "-g", + "fixture-gateway", + "--tty", + "--timeout", + "0", + "--", + "/bin/bash", + "--noprofile", + "--norc", + "-p", + "-c", + process.env.NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + "nemoclaw-runtime-env", + "bash", + "-lc", + "openclaw tui", + ]; + const result = childProcess.spawnSync(process.env.NEMOCLAW_OPENSHELL_BIN, args, { + env: subprocessEnv, + stdio: "inherit", + timeout: 14_000, + killSignal: "SIGKILL", + }); + process.exit(result.status ?? 66); +} + +(async () => { + if (!process.stdin.isTTY || !process.stdout.isTTY) process.exit(64); + fs.appendFileSync(process.env.NEMOCLAW_FIXTURE_TUI_PIDS, process.pid + "\n"); + const monitorRoot = process.env.NEMOCLAW_FIXTURE_PTY_MONITOR_ROOT; + const socketPath = monitorRoot + "/pty-input-mode.sock"; + const socketDeadline = Date.now() + 2_000; + while (!fs.existsSync(socketPath)) { + if (Date.now() >= socketDeadline) process.exit(70); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const socketStats = fs.lstatSync(socketPath); + const rootStats = fs.lstatSync(monitorRoot); + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_PTY_SOCKET_RECEIPT, JSON.stringify({ + rootMode: rootStats.mode & 0o777, + rootUid: rootStats.uid, + socketIsSocket: socketStats.isSocket(), + socketMode: socketStats.mode & 0o777, + socketNlink: socketStats.nlink, + socketUid: socketStats.uid, + })); + const monitorPid = fs.readdirSync("/proc").find((name) => { + try { + const commandLine = fs.readFileSync("/proc/" + name + "/cmdline", "utf8"); + const argv = commandLine.split("\0"); + return argv.includes("nemoclaw-pty-input-mode-monitor") && + argv.includes(process.env.NEMOCLAW_FIXTURE_RUN_ID); + } catch { + return false; + } + }); + if (!monitorPid) process.exit(71); + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_MONITOR_PID, monitorPid); + if (mode === "pty-socket-invalid" || mode === "pty-response-identity") { + fs.unlinkSync(socketPath); + const ttyPath = fs.realpathSync("/proc/self/fd/0"); + const ttyStats = fs.fstatSync(0, { bigint: true }); + const replacement = net.createServer((client) => { + const body = mode === "pty-socket-invalid" + ? "{}\n" + : JSON.stringify({ + ttyPath, + dev: ttyStats.dev.toString(), + ino: ttyStats.ino.toString(), + rdev: ttyStats.rdev === 0n ? "1" : "0", + state: "noncanonical", + status: null, + signal: null, + errorCode: null, + stderr: "", + }) + "\n"; + client.end(body); + }); + process.umask(0o177); + await new Promise((resolve, reject) => { + replacement.once("error", reject); + replacement.listen(socketPath, resolve); + }); + fs.chmodSync(socketPath, 0o600); + } + if (mode === "pty-socket-permission") fs.chmodSync(socketPath, 0); + switch (mode) { + case "pty-path-unreadable": { + const ttyPath = fs.realpathSync("/proc/self/fd/0"); + const ttyMode = fs.lstatSync(ttyPath).mode & 0o777; + fs.chmodSync(ttyPath, 0); + const pathQuery = childProcess.spawnSync( + "/usr/bin/stty", + ["-F", ttyPath, "-a"], + { encoding: "utf8" }, + ); + fs.writeFileSync( + process.env.NEMOCLAW_FIXTURE_PTY_PATH_UNREADABLE_MARKER, + JSON.stringify({ + errorCode: pathQuery.error?.code ?? null, + status: pathQuery.status, + }), + ); + if (pathQuery.status === 0) process.exit(69); + process.on("exit", () => { + try { fs.chmodSync(ttyPath, ttyMode); } catch {} + }); + break; + } + } + if (mode === "pty-cleanup-unknown-entry") { + fs.writeFileSync(monitorRoot + "/unexpected", "owned test residue"); + } + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_TTY_MARKER, ""); + const sessionFile = process.env.NEMOCLAW_FIXTURE_SESSION_FILE; + const terminalCopy = process.env.NEMOCLAW_FIXTURE_TERMINAL_COPY; + const messageIndex = process.argv.indexOf("--message"); + const firstInput = messageIndex === -1 ? "" : process.argv[messageIndex + 1]; + if (!firstInput) process.exit(75); + const append = (role, content) => fs.appendFileSync( + sessionFile, + JSON.stringify({ message: { content: [{ text: content, type: "text" }], role }, type: "message" }) + "\n", + ); + if (mode === "restored-canonical-timeout") { + process.stdin.setRawMode(true); + await new Promise((resolve) => setTimeout(resolve, 750)); + process.stdin.setRawMode(false); + append("user", firstInput); + append("assistant", "first response"); + const recordUnexpectedInput = () => + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER, ""); + process.stdin.on("data", recordUnexpectedInput); + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER, ""); + await new Promise((resolve) => setTimeout(resolve, 20_000)); + process.stdin.off("data", recordUnexpectedInput); + } + if (mode === "input-mode-timeout") { + append("user", firstInput); + append("assistant", "first response"); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + if (mode === "delayed-input-attachment" || mode === "delayed-tui-ready") { + const recordEarlyInput = () => + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER, ""); + if (mode === "delayed-tui-ready") process.stdin.setRawMode(true); + process.stdin.on("data", recordEarlyInput); + await new Promise((resolve) => setTimeout(resolve, 1_500)); + process.stdin.off("data", recordEarlyInput); + if (fs.existsSync(process.env.NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER)) process.exit(67); + } + const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true }); + const ask = () => new Promise((resolve) => rl.question("", resolve)); + if (terminalCopy === "ansi") process.stdout.write("\u001b[2Kgateway connected | idle\r"); + if (terminalCopy === "reordered") process.stdout.write("idle | gateway connected\n"); + + if (mode === "delayed-recording") { + const publicationDeadline = Date.now() + 2_000; + while (!fs.existsSync(process.env.NEMOCLAW_FIXTURE_PENDING_QUALIFICATION_MARKER)) { + if (Date.now() >= publicationDeadline) process.exit(68); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } + if (mode === "recording-timeout") await new Promise((resolve) => setTimeout(resolve, 10_000)); + if (mode === "invalid-order") { + append("assistant", "response before input"); + append("user", firstInput); + } else { + append("user", firstInput); + append("assistant", "first response"); + } + + process.kill(Number(monitorPid), "SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 50)); + const monitorStat = fs.readFileSync("/proc/" + monitorPid + "/stat", "utf8"); + const monitorState = monitorStat ? monitorStat.slice(monitorStat.lastIndexOf(") ") + 2)[0] : null; + if (!monitorState || monitorState === "Z") process.exit(71); + + const second = await ask(); + append("user", second); + append("assistant", "second response"); + if (mode === "delayed-input-attachment") process.exit(0); + const exitCommand = await ask(); + if (mode === "late-extra") append("user", firstInput); + rl.close(); + if (exitCommand !== "/exit") process.exit(65); + process.exit(mode.includes("nonzero") ? 23 : 0); +})().catch(() => process.exit(66)); +`, + ); + writeFileSync( + fakeOpenshell, + String.raw`#!/usr/bin/env bash +set -euo pipefail +node -e ' +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const authorityNames = Object.keys(process.env) + .filter( + (name) => + name === "NEMOCLAW_OPENSHELL_BIN" || + name === "NEMOCLAW_OPENSHELL_COMMAND" || + name.startsWith("NEMOCLAW_LAUNCH_") || + name.startsWith("OPENSHELL_NEMOCLAW_LAUNCH_"), + ) + .sort(); +fs.writeFileSync( + path.join( + process.env.NEMOCLAW_FIXTURE_OPENSHELL_CALLS, + process.pid + "-" + crypto.randomUUID() + ".json", + ), + JSON.stringify({ argv: process.argv.slice(1), authorityNames }), + { flag: "wx", mode: 0o600 }, +); +' "$@" +while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done +[[ "$#" -gt 0 ]] +shift +case "$NEMOCLAW_FIXTURE_MODE:$4" in + pty-socket-invalid:input-mode|pty-socket-permission:input-mode|pty-response-identity:input-mode) + [[ -e "$NEMOCLAW_FIXTURE_TTY_MARKER" ]] || exit 1 + ;; +esac +if [[ "$NEMOCLAW_FIXTURE_MODE" == "restored-canonical-timeout" && "$4" == "input-mode" ]]; then + for _ in {1..200}; do + [[ ! -e "$NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER" ]] || break + sleep 0.01 + done + [[ -e "$NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER" ]] || exit 1 +fi +if [[ "$NEMOCLAW_FIXTURE_MODE" == "cleanup-failure" && "$4" == "cleanup-baseline" ]]; then + exit 71 +fi +if [[ ( "$NEMOCLAW_FIXTURE_MODE" == "pty-cleanup-failure" || "$NEMOCLAW_FIXTURE_MODE" == "nonzero-pty-cleanup-failure" ) && "$4" == "cleanup-pty" ]]; then + exit 71 +fi +if [[ "$NEMOCLAW_FIXTURE_MODE" == "pty-socket-timeout" && "$4" == "$NEMOCLAW_FIXTURE_RUN_ID" ]]; then + exec node -e 'setTimeout(() => process.exit(0), 10_000)' +fi +if [[ "$NEMOCLAW_FIXTURE_MODE" == "delayed-recording" && "$4" == "qualify" && "$7" == "1" ]]; then + set +e + "$@" + status=$? + set -e + [[ "$status" != "1" ]] || : > "$NEMOCLAW_FIXTURE_PENDING_QUALIFICATION_MARKER" + exit "$status" +fi +exec "$@" +`, + ); + chmodSync(fakeLaunch, 0o755); + chmodSync(fakeOpenshell, 0o755); + chmodSync(fakeStty, 0o755); + + const unavailablePtyMonitorStarterScript = OPENCLAW_PTY_MONITOR_STARTER_SCRIPT.replace( + 'const termiosCommand = "/usr/bin/stty";', + `const termiosCommand = ${JSON.stringify(fakeStty)};`, + ); + expect(unavailablePtyMonitorStarterScript).not.toBe(OPENCLAW_PTY_MONITOR_STARTER_SCRIPT); + const ptyMonitorStarterScript = + mode === "pty-termios-unavailable" + ? unavailablePtyMonitorStarterScript + : OPENCLAW_PTY_MONITOR_STARTER_SCRIPT; + + const result = spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { + encoding: "utf8", + killSignal: "SIGKILL", + env: { + ...process.env, + HOME: fixtureRoot, + NEMOCLAW_FIXTURE_BIN_ROOT: fixtureRoot, + NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER: canonicalRestoredMarker, + NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER: earlyInputMarker, + NEMOCLAW_FIXTURE_MODE: mode, + NEMOCLAW_FIXTURE_MONITOR_PID: monitorPidPath, + NEMOCLAW_FIXTURE_OPENSHELL_CALLS: openshellCallsRoot, + NEMOCLAW_FIXTURE_PENDING_QUALIFICATION_MARKER: pendingQualificationMarker, + NEMOCLAW_FIXTURE_PTY_MONITOR_ROOT: ptyMonitorRoot, + NEMOCLAW_FIXTURE_PTY_PATH_UNREADABLE_MARKER: ptyPathUnreadableMarker, + NEMOCLAW_FIXTURE_PTY_SOCKET_RECEIPT: ptySocketReceiptPath, + NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), + NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, + NEMOCLAW_FIXTURE_RUN_ID: runId, + NEMOCLAW_FIXTURE_TUI_PIDS: tuiPidsPath, + NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, + NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, + NEMOCLAW_LAUNCH_ENTRYPOINT: "", + NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", + NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", + NEMOCLAW_LAUNCH_HOST_TMP_ROOT: fixtureRoot, + NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, + NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT: ptyMonitorStarterScript, + NEMOCLAW_LAUNCH_RUN_ID: runId, + NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT: OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + NEMOCLAW_LAUNCH_SANDBOX: "sandbox", + NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: + mode === "restored-canonical-timeout" + ? "10" + : mode === "pty-socket-timeout" + ? "5" + : mode.endsWith("-timeout") + ? "2" + : "230", + NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", + NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, + NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, + NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, + PATH: `${fixtureRoot}:${process.env.PATH ?? ""}`, + TERM: "xterm-256color", + }, + timeout: 15_000, + }); + + const tuiProcessIds = existsSync(tuiPidsPath) + ? readFileSync(tuiPidsPath, "utf8").trim().split("\n").filter(Boolean) + : []; + const monitorProcessIds = existsSync(monitorPidPath) + ? [readFileSync(monitorPidPath, "utf8").trim()].filter(Boolean) + : []; + const processExitDeadline = Date.now() + 1_500; + while ( + (tuiProcessIds.some((pid) => existsSync(`/proc/${pid}`)) || + monitorProcessIds.some((pid) => existsSync(`/proc/${pid}`))) && + Date.now() < processExitDeadline + ) { + Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); + } + return { + baselineRemoved: !existsSync(baselinePath), + canonicalRestored: existsSync(canonicalRestoredMarker), + earlyInputObserved: existsSync(earlyInputMarker), + hostSessionResidue: readdirSync(fixtureRoot).filter((name) => + name.startsWith("nemoclaw-launch-host."), + ), + orphanedMonitorProcessIds: monitorProcessIds.filter((pid) => existsSync(`/proc/${pid}`)), + orphanedTuiProcessIds: tuiProcessIds.filter((pid) => existsSync(`/proc/${pid}`)), + openshellCalls: readdirSync(openshellCallsRoot) + .sort() + .map( + (name) => + JSON.parse(readFileSync(join(openshellCallsRoot, name), "utf8")) as { + argv: string[]; + authorityNames: string[]; + }, + ), + pendingQualificationObserved: existsSync(pendingQualificationMarker), + ptyPathQueryResult: existsSync(ptyPathUnreadableMarker) + ? JSON.parse(readFileSync(ptyPathUnreadableMarker, "utf8")) + : null, + ptyMonitorRemoved: !existsSync(ptyMonitorRoot), + ptySocketReceipt: existsSync(ptySocketReceiptPath) + ? JSON.parse(readFileSync(ptySocketReceiptPath, "utf8")) + : null, + result, + tuiProcessIds, + ttyObserved: existsSync(ttyMarker), + }; + } finally { + const ptySocketPath = join(ptyMonitorRoot, "pty-input-mode.sock"); + existsSync(ptyMonitorRoot) ? chmodSync(ptyMonitorRoot, 0o700) : undefined; + existsSync(ptySocketPath) ? chmodSync(ptySocketPath, 0o600) : undefined; + rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(baselinePath, { force: true }); + rmSync(`${baselinePath}.tmp`, { force: true }); + rmSync(ptyMonitorRoot, { force: true, recursive: true }); + } + } + + function openShellLaunchArgv(sandboxName: string, gatewayArgs: string[]): string[] { + return [ + "sandbox", + "exec", + "--name", + sandboxName, + ...gatewayArgs, + "--tty", + "--timeout", + "0", + "--", + "/bin/bash", + "--noprofile", + "--norc", + "-p", + "-c", + OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + "nemoclaw-runtime-env", + "bash", + "-lc", + "openclaw tui", + ]; + } + + function runOpenShellShimFixture(gatewayArgs: string[]) { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-shim-")); + const realOpenShell = join(fixtureRoot, "openshell-real"); + const shim = join(fixtureRoot, "openshell-shim"); + const callsPath = join(fixtureRoot, "calls.jsonl"); + const interceptPath = join(fixtureRoot, "intercept.json"); + const runId = randomUUID().replaceAll("-", ""); + const sandboxName = "sandbox"; + writeFileSync( + realOpenShell, + String.raw`#!/usr/bin/env node +require("node:fs").appendFileSync( + ${JSON.stringify(callsPath)}, + JSON.stringify({ + argv: process.argv.slice(2), + authorityNames: Object.keys(process.env) + .filter( + (name) => + name === "NEMOCLAW_OPENSHELL_BIN" || + name === "NEMOCLAW_OPENSHELL_COMMAND" || + name.startsWith("NEMOCLAW_LAUNCH_") || + name.startsWith("OPENSHELL_NEMOCLAW_LAUNCH_"), + ) + .sort(), + }) + "\n", +); +`, + ); + writeFileSync(shim, OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT); + chmodSync(realOpenShell, 0o755); + chmodSync(shim, 0o755); + const hostEnv = { + ...process.env, + NEMOCLAW_LAUNCH_COMMAND: "nemoclaw", + NEMOCLAW_LAUNCH_FIRST_INPUT: "fixture input", + NEMOCLAW_LAUNCH_INTERCEPT_PATH: interceptPath, + NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT: OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, + NEMOCLAW_LAUNCH_RUN_ID: runId, + NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT: OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + NEMOCLAW_LAUNCH_SANDBOX: sandboxName, + NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, + NEMOCLAW_OPENSHELL_BIN: shim, + NEMOCLAW_OPENSHELL_COMMAND: realOpenShell, + OPENSHELL_NEMOCLAW_LAUNCH_INTERCEPT_PATH: interceptPath, + OPENSHELL_NEMOCLAW_LAUNCH_FIRST_INPUT: "fixture input", + OPENSHELL_NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT: OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, + OPENSHELL_NEMOCLAW_LAUNCH_REAL_COMMAND: realOpenShell, + OPENSHELL_NEMOCLAW_LAUNCH_RUN_ID: runId, + OPENSHELL_NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT: OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + OPENSHELL_NEMOCLAW_LAUNCH_SANDBOX: sandboxName, + }; + const env = Object.fromEntries( + Object.entries(hostEnv).filter( + (entry): entry is [string, string] => + entry[1] !== undefined && isSubprocessEnvNameAllowed(entry[0]), + ), + ); + const runShim = (args: string[], childEnv: NodeJS.ProcessEnv = env) => + spawnSync(process.execPath, [shim, ...args], { + encoding: "utf8", + env: childEnv, + timeout: 2_000, + killSignal: "SIGKILL", + }); + const passThroughArgv = ["sandbox", "exec", "--name", sandboxName, "--", "true"]; + const ttyPassThroughArgv = [ + "sandbox", + "exec", + "--name", + sandboxName, + "--tty", + "--timeout", + "0", + "--", + "bash", + "-lc", + "printf '%s\\n' --tty", + ]; + const exactArgv = openShellLaunchArgv(sandboxName, gatewayArgs); + const malformedArgv = [...exactArgv]; + const ttyIndex = malformedArgv.indexOf("--tty"); + malformedArgv.splice(ttyIndex, 3, "--timeout", "0", "--tty"); + try { + const passThrough = runShim(passThroughArgv, hostEnv); + const ttyPassThrough = runShim(ttyPassThroughArgv); + const malformed = runShim(malformedArgv); + const invalidFirstInput = runShim(exactArgv, { + ...env, + OPENSHELL_NEMOCLAW_LAUNCH_FIRST_INPUT: "", + }); + const intercepted = runShim(exactArgv); + const duplicate = runShim(exactArgv); + const records = readFileSync(callsPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as { argv: string[]; authorityNames: string[] }); + return { + authorityNames: records.map((record) => record.authorityNames), + calls: records.map((record) => record.argv), + duplicate, + exactArgv, + interceptMode: statSync(interceptPath).mode & 0o777, + intercepted, + invalidFirstInput, + malformed, + passThrough, + passThroughArgv, + ttyPassThrough, + ttyPassThroughArgv, + monitorRoot: `/tmp/nemoclaw-launch-turn-${runId}`, + runId, + }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } + } + + if (group === "evidence") { + it("qualifies two ordered structured turns without comparing message content (#9160)", () => { + const { baseline, baselineKeys, baselineMode, baselineNlink, baselineUid, qualification } = + runEvidenceFixture({ + after: { + "session-a": [ + message("user", "first arbitrary input"), + message("assistant", "first arbitrary response"), + message("user", "different second input"), + message("assistant", "different second response"), + ], + }, + expectedTurns: 2, + }); + + expect(baseline.status).toBe(0); + expect(baselineKeys).toEqual(["schemaVersion", "sessions"]); + expect(baselineMode).toBe(0o600); + expect(baselineNlink).toBe(1); + expect(baselineUid).toBe(process.getuid?.()); + expect(qualification.status).toBe(0); + }); + + it("keeps a partial structured turn pending (#9160)", () => { + const { baseline, qualification } = runEvidenceFixture({ + after: { "session-a": [message("user")] }, + expectedTurns: 1, + }); + + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(1); + }); + + it("does not qualify structured turns recorded before the baseline (#9160)", () => { + const { baseline, qualification } = runEvidenceFixture({ + before: { "session-a": [message("user"), message("assistant")] }, + after: {}, + expectedTurns: 1, + }); + + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(1); + }); + + it.each([ + { "session-a": [message("assistant"), message("user")] }, + { "session-a": [message("user"), message("user"), message("assistant")] }, + { "session-a": [message("user"), message("assistant"), message("assistant")] }, + { "session-a": [message("user"), "not-json", message("assistant")] }, + { "session-a": [emptyMessage("user"), message("assistant")] }, + { "session-a": [message("user"), message("assistant")], "session-b": [message("user")] }, + ] as SessionRecords[])( + "rejects malformed, empty, duplicated, extra, out-of-order, or cross-session records [case %#] (#9160)", + (after) => { + const { baseline, qualification } = runEvidenceFixture({ after, expectedTurns: 1 }); + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(2); + }, + ); + + it("rejects an unterminated appended session record (#9160)", () => { + const { baseline, qualification } = runEvidenceFixture({ + after: { + "session-a": [ + message("user"), + message("assistant"), + message("user"), + message("assistant"), + message("user"), + ], + }, + afterFinalNewline: false, + expectedTurns: 2, + }); + + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(2); + }); + + it.each(["invalid", "removed", "rewritten", "truncated"] as const)( + "rejects an invalid baseline or a removed, rewritten, or truncated session [case %#] (#9160)", + (mutation) => { + const { baseline, qualification } = runBaselineMutationFixture(mutation); + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(2); + }, + ); + + it.each([[], ["-g", "fixture-gateway"]].map((gatewayArgs) => [gatewayArgs] as const))( + "intercepts one OpenClaw launch, preserves pass-through argv, and strips launch authority from filtered and inherited environments [case %#] (#9160)", + (gatewayArgs) => { + const fixture = runOpenShellShimFixture(gatewayArgs); + const separator = fixture.exactArgv.indexOf("--"); + const expectedRemote = fixture.exactArgv.slice(separator + 1); + + expect(fixture.passThrough.status, fixture.passThrough.stderr).toBe(0); + expect(fixture.ttyPassThrough.status, fixture.ttyPassThrough.stderr).toBe(0); + expect(fixture.malformed.status).toBe(73); + expect(fixture.malformed.stderr).toContain( + '"reason":"openshell_launch_invocation_invalid"', + ); + expect(fixture.invalidFirstInput.status).toBe(73); + expect(fixture.invalidFirstInput.stderr).toContain( + '"reason":"openshell_shim_first_input_invalid"', + ); + expect(fixture.intercepted.status, fixture.intercepted.stderr).toBe(0); + expect(fixture.duplicate.status).toBe(73); + expect(fixture.duplicate.stderr).toContain( + '"reason":"openshell_launch_intercept_duplicate"', + ); + expect(fixture.interceptMode).toBe(0o600); + expect(fixture.calls).toHaveLength(3); + expect(fixture.authorityNames).toEqual([[], [], []]); + expect(fixture.calls[0]).toEqual(fixture.passThroughArgv); + expect(fixture.calls[1]).toEqual(fixture.ttyPassThroughArgv); + expect(fixture.calls[2]?.slice(0, separator + 1)).toEqual( + fixture.exactArgv.slice(0, separator + 1), + ); + expect(fixture.calls[2]?.slice(separator + 1)).toEqual([ + "node", + "-e", + OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, + fixture.runId, + fixture.monitorRoot, + ...expectedRemote.slice(0, -1), + 'exec openclaw tui --message "$1"', + "nemoclaw-launch-first-turn", + "fixture input", + ]); + }, + ); + + it.runIf(process.platform === "linux")( + "rejects a monitor starter whose standard input is not a PTY (#9160)", + () => { + const runId = randomUUID().replaceAll("-", ""); + const monitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + try { + const result = spawnSync( + process.execPath, + ["-e", OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, runId, monitorRoot, "/usr/bin/env", "true"], + { encoding: "utf8", timeout: 2_000, killSignal: "SIGKILL" }, + ); + + expect(result.status).toBe(72); + expect(result.stderr).toContain('"reason":"pty_stdin_not_pty"'); + expect(statSync(monitorRoot).mode & 0o777).toBe(0o700); + expect(existsSync(join(monitorRoot, "pty-input-mode.sock"))).toBe(false); + } finally { + rmSync(monitorRoot, { force: true, recursive: true }); + } + }, + ); + } + + if (group === "readiness") { + it.runIf(process.platform === "linux").each(["absent", "ansi", "reordered"] as const)( + "keeps the monitor alive through SIGTERM, records an auto-message and PTY turn, sends /exit, strips launch authority, and ignores terminal copy evidence [%s] (#9160, #9384)", + (terminalCopy) => { + const { + baselineRemoved, + hostSessionResidue, + openshellCalls, + orphanedMonitorProcessIds, + orphanedTuiProcessIds, + ptyMonitorRemoved, + ptySocketReceipt, + result, + tuiProcessIds, + ttyObserved, + } = runLaunchSessionFixture("valid", terminalCopy); + + expect(ttyObserved, result.stderr).toBe(true); + expect(baselineRemoved).toBe(true); + expect(ptyMonitorRemoved).toBe(true); + expect(hostSessionResidue).toEqual([]); + expect(openshellCalls.length).toBeGreaterThan(3); + expect(openshellCalls.every((call) => call.authorityNames.length === 0)).toBe(true); + expect(openshellCalls.some((call) => call.argv.includes("baseline"))).toBe(true); + expect( + openshellCalls.some((call) => call.argv.includes(OPENCLAW_PTY_MONITOR_STARTER_SCRIPT)), + ).toBe(true); + expect(tuiProcessIds).toHaveLength(1); + expect(orphanedMonitorProcessIds).toEqual([]); + expect(orphanedTuiProcessIds).toEqual([]); + expect(ptySocketReceipt).toEqual({ + rootMode: 0o700, + rootUid: process.getuid?.(), + socketIsSocket: true, + socketMode: 0o600, + socketNlink: 1, + socketUid: process.getuid?.(), + }); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + }, + ); + + it.runIf(process.platform === "linux")( + "waits for OpenClaw input mode and accepts a clean exit after two turns (#9160, #9384)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "delayed-input-attachment", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + }, + ); + + it.runIf(process.platform === "linux")( + "waits for OpenClaw startup to accept its auto-message before submitting one PTY input (#9384)", + () => { + const fixture = runLaunchSessionFixture("delayed-tui-ready", "absent"); + + expect(fixture.earlyInputObserved, fixture.result.stderr).toBe(false); + expect(fixture.baselineRemoved, fixture.result.stderr).toBe(true); + expect(fixture.result.status, fixture.result.stderr).toBe(0); + }, + ); + + it.runIf(process.platform === "linux" && process.getuid?.() !== 0)( + "uses the inherited PTY descriptor when the sandbox user cannot reopen the device path (#9384)", + () => { + const { + baselineRemoved, + hostSessionResidue, + orphanedMonitorProcessIds, + orphanedTuiProcessIds, + ptyPathQueryResult, + ptyMonitorRemoved, + result, + ttyObserved, + } = runLaunchSessionFixture("pty-path-unreadable", "absent"); + + expect(ttyObserved, result.stderr).toBe(true); + expect(ptyPathQueryResult, result.stderr).toEqual({ errorCode: null, status: 1 }); + expect(baselineRemoved, result.stderr).toBe(true); + expect(ptyMonitorRemoved, result.stderr).toBe(true); + expect(hostSessionResidue, result.stderr).toEqual([]); + expect(orphanedMonitorProcessIds, result.stderr).toEqual([]); + expect(orphanedTuiProcessIds, result.stderr).toEqual([]); + expect(result.signal, result.stderr).toBeNull(); + expect(result.status, result.stderr).toBe(0); + }, + ); + + it.runIf(process.platform === "linux").each([ + { + mode: "pty-socket-invalid", + reason: "pty_termios_response_invalid", + behavior: "malformed response from a replacement socket", + expectedDiagnostic: { reason: "pty_termios_response_invalid" }, + monitorRemoved: false, + }, + { + mode: "pty-socket-permission", + reason: "pty_socket_invalid", + behavior: "PTY monitor socket whose mode is not 0600", + expectedDiagnostic: { reason: "pty_socket_invalid" }, + monitorRemoved: false, + }, + { + mode: "pty-response-identity", + reason: "pty_identity_changed", + behavior: "response with PTY identity that does not match its device path", + expectedDiagnostic: { reason: "pty_identity_changed" }, + monitorRemoved: false, + }, + { + mode: "pty-termios-unavailable", + reason: "pty_termios_unavailable", + behavior: "unavailable PTY input-mode evidence", + expectedDiagnostic: { + reason: "pty_termios_unavailable", + sttyStatus: 1, + sttySignal: null, + sttyErrorCode: null, + sttyStderr: "fixture stty denied", + }, + monitorRemoved: true, + }, + ] as const)( + "rejects $behavior before PTY input (#9160, #9384)", + ({ expectedDiagnostic, mode, monitorRemoved, reason }) => { + const { + baselineRemoved, + earlyInputObserved, + orphanedMonitorProcessIds, + orphanedTuiProcessIds, + ptyMonitorRemoved, + result, + ttyObserved, + } = runLaunchSessionFixture(mode, "absent"); + const failureEvidence = `${mode}: ${result.stderr}`; + const diagnostics = result.stderr.split("\n").flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + }); + + expect(ttyObserved, failureEvidence).toBe(true); + expect(orphanedMonitorProcessIds, failureEvidence).toEqual([]); + expect(orphanedTuiProcessIds, failureEvidence).toEqual([]); + expect(baselineRemoved, failureEvidence).toBe(true); + expect(earlyInputObserved, failureEvidence).toBe(false); + expect(result.signal, failureEvidence).toBeNull(); + expect(result.status, failureEvidence).toBe(1); + expect(result.stderr, failureEvidence).toContain(`"reason":"${reason}"`); + expect(diagnostics, failureEvidence).toEqual( + expect.arrayContaining([expect.objectContaining(expectedDiagnostic)]), + ); + expect(ptyMonitorRemoved, failureEvidence).toBe(monitorRemoved); + }, + testTimeout(20_000), + ); + + it.runIf(process.platform === "linux")( + "submits each PTY turn once while structured recording is delayed (#9160)", + () => { + const { baselineRemoved, pendingQualificationObserved, result, ttyObserved } = + runLaunchSessionFixture("delayed-recording", "absent"); + + expect(ttyObserved).toBe(true); + expect(pendingQualificationObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + }, + ); + + it.runIf(process.platform === "linux")( + "fails when the PTY remains in canonical input mode until the session deadline (#9160)", + () => { + const { + baselineRemoved, + orphanedMonitorProcessIds, + ptyMonitorRemoved, + result, + ttyObserved, + } = runLaunchSessionFixture("input-mode-timeout", "absent"); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(ptyMonitorRemoved).toBe(true); + expect(orphanedMonitorProcessIds).toEqual([]); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "launch did not observe noncanonical PTY input mode before the session deadline or before the PTY child process exited", + ); + expect(result.stderr).toContain('"reason":"pty_input_canonical"'); + }, + testTimeout(20_000), + ); + } + + if (group === "termios") { + it.runIf(process.platform === "linux")( + "requires a current noncanonical observation after the PTY returns to canonical mode (#9384)", + () => { + const { + baselineRemoved, + canonicalRestored, + earlyInputObserved, + orphanedMonitorProcessIds, + ptyMonitorRemoved, + result, + ttyObserved, + } = runLaunchSessionFixture("restored-canonical-timeout", "absent"); + + expect(ttyObserved).toBe(true); + expect(canonicalRestored).toBe(true); + expect(earlyInputObserved).toBe(false); + expect(baselineRemoved).toBe(true); + expect(ptyMonitorRemoved).toBe(true); + expect(orphanedMonitorProcessIds).toEqual([]); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain('"reason":"pty_input_canonical"'); + }, + testTimeout(30_000), + ); + } + + if (group === "failure-paths") { + it.runIf(process.platform === "linux")( + "fails when the PTY monitor socket remains missing until the session deadline (#9160)", + () => { + const { baselineRemoved, ptyMonitorRemoved, result, ttyObserved } = runLaunchSessionFixture( + "pty-socket-timeout", + "absent", + ); + + expect(ttyObserved).toBe(false); + expect(baselineRemoved).toBe(true); + expect(ptyMonitorRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain('"reason":"pty_socket_missing"'); + }, + testTimeout(20_000), + ); + + it.runIf(process.platform === "linux")( + "reports missing structured turns before the PTY child timeout (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "recording-timeout", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "launch did not record the required structured session turns", + ); + }, + ); + + it.runIf(process.platform === "linux")( + "rejects out-of-order structured records even when the PTY process remains active (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "invalid-order", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + }, + ); + + it.runIf(process.platform === "linux")( + "rejects a late extra structured record before baseline cleanup (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "late-extra", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "launch final structured session evidence did not qualify (status 2)", + ); + expect(result.stderr).toContain('"reason":"extra_message"'); + }, + ); + + it.runIf(process.platform === "linux")( + "propagates a nonzero TUI exit after two structured turns (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "nonzero", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(23); + }, + ); + + it.runIf(process.platform === "linux")( + "fails when a qualified PTY session cannot run PTY monitor cleanup (#9160)", + () => { + const { hostSessionResidue, orphanedTuiProcessIds, ptyMonitorRemoved, result } = + runLaunchSessionFixture("pty-cleanup-failure", "absent"); + + expect(ptyMonitorRemoved).toBe(false); + expect(hostSessionResidue).toEqual([]); + expect(orphanedTuiProcessIds).toEqual([]); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain("launch could not remove the PTY monitor"); + }, + ); + + it.runIf(process.platform === "linux")( + "refuses to remove an unknown entry from the mode-0700 PTY monitor directory (#9160)", + () => { + const { orphanedTuiProcessIds, ptyMonitorRemoved, result } = runLaunchSessionFixture( + "pty-cleanup-unknown-entry", + "absent", + ); + + expect(ptyMonitorRemoved).toBe(false); + expect(orphanedTuiProcessIds).toEqual([]); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain('"reason":"pty_monitor_cleanup_unknown_entry"'); + }, + ); + + it.runIf(process.platform === "linux")( + "fails when a successful PTY session cannot remove its structured baseline (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "cleanup-failure", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(false); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain("structured session baseline cleanup failed"); + }, + ); + + it.runIf(process.platform === "linux")( + "preserves a nonzero PTY exit when PTY monitor cleanup also fails (#9160)", + () => { + const { baselineRemoved, ptyMonitorRemoved, result, ttyObserved } = runLaunchSessionFixture( + "nonzero-pty-cleanup-failure", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(ptyMonitorRemoved).toBe(false); + expect(result.signal).toBeNull(); + expect(result.status).toBe(23); + }, + ); + } + + if (group === "termios") { + it.runIf(process.platform === "linux")( + "rejects a relative OpenShell command before launching a host command (#9160)", + async () => { + let commandCallCount = 0; + const host = { + command: async () => { + commandCallCount += 1; + return { exitCode: 0, signal: null, stderr: "", stdout: "" }; + }, + openshellCommandPath: "openshell", + }; + + await expect( + runOpenClawLaunchSession({ + artifactName: "relative-openshell-command", + cliCommand: "node", + env: {}, + host: host as never, + redactionValues: [], + sandboxName: "alpha", + }), + ).rejects.toThrow("launch session coverage requires an absolute OpenShell command path"); + expect(commandCallCount).toBe(0); + }, + ); + + it.each(["", "relative-tmp", "/tmp/absolute-tmp"])( + "passes an absolute host temporary root for empty, relative, or absolute TMPDIR input [%s] (#9160)", + async (root) => { + const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + const roots: Array = []; + const host = { + command: async ( + _command: string, + _args: string[], + options?: { env?: NodeJS.ProcessEnv }, + ) => { + roots.push(options?.env?.NEMOCLAW_LAUNCH_HOST_TMP_ROOT); + return { exitCode: 0, signal: null, stdout: "", stderr: "" }; + }, + openshellCommandPath: "/usr/bin/openshell", + }; + + try { + await runOpenClawLaunchSession({ + artifactName: "host-temporary-root", + cliCommand: "node", + env: { TMPDIR: root }, + host: host as never, + redactionValues: [], + sandboxName: "alpha", + }); + + expect(roots).toEqual([root === "" ? resolve("/tmp") : resolve(root)]); + } finally { + platform.mockRestore(); + } + }, + ); + + it.runIf(process.platform === "linux")( + "runs the producer then two PTY launch sessions under one lease (#8942, #9023, #9160)", + async () => { + const calls: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = []; + let launchPhaseStartedAtCallCount = -1; + const host = { + command: async ( + command: string, + args: string[], + options?: { env?: NodeJS.ProcessEnv }, + ) => { + calls.push({ command, args, env: options?.env }); + return { exitCode: 0, signal: null, stdout: "", stderr: "" }; + }, + openshellCommandPath: "/usr/bin/openshell", + }; + + await runOpenClawLaunchReadinessLeaseTurns({ + artifactName: "lease-turn", + cliCommand: "node", + cliEntrypoint: "/repo/bin/nemoclaw.js", + env: {}, + exitCommand: "/exit", + host: host as never, + redactionValues: [], + sandboxName: "alpha", + beforeLaunchTurns: () => { + launchPhaseStartedAtCallCount = calls.length; + }, + }); + + expect(launchPhaseStartedAtCallCount).toBe(1); + expect(calls).toHaveLength(3); + expect(calls[0]).toMatchObject({ + command: "node", + args: ["/repo/bin/nemoclaw.js", "alpha", "connect", "--probe-only"], + }); + expect(calls.slice(1).map((call) => call.command)).toEqual(["bash", "bash"]); + expect(calls.slice(1).map((call) => call.args)).toEqual([ + ["-lc", LAUNCH_TURN_SCRIPT], + ["-lc", LAUNCH_TURN_SCRIPT], + ]); + expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_LAUNCH_EXIT_COMMAND)).toEqual([ + "/exit", + "/exit", + ]); + expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_OPENSHELL_COMMAND)).toEqual([ + "/usr/bin/openshell", + "/usr/bin/openshell", + ]); + calls.slice(1).forEach((call) => { + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_EXPECTED_REPLY"); + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT"); + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_PROMPT"); + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_READY_TEXT"); + expect(typeof call.env?.NEMOCLAW_LAUNCH_FIRST_INPUT).toBe("string"); + expect(typeof call.env?.NEMOCLAW_LAUNCH_SECOND_INPUT).toBe("string"); + expect(call.env?.NEMOCLAW_LAUNCH_FIRST_INPUT).not.toBe( + call.env?.NEMOCLAW_LAUNCH_SECOND_INPUT, + ); + expect(call.env?.NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT).toBe( + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + ); + expect(call.env?.NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT).toBe( + OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, + ); + expect(call.env?.NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT).toBe( + OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, + ); + expect(call.env?.NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT).toBe( + OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, + ); + }); + }, + ); + } +} diff --git a/test/e2e/support/launch-agent-turn-termios.test.ts b/test/e2e/support/launch-agent-turn-termios.test.ts new file mode 100644 index 00000000000..235ecd17e03 --- /dev/null +++ b/test/e2e/support/launch-agent-turn-termios.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerLaunchAgentTurnTests } from "./launch-agent-turn-suite"; + +registerLaunchAgentTurnTests("termios"); diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts deleted file mode 100644 index ff96afb4a73..00000000000 --- a/test/e2e/support/launch-agent-turn.test.ts +++ /dev/null @@ -1,1499 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { - appendFileSync, - chmodSync, - closeSync, - constants, - existsSync, - fstatSync, - fsyncSync, - ftruncateSync, - mkdtempSync, - mkdirSync, - openSync, - readFileSync, - readdirSync, - rmSync, - type Stats, - statSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { createServer } from "node:net"; -import { join, resolve } from "node:path"; - -import { expect, it, vi } from "vitest"; -import { - isSubprocessEnvNameAllowed, - SUBPROCESS_ENV_ALLOWED_NAMES, - SUBPROCESS_ENV_ALLOWED_PREFIXES, -} from "../../../src/lib/subprocess-env"; -import { testTimeout } from "../../helpers/timeouts"; -import { - LAUNCH_TURN_SCRIPT, - OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, - OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, - OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - runOpenClawLaunchSession, - runOpenClawLaunchReadinessLeaseTurns, -} from "../live/launch-agent-turn.ts"; -const PROCESS_EXIT_WAIT = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); -type SessionRecords = Record; -type FixtureMode = - | "cleanup-failure" - | "delayed-input-attachment" - | "delayed-recording" - | "delayed-tui-ready" - | "input-mode-timeout" - | "invalid-order" - | "late-extra" - | "nonzero" - | "nonzero-pty-cleanup-failure" - | "pty-cleanup-failure" - | "pty-cleanup-unknown-entry" - | "pty-socket-invalid" - | "pty-socket-permission" - | "pty-response-identity" - | "pty-socket-timeout" - | "pty-path-unreadable" - | "pty-termios-unavailable" - | "recording-timeout" - | "restored-canonical-timeout" - | "valid"; - -function message(role: "assistant" | "user", content = "nonempty"): string { - return JSON.stringify({ - message: { content: [{ text: content, type: "text" }], role }, - type: "message", - }); -} - -function emptyMessage(role: "assistant" | "user"): string { - return JSON.stringify({ message: { content: [], role }, type: "message" }); -} - -function writeSessionRecords( - root: string, - sessions: SessionRecords, - append: boolean, - finalNewline = true, -): void { - for (const [sessionId, records] of Object.entries(sessions)) { - const filePath = join(root, `${sessionId}.jsonl`); - const body = records.length > 0 ? `${records.join("\n")}${finalNewline ? "\n" : ""}` : ""; - const writeRecords = append ? appendFileSync : writeFileSync; - writeRecords(filePath, body); - } -} - -function withOwnedFixtureFile( - filePath: string, - flags: number, - action: (descriptor: number, stats: Stats) => T, -): T { - const descriptor = openSync(filePath, flags | constants.O_NOFOLLOW, 0o600); - try { - const stats = fstatSync(descriptor); - expect([stats.isFile(), stats.uid, stats.mode & 0o777, stats.nlink]).toEqual([ - true, - process.getuid?.(), - 0o600, - 1, - ]); - return action(descriptor, stats); - } finally { - closeSync(descriptor); - } -} - -function runEvidenceFixture(input: { - after: SessionRecords; - afterFinalNewline?: boolean; - before?: SessionRecords; - expectedTurns: number; -}) { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-evidence-")); - const runId = randomUUID().replaceAll("-", ""); - const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - const sessionRoot = join(fixtureRoot, "sessions"); - mkdirSync(sessionRoot); - try { - writeSessionRecords(sessionRoot, input.before ?? {}, false); - const baseline = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "baseline", - sessionRoot, - baselinePath, - "", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - writeSessionRecords(sessionRoot, input.after, true, input.afterFinalNewline ?? true); - const qualification = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "qualify", - sessionRoot, - baselinePath, - String(input.expectedTurns), - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - const baselineFile = withOwnedFixtureFile( - baselinePath, - constants.O_RDONLY, - (descriptor, stats) => ({ body: readFileSync(descriptor, "utf8"), stats }), - ); - return { - baseline, - baselineKeys: Object.keys(JSON.parse(baselineFile.body)).sort(), - baselineMode: baselineFile.stats.mode & 0o777, - baselineNlink: baselineFile.stats.nlink, - baselineUid: baselineFile.stats.uid, - qualification, - }; - } finally { - rmSync(fixtureRoot, { force: true, recursive: true }); - rmSync(baselinePath, { force: true }); - rmSync(`${baselinePath}.tmp`, { force: true }); - rmSync(ptyMonitorRoot, { force: true, recursive: true }); - } -} - -function runBaselineMutationFixture(mutation: "invalid" | "removed" | "rewritten" | "truncated") { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-baseline-")); - const runId = randomUUID().replaceAll("-", ""); - const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - const sessionRoot = join(fixtureRoot, "sessions"); - const sessionPath = join(sessionRoot, "session-a.jsonl"); - mkdirSync(sessionRoot); - writeSessionRecords(sessionRoot, { "session-a": [message("user"), message("assistant")] }, false); - try { - const baseline = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "baseline", - sessionRoot, - baselinePath, - "", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - const applyMutation: Record void> = { - invalid: () => - withOwnedFixtureFile(baselinePath, constants.O_WRONLY, (descriptor) => { - ftruncateSync(descriptor, 0); - writeFileSync(descriptor, "{}"); - fsyncSync(descriptor); - }), - removed: () => rmSync(sessionPath), - rewritten: () => - writeFileSync( - sessionPath, - readFileSync(sessionPath, "utf8").replace("nonempty", "changed!"), - ), - truncated: () => writeFileSync(sessionPath, ""), - }; - applyMutation[mutation](); - const qualification = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "qualify", - sessionRoot, - baselinePath, - "1", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - return { baseline, qualification }; - } finally { - rmSync(fixtureRoot, { force: true, recursive: true }); - rmSync(baselinePath, { force: true }); - rmSync(`${baselinePath}.tmp`, { force: true }); - rmSync(ptyMonitorRoot, { force: true, recursive: true }); - } -} - -it("reports a residual PTY monitor socket without removing it (#9384)", async () => { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-monitor-cleanup-")); - const runId = randomUUID().replaceAll("-", ""); - const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - const socketPath = join(ptyMonitorRoot, "pty-input-mode.sock"); - const server = createServer(); - mkdirSync(ptyMonitorRoot, { mode: 0o700 }); - chmodSync(ptyMonitorRoot, 0o700); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(socketPath, resolve); - }); - chmodSync(socketPath, 0o600); - try { - const cleanup = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "cleanup-pty", - fixtureRoot, - baselinePath, - "", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - - expect(cleanup.status).toBe(2); - expect(cleanup.stderr).toContain('"reason":"pty_monitor_socket_still_present"'); - expect(existsSync(socketPath)).toBe(true); - } finally { - await new Promise((resolve) => server.close(() => resolve())); - rmSync(fixtureRoot, { force: true, recursive: true }); - rmSync(ptyMonitorRoot, { force: true, recursive: true }); - } -}); - -function runLaunchSessionFixture(mode: FixtureMode, terminalCopy: "absent" | "ansi" | "reordered") { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-turn-")); - const canonicalRestoredMarker = join(fixtureRoot, "canonical-restored"); - const earlyInputMarker = join(fixtureRoot, "early-input"); - const fakeLaunch = join(fixtureRoot, "openclaw"); - const fakeOpenshell = join(fixtureRoot, "openshell"); - const fakeStty = join(fixtureRoot, "stty"); - const monitorPidPath = join(fixtureRoot, "monitor-pid"); - const sessionRoot = join(fixtureRoot, "sessions"); - const tuiPidsPath = join(fixtureRoot, "tui-pids"); - const ttyMarker = join(fixtureRoot, "tty-observed"); - const openshellCallsRoot = join(fixtureRoot, "openshell-calls"); - const pendingQualificationMarker = join(fixtureRoot, "pending-qualification-observed"); - const ptyPathUnreadableMarker = join(fixtureRoot, "pty-path-unreadable"); - const ptySocketReceiptPath = join(fixtureRoot, "pty-socket-receipt.json"); - const runId = randomUUID().replaceAll("-", ""); - const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - mkdirSync(sessionRoot); - mkdirSync(openshellCallsRoot, { mode: 0o700 }); - writeFileSync( - join(fixtureRoot, ".bash_profile"), - 'export PATH="$NEMOCLAW_FIXTURE_BIN_ROOT:$PATH"\n', - ); - try { - writeFileSync( - join(fixtureRoot, "sleep"), - '#!/bin/bash\n[[ "$1:$NEMOCLAW_FIXTURE_MODE" =~ ^0\.05:pty-(socket-(invalid|permission)|response-identity)$ ]] || exec /usr/bin/sleep "$@"\n', - { mode: 0o755 }, - ); - writeFileSync( - fakeStty, - String.raw`#!/bin/bash -marker=${JSON.stringify(ttyMarker)} -if [[ ! -e "$marker" ]]; then - exec /usr/bin/stty "$@" -fi -printf 'fixture stty denied\n' >&2 -exit 1 -`, - ); - writeFileSync( - fakeLaunch, - String.raw`#!/usr/bin/env node -const childProcess = require("node:child_process"); -const fs = require("node:fs"); -const net = require("node:net"); -const readline = require("node:readline"); - -const mode = process.env.NEMOCLAW_FIXTURE_MODE; -if (process.argv[2] !== "tui") { - const allowedNames = new Set(${JSON.stringify(SUBPROCESS_ENV_ALLOWED_NAMES)}); - const allowedPrefixes = ${JSON.stringify(SUBPROCESS_ENV_ALLOWED_PREFIXES)}; - const subprocessEnv = Object.fromEntries( - Object.entries(process.env).filter( - ([name, value]) => - value !== undefined && - (allowedNames.has(name) || - allowedPrefixes.some((prefix) => name.startsWith(prefix)) || - name.startsWith("NEMOCLAW_FIXTURE_")), - ), - ); - const args = [ - "sandbox", - "exec", - "--name", - process.argv[3], - "-g", - "fixture-gateway", - "--tty", - "--timeout", - "0", - "--", - "/bin/bash", - "--noprofile", - "--norc", - "-p", - "-c", - process.env.NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, - "nemoclaw-runtime-env", - "bash", - "-lc", - "openclaw tui", - ]; - const result = childProcess.spawnSync(process.env.NEMOCLAW_OPENSHELL_BIN, args, { - env: subprocessEnv, - stdio: "inherit", - timeout: 14_000, - killSignal: "SIGKILL", - }); - process.exit(result.status ?? 66); -} - -(async () => { - if (!process.stdin.isTTY || !process.stdout.isTTY) process.exit(64); - fs.appendFileSync(process.env.NEMOCLAW_FIXTURE_TUI_PIDS, process.pid + "\n"); - const monitorRoot = process.env.NEMOCLAW_FIXTURE_PTY_MONITOR_ROOT; - const socketPath = monitorRoot + "/pty-input-mode.sock"; - const socketDeadline = Date.now() + 2_000; - while (!fs.existsSync(socketPath)) { - if (Date.now() >= socketDeadline) process.exit(70); - await new Promise((resolve) => setTimeout(resolve, 10)); - } - const socketStats = fs.lstatSync(socketPath); - const rootStats = fs.lstatSync(monitorRoot); - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_PTY_SOCKET_RECEIPT, JSON.stringify({ - rootMode: rootStats.mode & 0o777, - rootUid: rootStats.uid, - socketIsSocket: socketStats.isSocket(), - socketMode: socketStats.mode & 0o777, - socketNlink: socketStats.nlink, - socketUid: socketStats.uid, - })); - const monitorPid = fs.readdirSync("/proc").find((name) => { - try { - const commandLine = fs.readFileSync("/proc/" + name + "/cmdline", "utf8"); - const argv = commandLine.split("\0"); - return argv.includes("nemoclaw-pty-input-mode-monitor") && - argv.includes(process.env.NEMOCLAW_FIXTURE_RUN_ID); - } catch { - return false; - } - }); - if (!monitorPid) process.exit(71); - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_MONITOR_PID, monitorPid); - if (mode === "pty-socket-invalid" || mode === "pty-response-identity") { - fs.unlinkSync(socketPath); - const ttyPath = fs.realpathSync("/proc/self/fd/0"); - const ttyStats = fs.fstatSync(0, { bigint: true }); - const replacement = net.createServer((client) => { - const body = mode === "pty-socket-invalid" - ? "{}\n" - : JSON.stringify({ - ttyPath, - dev: ttyStats.dev.toString(), - ino: ttyStats.ino.toString(), - rdev: ttyStats.rdev === 0n ? "1" : "0", - state: "noncanonical", - status: null, - signal: null, - errorCode: null, - stderr: "", - }) + "\n"; - client.end(body); - }); - process.umask(0o177); - await new Promise((resolve, reject) => { - replacement.once("error", reject); - replacement.listen(socketPath, resolve); - }); - fs.chmodSync(socketPath, 0o600); - } - if (mode === "pty-socket-permission") fs.chmodSync(socketPath, 0); - switch (mode) { - case "pty-path-unreadable": { - const ttyPath = fs.realpathSync("/proc/self/fd/0"); - const ttyMode = fs.lstatSync(ttyPath).mode & 0o777; - fs.chmodSync(ttyPath, 0); - const pathQuery = childProcess.spawnSync( - "/usr/bin/stty", - ["-F", ttyPath, "-a"], - { encoding: "utf8" }, - ); - fs.writeFileSync( - process.env.NEMOCLAW_FIXTURE_PTY_PATH_UNREADABLE_MARKER, - JSON.stringify({ - errorCode: pathQuery.error?.code ?? null, - status: pathQuery.status, - }), - ); - if (pathQuery.status === 0) process.exit(69); - process.on("exit", () => { - try { fs.chmodSync(ttyPath, ttyMode); } catch {} - }); - break; - } - } - if (mode === "pty-cleanup-unknown-entry") { - fs.writeFileSync(monitorRoot + "/unexpected", "owned test residue"); - } - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_TTY_MARKER, ""); - const sessionFile = process.env.NEMOCLAW_FIXTURE_SESSION_FILE; - const terminalCopy = process.env.NEMOCLAW_FIXTURE_TERMINAL_COPY; - const messageIndex = process.argv.indexOf("--message"); - const firstInput = messageIndex === -1 ? "" : process.argv[messageIndex + 1]; - if (!firstInput) process.exit(75); - const append = (role, content) => fs.appendFileSync( - sessionFile, - JSON.stringify({ message: { content: [{ text: content, type: "text" }], role }, type: "message" }) + "\n", - ); - if (mode === "restored-canonical-timeout") { - process.stdin.setRawMode(true); - await new Promise((resolve) => setTimeout(resolve, 750)); - process.stdin.setRawMode(false); - append("user", firstInput); - append("assistant", "first response"); - const recordUnexpectedInput = () => - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER, ""); - process.stdin.on("data", recordUnexpectedInput); - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER, ""); - await new Promise((resolve) => setTimeout(resolve, 20_000)); - process.stdin.off("data", recordUnexpectedInput); - } - if (mode === "input-mode-timeout") { - append("user", firstInput); - append("assistant", "first response"); - await new Promise((resolve) => setTimeout(resolve, 10_000)); - } - if (mode === "delayed-input-attachment" || mode === "delayed-tui-ready") { - const recordEarlyInput = () => - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER, ""); - if (mode === "delayed-tui-ready") process.stdin.setRawMode(true); - process.stdin.on("data", recordEarlyInput); - await new Promise((resolve) => setTimeout(resolve, 1_500)); - process.stdin.off("data", recordEarlyInput); - if (fs.existsSync(process.env.NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER)) process.exit(67); - } - const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true }); - const ask = () => new Promise((resolve) => rl.question("", resolve)); - if (terminalCopy === "ansi") process.stdout.write("\u001b[2Kgateway connected | idle\r"); - if (terminalCopy === "reordered") process.stdout.write("idle | gateway connected\n"); - - if (mode === "delayed-recording") { - const publicationDeadline = Date.now() + 2_000; - while (!fs.existsSync(process.env.NEMOCLAW_FIXTURE_PENDING_QUALIFICATION_MARKER)) { - if (Date.now() >= publicationDeadline) process.exit(68); - await new Promise((resolve) => setTimeout(resolve, 10)); - } - } - if (mode === "recording-timeout") await new Promise((resolve) => setTimeout(resolve, 10_000)); - if (mode === "invalid-order") { - append("assistant", "response before input"); - append("user", firstInput); - } else { - append("user", firstInput); - append("assistant", "first response"); - } - - process.kill(Number(monitorPid), "SIGTERM"); - await new Promise((resolve) => setTimeout(resolve, 50)); - const monitorStat = fs.readFileSync("/proc/" + monitorPid + "/stat", "utf8"); - const monitorState = monitorStat ? monitorStat.slice(monitorStat.lastIndexOf(") ") + 2)[0] : null; - if (!monitorState || monitorState === "Z") process.exit(71); - - const second = await ask(); - append("user", second); - append("assistant", "second response"); - if (mode === "delayed-input-attachment") process.exit(0); - const exitCommand = await ask(); - if (mode === "late-extra") append("user", firstInput); - rl.close(); - if (exitCommand !== "/exit") process.exit(65); - process.exit(mode.includes("nonzero") ? 23 : 0); -})().catch(() => process.exit(66)); -`, - ); - writeFileSync( - fakeOpenshell, - String.raw`#!/usr/bin/env bash -set -euo pipefail -node -e ' -const crypto = require("node:crypto"); -const fs = require("node:fs"); -const path = require("node:path"); -const authorityNames = Object.keys(process.env) - .filter( - (name) => - name === "NEMOCLAW_OPENSHELL_BIN" || - name === "NEMOCLAW_OPENSHELL_COMMAND" || - name.startsWith("NEMOCLAW_LAUNCH_") || - name.startsWith("OPENSHELL_NEMOCLAW_LAUNCH_"), - ) - .sort(); -fs.writeFileSync( - path.join( - process.env.NEMOCLAW_FIXTURE_OPENSHELL_CALLS, - process.pid + "-" + crypto.randomUUID() + ".json", - ), - JSON.stringify({ argv: process.argv.slice(1), authorityNames }), - { flag: "wx", mode: 0o600 }, -); -' "$@" -while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done -[[ "$#" -gt 0 ]] -shift -case "$NEMOCLAW_FIXTURE_MODE:$4" in - pty-socket-invalid:input-mode|pty-socket-permission:input-mode|pty-response-identity:input-mode) - [[ -e "$NEMOCLAW_FIXTURE_TTY_MARKER" ]] || exit 1 - ;; -esac -if [[ "$NEMOCLAW_FIXTURE_MODE" == "restored-canonical-timeout" && "$4" == "input-mode" ]]; then - for _ in {1..200}; do - [[ ! -e "$NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER" ]] || break - sleep 0.01 - done - [[ -e "$NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER" ]] || exit 1 -fi -if [[ "$NEMOCLAW_FIXTURE_MODE" == "cleanup-failure" && "$4" == "cleanup-baseline" ]]; then - exit 71 -fi -if [[ ( "$NEMOCLAW_FIXTURE_MODE" == "pty-cleanup-failure" || "$NEMOCLAW_FIXTURE_MODE" == "nonzero-pty-cleanup-failure" ) && "$4" == "cleanup-pty" ]]; then - exit 71 -fi -if [[ "$NEMOCLAW_FIXTURE_MODE" == "pty-socket-timeout" && "$4" == "$NEMOCLAW_FIXTURE_RUN_ID" ]]; then - exec node -e 'setTimeout(() => process.exit(0), 10_000)' -fi -if [[ "$NEMOCLAW_FIXTURE_MODE" == "delayed-recording" && "$4" == "qualify" && "$7" == "1" ]]; then - set +e - "$@" - status=$? - set -e - [[ "$status" != "1" ]] || : > "$NEMOCLAW_FIXTURE_PENDING_QUALIFICATION_MARKER" - exit "$status" -fi -exec "$@" -`, - ); - chmodSync(fakeLaunch, 0o755); - chmodSync(fakeOpenshell, 0o755); - chmodSync(fakeStty, 0o755); - - const unavailablePtyMonitorStarterScript = OPENCLAW_PTY_MONITOR_STARTER_SCRIPT.replace( - 'const termiosCommand = "/usr/bin/stty";', - `const termiosCommand = ${JSON.stringify(fakeStty)};`, - ); - expect(unavailablePtyMonitorStarterScript).not.toBe(OPENCLAW_PTY_MONITOR_STARTER_SCRIPT); - const ptyMonitorStarterScript = - mode === "pty-termios-unavailable" - ? unavailablePtyMonitorStarterScript - : OPENCLAW_PTY_MONITOR_STARTER_SCRIPT; - - const result = spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { - encoding: "utf8", - killSignal: "SIGKILL", - env: { - ...process.env, - HOME: fixtureRoot, - NEMOCLAW_FIXTURE_BIN_ROOT: fixtureRoot, - NEMOCLAW_FIXTURE_CANONICAL_RESTORED_MARKER: canonicalRestoredMarker, - NEMOCLAW_FIXTURE_EARLY_INPUT_MARKER: earlyInputMarker, - NEMOCLAW_FIXTURE_MODE: mode, - NEMOCLAW_FIXTURE_MONITOR_PID: monitorPidPath, - NEMOCLAW_FIXTURE_OPENSHELL_CALLS: openshellCallsRoot, - NEMOCLAW_FIXTURE_PENDING_QUALIFICATION_MARKER: pendingQualificationMarker, - NEMOCLAW_FIXTURE_PTY_MONITOR_ROOT: ptyMonitorRoot, - NEMOCLAW_FIXTURE_PTY_PATH_UNREADABLE_MARKER: ptyPathUnreadableMarker, - NEMOCLAW_FIXTURE_PTY_SOCKET_RECEIPT: ptySocketReceiptPath, - NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), - NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, - NEMOCLAW_FIXTURE_RUN_ID: runId, - NEMOCLAW_FIXTURE_TUI_PIDS: tuiPidsPath, - NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, - NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, - NEMOCLAW_LAUNCH_ENTRYPOINT: "", - NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", - NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", - NEMOCLAW_LAUNCH_HOST_TMP_ROOT: fixtureRoot, - NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, - NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT: ptyMonitorStarterScript, - NEMOCLAW_LAUNCH_RUN_ID: runId, - NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT: OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, - NEMOCLAW_LAUNCH_SANDBOX: "sandbox", - NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: mode === "restored-canonical-timeout" - ? "10" - : mode === "pty-socket-timeout" - ? "5" - : mode.endsWith("-timeout") - ? "2" - : "230", - NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", - NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, - NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, - NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, - PATH: `${fixtureRoot}:${process.env.PATH ?? ""}`, - TERM: "xterm-256color", - }, - timeout: 15_000, - }); - - const tuiProcessIds = existsSync(tuiPidsPath) - ? readFileSync(tuiPidsPath, "utf8").trim().split("\n").filter(Boolean) - : []; - const monitorProcessIds = existsSync(monitorPidPath) - ? [readFileSync(monitorPidPath, "utf8").trim()].filter(Boolean) - : []; - const processExitDeadline = Date.now() + 1_500; - while ( - (tuiProcessIds.some((pid) => existsSync(`/proc/${pid}`)) || - monitorProcessIds.some((pid) => existsSync(`/proc/${pid}`))) && - Date.now() < processExitDeadline - ) { - Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); - } - return { - baselineRemoved: !existsSync(baselinePath), - canonicalRestored: existsSync(canonicalRestoredMarker), - earlyInputObserved: existsSync(earlyInputMarker), - hostSessionResidue: readdirSync(fixtureRoot).filter((name) => - name.startsWith("nemoclaw-launch-host."), - ), - orphanedMonitorProcessIds: monitorProcessIds.filter((pid) => existsSync(`/proc/${pid}`)), - orphanedTuiProcessIds: tuiProcessIds.filter((pid) => existsSync(`/proc/${pid}`)), - openshellCalls: readdirSync(openshellCallsRoot) - .sort() - .map( - (name) => - JSON.parse(readFileSync(join(openshellCallsRoot, name), "utf8")) as { - argv: string[]; - authorityNames: string[]; - }, - ), - pendingQualificationObserved: existsSync(pendingQualificationMarker), - ptyPathQueryResult: existsSync(ptyPathUnreadableMarker) - ? JSON.parse(readFileSync(ptyPathUnreadableMarker, "utf8")) - : null, - ptyMonitorRemoved: !existsSync(ptyMonitorRoot), - ptySocketReceipt: existsSync(ptySocketReceiptPath) - ? JSON.parse(readFileSync(ptySocketReceiptPath, "utf8")) - : null, - result, - tuiProcessIds, - ttyObserved: existsSync(ttyMarker), - }; - } finally { - const ptySocketPath = join(ptyMonitorRoot, "pty-input-mode.sock"); - existsSync(ptyMonitorRoot) ? chmodSync(ptyMonitorRoot, 0o700) : undefined; - existsSync(ptySocketPath) ? chmodSync(ptySocketPath, 0o600) : undefined; - rmSync(fixtureRoot, { force: true, recursive: true }); - rmSync(baselinePath, { force: true }); - rmSync(`${baselinePath}.tmp`, { force: true }); - rmSync(ptyMonitorRoot, { force: true, recursive: true }); - } -} - -function openShellLaunchArgv(sandboxName: string, gatewayArgs: string[]): string[] { - return [ - "sandbox", - "exec", - "--name", - sandboxName, - ...gatewayArgs, - "--tty", - "--timeout", - "0", - "--", - "/bin/bash", - "--noprofile", - "--norc", - "-p", - "-c", - OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, - "nemoclaw-runtime-env", - "bash", - "-lc", - "openclaw tui", - ]; -} - -function runOpenShellShimFixture(gatewayArgs: string[]) { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-shim-")); - const realOpenShell = join(fixtureRoot, "openshell-real"); - const shim = join(fixtureRoot, "openshell-shim"); - const callsPath = join(fixtureRoot, "calls.jsonl"); - const interceptPath = join(fixtureRoot, "intercept.json"); - const runId = randomUUID().replaceAll("-", ""); - const sandboxName = "sandbox"; - writeFileSync( - realOpenShell, - String.raw`#!/usr/bin/env node -require("node:fs").appendFileSync( - ${JSON.stringify(callsPath)}, - JSON.stringify({ - argv: process.argv.slice(2), - authorityNames: Object.keys(process.env) - .filter( - (name) => - name === "NEMOCLAW_OPENSHELL_BIN" || - name === "NEMOCLAW_OPENSHELL_COMMAND" || - name.startsWith("NEMOCLAW_LAUNCH_") || - name.startsWith("OPENSHELL_NEMOCLAW_LAUNCH_"), - ) - .sort(), - }) + "\n", -); -`, - ); - writeFileSync(shim, OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT); - chmodSync(realOpenShell, 0o755); - chmodSync(shim, 0o755); - const hostEnv = { - ...process.env, - NEMOCLAW_LAUNCH_COMMAND: "nemoclaw", - NEMOCLAW_LAUNCH_FIRST_INPUT: "fixture input", - NEMOCLAW_LAUNCH_INTERCEPT_PATH: interceptPath, - NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT: OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, - NEMOCLAW_LAUNCH_RUN_ID: runId, - NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT: OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, - NEMOCLAW_LAUNCH_SANDBOX: sandboxName, - NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, - NEMOCLAW_OPENSHELL_BIN: shim, - NEMOCLAW_OPENSHELL_COMMAND: realOpenShell, - OPENSHELL_NEMOCLAW_LAUNCH_INTERCEPT_PATH: interceptPath, - OPENSHELL_NEMOCLAW_LAUNCH_FIRST_INPUT: "fixture input", - OPENSHELL_NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT: OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, - OPENSHELL_NEMOCLAW_LAUNCH_REAL_COMMAND: realOpenShell, - OPENSHELL_NEMOCLAW_LAUNCH_RUN_ID: runId, - OPENSHELL_NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT: OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT, - OPENSHELL_NEMOCLAW_LAUNCH_SANDBOX: sandboxName, - }; - const env = Object.fromEntries( - Object.entries(hostEnv).filter( - (entry): entry is [string, string] => - entry[1] !== undefined && isSubprocessEnvNameAllowed(entry[0]), - ), - ); - const runShim = (args: string[], childEnv: NodeJS.ProcessEnv = env) => - spawnSync(process.execPath, [shim, ...args], { - encoding: "utf8", - env: childEnv, - timeout: 2_000, - killSignal: "SIGKILL", - }); - const passThroughArgv = ["sandbox", "exec", "--name", sandboxName, "--", "true"]; - const ttyPassThroughArgv = [ - "sandbox", - "exec", - "--name", - sandboxName, - "--tty", - "--timeout", - "0", - "--", - "bash", - "-lc", - "printf '%s\\n' --tty", - ]; - const exactArgv = openShellLaunchArgv(sandboxName, gatewayArgs); - const malformedArgv = [...exactArgv]; - const ttyIndex = malformedArgv.indexOf("--tty"); - malformedArgv.splice(ttyIndex, 3, "--timeout", "0", "--tty"); - try { - const passThrough = runShim(passThroughArgv, hostEnv); - const ttyPassThrough = runShim(ttyPassThroughArgv); - const malformed = runShim(malformedArgv); - const invalidFirstInput = runShim(exactArgv, { - ...env, - OPENSHELL_NEMOCLAW_LAUNCH_FIRST_INPUT: "", - }); - const intercepted = runShim(exactArgv); - const duplicate = runShim(exactArgv); - const records = readFileSync(callsPath, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as { argv: string[]; authorityNames: string[] }); - return { - authorityNames: records.map((record) => record.authorityNames), - calls: records.map((record) => record.argv), - duplicate, - exactArgv, - interceptMode: statSync(interceptPath).mode & 0o777, - intercepted, - invalidFirstInput, - malformed, - passThrough, - passThroughArgv, - ttyPassThrough, - ttyPassThroughArgv, - monitorRoot: `/tmp/nemoclaw-launch-turn-${runId}`, - runId, - }; - } finally { - rmSync(fixtureRoot, { force: true, recursive: true }); - } -} - -it("qualifies two ordered structured turns without comparing message content (#9160)", () => { - const { baseline, baselineKeys, baselineMode, baselineNlink, baselineUid, qualification } = - runEvidenceFixture({ - after: { - "session-a": [ - message("user", "first arbitrary input"), - message("assistant", "first arbitrary response"), - message("user", "different second input"), - message("assistant", "different second response"), - ], - }, - expectedTurns: 2, - }); - - expect(baseline.status).toBe(0); - expect(baselineKeys).toEqual(["schemaVersion", "sessions"]); - expect(baselineMode).toBe(0o600); - expect(baselineNlink).toBe(1); - expect(baselineUid).toBe(process.getuid?.()); - expect(qualification.status).toBe(0); -}); - -it("keeps a partial structured turn pending (#9160)", () => { - const { baseline, qualification } = runEvidenceFixture({ - after: { "session-a": [message("user")] }, - expectedTurns: 1, - }); - - expect(baseline.status).toBe(0); - expect(qualification.status).toBe(1); -}); - -it("does not qualify structured turns recorded before the baseline (#9160)", () => { - const { baseline, qualification } = runEvidenceFixture({ - before: { "session-a": [message("user"), message("assistant")] }, - after: {}, - expectedTurns: 1, - }); - - expect(baseline.status).toBe(0); - expect(qualification.status).toBe(1); -}); - -it.each([ - { "session-a": [message("assistant"), message("user")] }, - { "session-a": [message("user"), message("user"), message("assistant")] }, - { "session-a": [message("user"), message("assistant"), message("assistant")] }, - { "session-a": [message("user"), "not-json", message("assistant")] }, - { "session-a": [emptyMessage("user"), message("assistant")] }, - { "session-a": [message("user"), message("assistant")], "session-b": [message("user")] }, -] as SessionRecords[])( - "rejects malformed, empty, duplicated, extra, out-of-order, or cross-session records [case %#] (#9160)", - (after) => { - const { baseline, qualification } = runEvidenceFixture({ after, expectedTurns: 1 }); - expect(baseline.status).toBe(0); - expect(qualification.status).toBe(2); - }, -); - -it("rejects an unterminated appended session record (#9160)", () => { - const { baseline, qualification } = runEvidenceFixture({ - after: { - "session-a": [ - message("user"), - message("assistant"), - message("user"), - message("assistant"), - message("user"), - ], - }, - afterFinalNewline: false, - expectedTurns: 2, - }); - - expect(baseline.status).toBe(0); - expect(qualification.status).toBe(2); -}); - -it.each(["invalid", "removed", "rewritten", "truncated"] as const)( - "rejects an invalid baseline or a removed, rewritten, or truncated session [case %#] (#9160)", - (mutation) => { - const { baseline, qualification } = runBaselineMutationFixture(mutation); - expect(baseline.status).toBe(0); - expect(qualification.status).toBe(2); - }, -); - -it.each([[], ["-g", "fixture-gateway"]].map((gatewayArgs) => [gatewayArgs] as const))( - "intercepts one OpenClaw launch, preserves pass-through argv, and strips launch authority from filtered and inherited environments [case %#] (#9160)", - (gatewayArgs) => { - const fixture = runOpenShellShimFixture(gatewayArgs); - const separator = fixture.exactArgv.indexOf("--"); - const expectedRemote = fixture.exactArgv.slice(separator + 1); - - expect(fixture.passThrough.status, fixture.passThrough.stderr).toBe(0); - expect(fixture.ttyPassThrough.status, fixture.ttyPassThrough.stderr).toBe(0); - expect(fixture.malformed.status).toBe(73); - expect(fixture.malformed.stderr).toContain('"reason":"openshell_launch_invocation_invalid"'); - expect(fixture.invalidFirstInput.status).toBe(73); - expect(fixture.invalidFirstInput.stderr).toContain( - '"reason":"openshell_shim_first_input_invalid"', - ); - expect(fixture.intercepted.status, fixture.intercepted.stderr).toBe(0); - expect(fixture.duplicate.status).toBe(73); - expect(fixture.duplicate.stderr).toContain('"reason":"openshell_launch_intercept_duplicate"'); - expect(fixture.interceptMode).toBe(0o600); - expect(fixture.calls).toHaveLength(3); - expect(fixture.authorityNames).toEqual([[], [], []]); - expect(fixture.calls[0]).toEqual(fixture.passThroughArgv); - expect(fixture.calls[1]).toEqual(fixture.ttyPassThroughArgv); - expect(fixture.calls[2]?.slice(0, separator + 1)).toEqual( - fixture.exactArgv.slice(0, separator + 1), - ); - expect(fixture.calls[2]?.slice(separator + 1)).toEqual([ - "node", - "-e", - OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, - fixture.runId, - fixture.monitorRoot, - ...expectedRemote.slice(0, -1), - 'exec openclaw tui --message "$1"', - "nemoclaw-launch-first-turn", - "fixture input", - ]); - }, -); - -it.runIf(process.platform === "linux")( - "rejects a monitor starter whose standard input is not a PTY (#9160)", - () => { - const runId = randomUUID().replaceAll("-", ""); - const monitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - try { - const result = spawnSync( - process.execPath, - ["-e", OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, runId, monitorRoot, "/usr/bin/env", "true"], - { encoding: "utf8", timeout: 2_000, killSignal: "SIGKILL" }, - ); - - expect(result.status).toBe(72); - expect(result.stderr).toContain('"reason":"pty_stdin_not_pty"'); - expect(statSync(monitorRoot).mode & 0o777).toBe(0o700); - expect(existsSync(join(monitorRoot, "pty-input-mode.sock"))).toBe(false); - } finally { - rmSync(monitorRoot, { force: true, recursive: true }); - } - }, -); - -it.runIf(process.platform === "linux").each(["absent", "ansi", "reordered"] as const)( - "keeps the monitor alive through SIGTERM, records an auto-message and PTY turn, sends /exit, strips launch authority, and ignores terminal copy evidence [%s] (#9160, #9384)", - (terminalCopy) => { - const { - baselineRemoved, - hostSessionResidue, - openshellCalls, - orphanedMonitorProcessIds, - orphanedTuiProcessIds, - ptyMonitorRemoved, - ptySocketReceipt, - result, - tuiProcessIds, - ttyObserved, - } = runLaunchSessionFixture("valid", terminalCopy); - - expect(ttyObserved, result.stderr).toBe(true); - expect(baselineRemoved).toBe(true); - expect(ptyMonitorRemoved).toBe(true); - expect(hostSessionResidue).toEqual([]); - expect(openshellCalls.length).toBeGreaterThan(3); - expect(openshellCalls.every((call) => call.authorityNames.length === 0)).toBe(true); - expect(openshellCalls.some((call) => call.argv.includes("baseline"))).toBe(true); - expect( - openshellCalls.some((call) => call.argv.includes(OPENCLAW_PTY_MONITOR_STARTER_SCRIPT)), - ).toBe(true); - expect(tuiProcessIds).toHaveLength(1); - expect(orphanedMonitorProcessIds).toEqual([]); - expect(orphanedTuiProcessIds).toEqual([]); - expect(ptySocketReceipt).toEqual({ - rootMode: 0o700, - rootUid: process.getuid?.(), - socketIsSocket: true, - socketMode: 0o600, - socketNlink: 1, - socketUid: process.getuid?.(), - }); - expect(result.signal).toBeNull(); - expect(result.status).toBe(0); - }, -); - -it.runIf(process.platform === "linux")( - "waits for OpenClaw input mode and accepts a clean exit after two turns (#9160, #9384)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "delayed-input-attachment", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(0); - }, -); - -it.runIf(process.platform === "linux")( - "waits for OpenClaw startup to accept its auto-message before submitting one PTY input (#9384)", - () => { - const fixture = runLaunchSessionFixture("delayed-tui-ready", "absent"); - - expect(fixture.earlyInputObserved, fixture.result.stderr).toBe(false); - expect(fixture.baselineRemoved, fixture.result.stderr).toBe(true); - expect(fixture.result.status, fixture.result.stderr).toBe(0); - }, -); - -it.runIf(process.platform === "linux" && process.getuid?.() !== 0)( - "uses the inherited PTY descriptor when the sandbox user cannot reopen the device path (#9384)", - () => { - const { - baselineRemoved, - hostSessionResidue, - orphanedMonitorProcessIds, - orphanedTuiProcessIds, - ptyPathQueryResult, - ptyMonitorRemoved, - result, - ttyObserved, - } = runLaunchSessionFixture("pty-path-unreadable", "absent"); - - expect(ttyObserved, result.stderr).toBe(true); - expect(ptyPathQueryResult, result.stderr).toEqual({ errorCode: null, status: 1 }); - expect(baselineRemoved, result.stderr).toBe(true); - expect(ptyMonitorRemoved, result.stderr).toBe(true); - expect(hostSessionResidue, result.stderr).toEqual([]); - expect(orphanedMonitorProcessIds, result.stderr).toEqual([]); - expect(orphanedTuiProcessIds, result.stderr).toEqual([]); - expect(result.signal, result.stderr).toBeNull(); - expect(result.status, result.stderr).toBe(0); - }, -); - -it.runIf(process.platform === "linux").each([ - { - mode: "pty-socket-invalid", - reason: "pty_termios_response_invalid", - behavior: "malformed response from a replacement socket", - expectedDiagnostic: { reason: "pty_termios_response_invalid" }, - monitorRemoved: false, - }, - { - mode: "pty-socket-permission", - reason: "pty_socket_invalid", - behavior: "PTY monitor socket whose mode is not 0600", - expectedDiagnostic: { reason: "pty_socket_invalid" }, - monitorRemoved: false, - }, - { - mode: "pty-response-identity", - reason: "pty_identity_changed", - behavior: "response with PTY identity that does not match its device path", - expectedDiagnostic: { reason: "pty_identity_changed" }, - monitorRemoved: false, - }, - { - mode: "pty-termios-unavailable", - reason: "pty_termios_unavailable", - behavior: "unavailable PTY input-mode evidence", - expectedDiagnostic: { - reason: "pty_termios_unavailable", - sttyStatus: 1, - sttySignal: null, - sttyErrorCode: null, - sttyStderr: "fixture stty denied", - }, - monitorRemoved: true, - }, -] as const)( - "rejects $behavior before PTY input (#9160, #9384)", - ({ expectedDiagnostic, mode, monitorRemoved, reason }) => { - const { - baselineRemoved, - earlyInputObserved, - orphanedMonitorProcessIds, - orphanedTuiProcessIds, - ptyMonitorRemoved, - result, - ttyObserved, - } = runLaunchSessionFixture(mode, "absent"); - const failureEvidence = `${mode}: ${result.stderr}`; - const diagnostics = result.stderr.split("\n").flatMap((line) => { - try { - return [JSON.parse(line)]; - } catch { - return []; - } - }); - - expect(ttyObserved, failureEvidence).toBe(true); - expect(orphanedMonitorProcessIds, failureEvidence).toEqual([]); - expect(orphanedTuiProcessIds, failureEvidence).toEqual([]); - expect(baselineRemoved, failureEvidence).toBe(true); - expect(earlyInputObserved, failureEvidence).toBe(false); - expect(result.signal, failureEvidence).toBeNull(); - expect(result.status, failureEvidence).toBe(1); - expect(result.stderr, failureEvidence).toContain(`"reason":"${reason}"`); - expect(diagnostics, failureEvidence).toEqual( - expect.arrayContaining([expect.objectContaining(expectedDiagnostic)]), - ); - expect(ptyMonitorRemoved, failureEvidence).toBe(monitorRemoved); - }, - testTimeout(20_000), -); - -it.runIf(process.platform === "linux")( - "submits each PTY turn once while structured recording is delayed (#9160)", - () => { - const { baselineRemoved, pendingQualificationObserved, result, ttyObserved } = - runLaunchSessionFixture("delayed-recording", "absent"); - - expect(ttyObserved).toBe(true); - expect(pendingQualificationObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(0); - }, -); - -it.runIf(process.platform === "linux")( - "fails when the PTY remains in canonical input mode until the session deadline (#9160)", - () => { - const { baselineRemoved, orphanedMonitorProcessIds, ptyMonitorRemoved, result, ttyObserved } = - runLaunchSessionFixture("input-mode-timeout", "absent"); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(ptyMonitorRemoved).toBe(true); - expect(orphanedMonitorProcessIds).toEqual([]); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "launch did not observe noncanonical PTY input mode before the session deadline or before the PTY child process exited", - ); - expect(result.stderr).toContain('"reason":"pty_input_canonical"'); - }, - testTimeout(20_000), -); - -it.runIf(process.platform === "linux")( - "requires a current noncanonical observation after the PTY returns to canonical mode (#9384)", - () => { - const { - baselineRemoved, - canonicalRestored, - earlyInputObserved, - orphanedMonitorProcessIds, - ptyMonitorRemoved, - result, - ttyObserved, - } = runLaunchSessionFixture("restored-canonical-timeout", "absent"); - - expect(ttyObserved).toBe(true); - expect(canonicalRestored).toBe(true); - expect(earlyInputObserved).toBe(false); - expect(baselineRemoved).toBe(true); - expect(ptyMonitorRemoved).toBe(true); - expect(orphanedMonitorProcessIds).toEqual([]); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain('"reason":"pty_input_canonical"'); - }, - testTimeout(30_000), -); - -it.runIf(process.platform === "linux")( - "fails when the PTY monitor socket remains missing until the session deadline (#9160)", - () => { - const { baselineRemoved, ptyMonitorRemoved, result, ttyObserved } = runLaunchSessionFixture( - "pty-socket-timeout", - "absent", - ); - - expect(ttyObserved).toBe(false); - expect(baselineRemoved).toBe(true); - expect(ptyMonitorRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain('"reason":"pty_socket_missing"'); - }, - testTimeout(20_000), -); - -it.runIf(process.platform === "linux")( - "reports missing structured turns before the PTY child timeout (#9160)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "recording-timeout", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain("launch did not record the required structured session turns"); - }, -); - -it.runIf(process.platform === "linux")( - "rejects out-of-order structured records even when the PTY process remains active (#9160)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "invalid-order", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - }, -); - -it.runIf(process.platform === "linux")( - "rejects a late extra structured record before baseline cleanup (#9160)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "late-extra", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "launch final structured session evidence did not qualify (status 2)", - ); - expect(result.stderr).toContain('"reason":"extra_message"'); - }, -); - -it.runIf(process.platform === "linux")( - "propagates a nonzero TUI exit after two structured turns (#9160)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture("nonzero", "absent"); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(23); - }, -); - -it.runIf(process.platform === "linux")( - "fails when a qualified PTY session cannot run PTY monitor cleanup (#9160)", - () => { - const { hostSessionResidue, orphanedTuiProcessIds, ptyMonitorRemoved, result } = - runLaunchSessionFixture("pty-cleanup-failure", "absent"); - - expect(ptyMonitorRemoved).toBe(false); - expect(hostSessionResidue).toEqual([]); - expect(orphanedTuiProcessIds).toEqual([]); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain("launch could not remove the PTY monitor"); - }, -); - -it.runIf(process.platform === "linux")( - "refuses to remove an unknown entry from the mode-0700 PTY monitor directory (#9160)", - () => { - const { orphanedTuiProcessIds, ptyMonitorRemoved, result } = runLaunchSessionFixture( - "pty-cleanup-unknown-entry", - "absent", - ); - - expect(ptyMonitorRemoved).toBe(false); - expect(orphanedTuiProcessIds).toEqual([]); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain('"reason":"pty_monitor_cleanup_unknown_entry"'); - }, -); - -it.runIf(process.platform === "linux")( - "fails when a successful PTY session cannot remove its structured baseline (#9160)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "cleanup-failure", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(false); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain("structured session baseline cleanup failed"); - }, -); - -it.runIf(process.platform === "linux")( - "preserves a nonzero PTY exit when PTY monitor cleanup also fails (#9160)", - () => { - const { baselineRemoved, ptyMonitorRemoved, result, ttyObserved } = runLaunchSessionFixture( - "nonzero-pty-cleanup-failure", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(ptyMonitorRemoved).toBe(false); - expect(result.signal).toBeNull(); - expect(result.status).toBe(23); - }, -); - -it.runIf(process.platform === "linux")( - "rejects a relative OpenShell command before launching a host command (#9160)", - async () => { - let commandCallCount = 0; - const host = { - command: async () => { - commandCallCount += 1; - return { exitCode: 0, signal: null, stderr: "", stdout: "" }; - }, - openshellCommandPath: "openshell", - }; - - await expect( - runOpenClawLaunchSession({ - artifactName: "relative-openshell-command", - cliCommand: "node", - env: {}, - host: host as never, - redactionValues: [], - sandboxName: "alpha", - }), - ).rejects.toThrow("launch session coverage requires an absolute OpenShell command path"); - expect(commandCallCount).toBe(0); - }, -); - -it.each(["", "relative-tmp", "/tmp/absolute-tmp"])( - "passes an absolute host temporary root for empty, relative, or absolute TMPDIR input [%s] (#9160)", - async (root) => { - const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); - const roots: Array = []; - const host = { - command: async (_command: string, _args: string[], options?: { env?: NodeJS.ProcessEnv }) => { - roots.push(options?.env?.NEMOCLAW_LAUNCH_HOST_TMP_ROOT); - return { exitCode: 0, signal: null, stdout: "", stderr: "" }; - }, - openshellCommandPath: "/usr/bin/openshell", - }; - - try { - await runOpenClawLaunchSession({ - artifactName: "host-temporary-root", - cliCommand: "node", - env: { TMPDIR: root }, - host: host as never, - redactionValues: [], - sandboxName: "alpha", - }); - - expect(roots).toEqual([root === "" ? resolve("/tmp") : resolve(root)]); - } finally { - platform.mockRestore(); - } - }, -); - -it.runIf(process.platform === "linux")( - "runs the producer then two PTY launch sessions under one lease (#8942, #9023, #9160)", - async () => { - const calls: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = []; - let launchPhaseStartedAtCallCount = -1; - const host = { - command: async (command: string, args: string[], options?: { env?: NodeJS.ProcessEnv }) => { - calls.push({ command, args, env: options?.env }); - return { exitCode: 0, signal: null, stdout: "", stderr: "" }; - }, - openshellCommandPath: "/usr/bin/openshell", - }; - - await runOpenClawLaunchReadinessLeaseTurns({ - artifactName: "lease-turn", - cliCommand: "node", - cliEntrypoint: "/repo/bin/nemoclaw.js", - env: {}, - exitCommand: "/exit", - host: host as never, - redactionValues: [], - sandboxName: "alpha", - beforeLaunchTurns: () => { - launchPhaseStartedAtCallCount = calls.length; - }, - }); - - expect(launchPhaseStartedAtCallCount).toBe(1); - expect(calls).toHaveLength(3); - expect(calls[0]).toMatchObject({ - command: "node", - args: ["/repo/bin/nemoclaw.js", "alpha", "connect", "--probe-only"], - }); - expect(calls.slice(1).map((call) => call.command)).toEqual(["bash", "bash"]); - expect(calls.slice(1).map((call) => call.args)).toEqual([ - ["-lc", LAUNCH_TURN_SCRIPT], - ["-lc", LAUNCH_TURN_SCRIPT], - ]); - expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_LAUNCH_EXIT_COMMAND)).toEqual([ - "/exit", - "/exit", - ]); - expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_OPENSHELL_COMMAND)).toEqual([ - "/usr/bin/openshell", - "/usr/bin/openshell", - ]); - calls.slice(1).forEach((call) => { - expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_EXPECTED_REPLY"); - expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT"); - expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_PROMPT"); - expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_READY_TEXT"); - expect(typeof call.env?.NEMOCLAW_LAUNCH_FIRST_INPUT).toBe("string"); - expect(typeof call.env?.NEMOCLAW_LAUNCH_SECOND_INPUT).toBe("string"); - expect(call.env?.NEMOCLAW_LAUNCH_FIRST_INPUT).not.toBe( - call.env?.NEMOCLAW_LAUNCH_SECOND_INPUT, - ); - expect(call.env?.NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT).toBe( - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - ); - expect(call.env?.NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT).toBe( - OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, - ); - expect(call.env?.NEMOCLAW_LAUNCH_PTY_MONITOR_STARTER_SCRIPT).toBe( - OPENCLAW_PTY_MONITOR_STARTER_SCRIPT, - ); - expect(call.env?.NEMOCLAW_LAUNCH_RUNTIME_ENV_SCRIPT).toBe(OPENCLAW_LAUNCH_RUNTIME_ENV_SCRIPT); - }); - }, -); diff --git a/test/helpers/classify-ci-failure-suite.ts b/test/helpers/classify-ci-failure-suite.ts new file mode 100644 index 00000000000..1326b420662 --- /dev/null +++ b/test/helpers/classify-ci-failure-suite.ts @@ -0,0 +1,1164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import { spawn, spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { + execute, + type GhResolverFilesystem, + resolveProductionGhExecutableForTest, +} from "../../.agents/skills/nemoclaw-maintainer-classify-ci-failure/scripts/classify-ci-failure.mts"; +import { artifactZip, artifactZipEntryDataOffset } from "./artifact-zip"; +const script = resolve( + ".agents/skills/nemoclaw-maintainer-classify-ci-failure/scripts/classify-ci-failure.mts", +); +const roots: string[] = []; +const uid = process.getuid?.() ?? "unknown"; +const REDACTION_CASES = [ + [ + "Slack bot token", + ["xoxb", "123456789012", "123456789012", "abcdefghijklmnopqrstuvwx"].join("-"), + ["xoxb", "123456789012", "123456789012", "abcdefghijklmnopqrstuvwx"].join("-"), + ], + [ + "Slack app token", + ["xapp", "1", "A1234567890", "1234567890123", "abcdefghijklmnopqrstuvwx"].join("-"), + ["xapp", "1", "A1234567890", "1234567890123", "abcdefghijklmnopqrstuvwx"].join("-"), + ], + [ + "OpenAI API key", + "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", + "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", + ], + [ + "OpenAI project API key", + "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", + "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", + ], + [ + "NVIDIA API key", + "nvapi-abcdefghijklmnopqrstuvwxyz0123456789", + "nvapi-abcdefghijklmnopqrstuvwxyz0123456789", + ], + [ + "NVIDIA Cloud Functions key", + "nvcf-abcdefghijklmnopqrstuvwxyz0123456789", + "nvcf-abcdefghijklmnopqrstuvwxyz0123456789", + ], + [ + "npm access token", + "npm_abcdefghijklmnopqrstuvwxyz0123456789", + "npm_abcdefghijklmnopqrstuvwxyz0123456789", + ], + ["quoted JSON secret", '"client_secret": "json-secret-value"', "json-secret-value"], + ["quoted JSON token", '"refresh-token": "json-token-value"', "json-token-value"], + ["quoted JSON password", '"password": "json-password-value"', "json-password-value"], + ["quoted JSON API key", '"api-key": "json-api-key-value"', "json-api-key-value"], + [ + "quoted JSON authorization", + '"authorization": "Basic json-authorization-value"', + "json-authorization-value", + ], +] as const; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function fixture(log: string, result?: Record, archive?: Buffer) { + const root = mkdtempSync(join(tmpdir(), "classify-ci-test-")); + roots.push(root); + const bin = join(root, "bin"); + mkdirSync(bin, { recursive: true }); + const zip = join(root, "artifact.zip"); + writeFileSync( + zip, + archive ?? + artifactZip([{ name: "sample.result.json", contents: JSON.stringify(result ?? {}) }]), + ); + const gh = join(bin, "gh"); + const fake = [ + `#!${process.execPath}`, + "const fs=require('fs');", + "const {spawn}=require('node:child_process');", + "const block=()=>{ if(process.env.EXIT_PROMPTLY_ON_TERM) { fs.writeFileSync(process.env.BLOCK_WRAPPER_MARKER,String(process.ppid)); fs.writeFileSync(process.env.BLOCK_COMMAND_MARKER,String(process.pid)); process.once('SIGTERM',()=>{ fs.appendFileSync(process.env.SIGNAL_LOG,'SIGTERM\\n'); process.exit(0); }); } if(process.env.BLOCK_DESCENDANT_MARKER) { process.on('SIGINT',()=>{}); process.on('SIGTERM',()=>{}); } if(process.env.BLOCK_GROUP_MARKER) fs.writeFileSync(process.env.BLOCK_GROUP_MARKER,String(process.pid)); if(process.env.BLOCK_DESCENDANT_MARKER) { const child=spawn(process.execPath,['-e',\"process.on('SIGINT',()=>{}); process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)\"],{stdio:process.env.EXIT_GROUP_LEADER ? ['ignore',process.stdout,process.stderr] : 'ignore'}); fs.writeFileSync(process.env.BLOCK_DESCENDANT_MARKER,String(child.pid)); if(process.env.EXIT_GROUP_LEADER) { fs.writeFileSync(process.env.BLOCK_MARKER,'ready'); process.exit(0); } } fs.writeFileSync(process.env.BLOCK_MARKER,'ready'); setInterval(()=>{},1000); };", + "const a=process.argv.slice(2).join(' ');", + "fs.appendFileSync(process.env.GH_CALLS,a+'\\n');", + "fs.appendFileSync(process.env.GH_ENV,JSON.stringify({GH_HOST:process.env.GH_HOST,GH_ENTERPRISE_TOKEN:process.env.GH_ENTERPRISE_TOKEN,GH_TOKEN:process.env.GH_TOKEN,GITHUB_TOKEN:process.env.GITHUB_TOKEN})+'\\n');", + "if(a==='api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123') { if(process.env.BLOCK_METADATA) { block(); } else if(process.env.FAIL_METADATA) { console.error(process.env.FAIL_METADATA); process.exit(8); } else console.log(JSON.stringify({id:123,run_id:456,name:process.env.JOB_NAME||'CLI tests',status:'completed',conclusion:'failure',html_url:process.env.JOB_URL||'https://example.test/job'})); }", + "else if(a==='api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123/logs') { if(process.env.BLOCK_LOG) { block(); } else if(process.env.FAIL_LOG) { console.error(process.env.FAIL_LOG); process.exit(8); } else process.stdout.write('discarded\\n'.repeat(Number(process.env.LOG_PREFIX_LINES||0))+process.env.TEST_LOG); }", + "else if(a==='api --hostname github.com --include repos/NVIDIA/NemoClaw/actions/runs/456/artifacts?per_page=100&page=1') { if(process.env.FAIL_INVENTORY) { console.error(process.env.FAIL_INVENTORY); process.exit(8); } const artifacts=process.env.DUPLICATE_ARTIFACTS ? [{id:789,name:'results',size_in_bytes:Number(process.env.ZIP_SIZE)},{id:790,name:'results',size_in_bytes:Number(process.env.ZIP_SIZE)}] : [{id:789,name:'results',size_in_bytes:Number(process.env.ZIP_SIZE)}]; process.stdout.write('HTTP/2 200\\r\\n\\r\\n'+JSON.stringify({total_count:artifacts.length,artifacts})); }", + "else if(a==='api --hostname github.com repos/NVIDIA/NemoClaw/actions/artifacts/789/zip') { if(process.env.BLOCK_ARTIFACT) { block(); } else if(process.env.STREAM_BYTES) process.stdout.write(Buffer.alloc(Number(process.env.STREAM_BYTES))); else process.stdout.write(fs.readFileSync(process.env.ZIP_PATH)); }", + "else { console.error('unexpected '+a); process.exit(9); }", + ].join("\n"); + writeFileSync(gh, fake); + chmodSync(gh, 0o755); + const dd = join(bin, "dd"); + writeFileSync( + dd, + [ + `#!${process.execPath}`, + "const {spawnSync}=require('node:child_process');", + "if(process.env.FAIL_PROBE_DD && process.argv.includes('count=1')) process.exit(7);", + "const result=spawnSync('/usr/bin/dd',process.argv.slice(2),{stdio:'inherit'});", + "process.exit(result.status ?? 1);", + ].join("\n"), + ); + chmodSync(dd, 0o755); + const wc = join(bin, "wc"); + writeFileSync( + wc, + [ + `#!${process.execPath}`, + "const {spawnSync}=require('node:child_process');", + "if(process.env.FAIL_PROBE_WC && process.argv.includes('-c')) process.exit(7);", + "const result=spawnSync('/usr/bin/wc',process.argv.slice(2),{stdio:'inherit'});", + "process.exit(result.status ?? 1);", + ].join("\n"), + ); + chmodSync(wc, 0o755); + const env: NodeJS.ProcessEnv = { + ...process.env, + PATH: bin + ":" + process.env.PATH, + TMPDIR: root, + GH_CALLS: join(root, "calls"), + GH_ENV: join(root, "environment"), + TEST_DD: dd, + TEST_GH: gh, + TEST_WC: wc, + TEST_LOG: log, + LOG_PREFIX_LINES: "0", + ZIP_PATH: zip, + ZIP_SIZE: String(statSync(zip).size), + }; + return { root, env }; +} +const classifierArgs = (extra: string[] = []) => [ + "--experimental-strip-types", + "--no-warnings", + script, + "--job-id", + "123", + ...extra, +]; +function importedClassifierArgs(env: NodeJS.ProcessEnv, extra: string[]): string[] { + const optionValue = (name: string): string | undefined => { + const index = extra.indexOf(name); + return index < 0 ? undefined : extra[index + 1]; + }; + const repo = optionValue("--repo"); + const artifactName = optionValue("--artifact-name"); + const maxLines = optionValue("--max-lines"); + const clipMode = optionValue("--clip-mode"); + const input: Record = { + workdir: process.cwd(), + jobId: "123", + ...(repo === undefined ? {} : { repo }), + ...(artifactName === undefined ? {} : { artifactName }), + ...(maxLines === undefined ? {} : { maxLines: Number(maxLines) }), + ...(clipMode === undefined ? {} : { clipMode }), + }; + return [ + "--experimental-strip-types", + "--no-warnings", + "--input-type=module", + "-e", + [ + `import { classifyCiFailureWithRuntimeForTest } from ${JSON.stringify(new URL("file://" + script).href)};`, + `const input = ${JSON.stringify(input)};`, + "const environment = { ...process.env };", + "const executables = { bash: '/usr/bin/bash', dd: process.env.TEST_DD || '/usr/bin/dd', gh: process.env.TEST_GH, stat: '/usr/bin/stat', tail: '/usr/bin/tail', wc: process.env.TEST_WC || '/usr/bin/wc' };", + "const timeouts = { metadataMs: process.env.TEST_METADATA_TIMEOUT_MS ? Number(process.env.TEST_METADATA_TIMEOUT_MS) : undefined, logMs: process.env.TEST_LOG_TIMEOUT_MS ? Number(process.env.TEST_LOG_TIMEOUT_MS) : undefined, artifactMs: process.env.TEST_ARTIFACT_TIMEOUT_MS ? Number(process.env.TEST_ARTIFACT_TIMEOUT_MS) : undefined };", + "void classifyCiFailureWithRuntimeForTest(input, { executables, environment, timeouts }).then((value) => console.log(JSON.stringify(value, null, 2))).catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; });", + ].join("\n"), + ]; +} +function run(env: NodeJS.ProcessEnv, extra: string[] = []) { + const useCli = + extra.some((value) => value === "--unknown") || + extra.some((value) => value === "--repo") || + extra.some((value) => value === "0" || value === "1.5" || value === "501"); + return spawnSync( + process.execPath, + useCli ? classifierArgs(extra) : importedClassifierArgs(env, extra), + { + encoding: "utf8", + env, + }, + ); +} +function classifierTemporaryDirectories(prefix: "nemoclaw-ci-log." | "nemoclaw-ci-classify.") { + const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; + return existsSync(temporaryRoot) + ? readdirSync(temporaryRoot).filter((name) => name.startsWith(prefix)) + : []; +} +async function waitForFile(path: string): Promise { + await vi.waitFor(() => expect(existsSync(path)).toBe(true), { timeout: 2_000, interval: 10 }); +} +type FakePath = { + type: "directory" | "file" | "symlink"; + mode?: number; + uid?: number; + realpath?: string; +}; +function resolverFilesystem(entries: Record): GhResolverFilesystem { + const missing = (path: string): never => { + const error = new Error("missing " + path) as NodeJS.ErrnoException; + error.code = "ENOENT"; + throw error; + }; + const entry = (path: string): FakePath => entries[path] ?? missing(path); + return { + lstat: (path) => { + const value = entry(path); + return { + isDirectory: () => value.type === "directory", + isFile: () => value.type === "file", + isSymbolicLink: () => value.type === "symlink", + mode: value.mode ?? 0o755, + uid: value.uid ?? 0, + }; + }, + realpath: (path) => entry(path).realpath ?? path, + access: (path) => { + entry(path); + }, + }; +} +function trustedDirectories(home = "/home/tester"): Record { + return { + "/": { type: "directory" }, + "/usr": { type: "directory" }, + "/usr/bin": { type: "directory" }, + "/usr/local": { type: "directory" }, + "/usr/local/bin": { type: "directory" }, + "/home": { type: "directory" }, + [home]: { type: "directory", uid: 1000, mode: 0o750 }, + [join(home, ".local")]: { type: "directory", uid: 1000, mode: 0o700 }, + [join(home, ".local", "bin")]: { type: "directory", uid: 1000 }, + }; +} + +export type ClassifyCiFailureGroup = "artifacts" | "logs" | "process-lifecycle"; + +export function registerClassifyCiFailureTests(group: ClassifyCiFailureGroup): void { + if (group === "logs") { + describe("GitHub CLI production resolver", () => { + test("selects a safe user-local executable without consulting PATH", () => { + const home = "/home/tester"; + const gh = join(home, ".local", "bin", "gh"); + const filesystem = resolverFilesystem({ + ...trustedDirectories(home), + [gh]: { type: "file", uid: 1000 }, + "/attacker/gh": { type: "file", uid: 1000 }, + }); + expect( + resolveProductionGhExecutableForTest({ HOME: home, PATH: "/attacker" }, 1000, filesystem), + ).toBe(gh); + }); + + test("skips a writable system candidate and selects a safe user-local executable", () => { + const home = "/home/tester"; + const gh = join(home, ".local", "bin", "gh"); + const filesystem = resolverFilesystem({ + ...trustedDirectories(home), + "/usr/bin/gh": { type: "file", mode: 0o777 }, + [gh]: { type: "file", uid: 1000 }, + }); + expect(resolveProductionGhExecutableForTest({ HOME: home }, 1000, filesystem)).toBe(gh); + }); + + test.each<[string, Record]>([ + [ + "writable path component", + { "/home/tester/.local": { type: "directory", uid: 1000, mode: 0o770 } }, + ], + ["foreign owner", { "/home/tester/.local/bin/gh": { type: "file", uid: 2000 } }], + [ + "symlink candidate", + { + "/home/tester/.local/bin/gh": { + type: "symlink", + uid: 1000, + realpath: "/attacker/gh", + }, + "/attacker/gh": { type: "file", uid: 1000 }, + }, + ], + ])("rejects a user-local executable with a %s", (_case, overrides) => { + const home = "/home/tester"; + const gh = join(home, ".local", "bin", "gh"); + const filesystem = resolverFilesystem({ + ...trustedDirectories(home), + [gh]: { type: "file", uid: 1000 }, + ...overrides, + }); + expect(() => + resolveProductionGhExecutableForTest({ HOME: home, PATH: "/attacker" }, 1000, filesystem), + ).toThrow("Could not find a trusted GitHub CLI executable"); + }); + }); + } + + describe.skipIf(process.platform !== "linux")("CI failure classifier process", () => { + if (group === "logs") { + test("redacts credentials from classified diagnostic output", () => { + const secrets = [ + "Authorization: Bearer full authorization value with spaces", + "> X-API-Key: x-header-secret", + "request: Api-Key: api-header-secret", + "cOoKiE: log-cookie-secret=plain", + "> sEt-CoOkIe: log-set-cookie-secret=prefixed; Secure", + "< Set-Cookie: log-response-cookie-secret=prefixed; HttpOnly", + "< Cookie: log-response-request-cookie-secret=prefixed", + "< Authorization: Bearer log-response-auth-secret", + "< API-Key: log-response-api-key-secret", + "request: COOKIE: log-request-cookie-secret=prefixed", + "unrelated diagnostic text", + "https://user:password@example.test/path", + "AWS_ACCESS_KEY_ID=AKIAEXAMPLEVALUE", + "BUILD_TOKEN=token-value", + "CLIENT_SECRET: secret-value", + "DB_PASSWORD=password-value", + "SERVICE_API_KEY=api-key-value", + "ghp_alpha gho_beta ghu_gamma ghs_delta ghr_epsilon github_pat_zeta", + ].join("\n"); + const known = fixture(`${secrets}\nAssertionError: expected true`); + const r = run(known.env); + expect(r.status).toBe(0); + expect(r.stdout).not.toMatch( + /full authorization|x-header-secret|api-header-secret|log-cookie-secret|log-set-cookie-secret|log-response-cookie-secret|log-response-request-cookie-secret|log-response-auth-secret|log-response-api-key-secret|log-request-cookie-secret|user:password|AKIAEXAMPLE|token-value|secret-value|password-value|api-key-value|ghp_alpha|gho_beta|ghu_gamma|ghs_delta|ghr_epsilon|github_pat_zeta/, + ); + expect(r.stdout).toContain("> X-API-Key: [REDACTED]"); + expect(r.stdout).toContain("request: Api-Key: [REDACTED]"); + expect(r.stdout).toContain("cOoKiE: [REDACTED]"); + expect(r.stdout).toContain("> sEt-CoOkIe: [REDACTED]"); + expect(r.stdout).toContain("< Set-Cookie: [REDACTED]"); + expect(r.stdout).toContain("< Cookie: [REDACTED]"); + expect(r.stdout).toContain("< Authorization: [REDACTED]"); + expect(r.stdout).toContain("< API-Key: [REDACTED]"); + expect(r.stdout).toContain("request: COOKIE: [REDACTED]"); + expect(r.stdout).toContain("unrelated diagnostic text"); + expect(r.stdout.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(13); + }); + test("does not pass executable lookup or startup injection to classifier subprocesses", () => { + const item = fixture("AssertionError: expected true"); + const attackerBin = join(item.root, "attacker-bin"); + const executableMarker = join(item.root, "attacker-executable-ran"); + const bashMarker = join(item.root, "bash-env-ran"); + const nodeMarker = join(item.root, "node-options-ran"); + mkdirSync(attackerBin); + const bashExecutable = join(attackerBin, "bash"); + const ghExecutable = join(attackerBin, "gh"); + writeFileSync( + bashExecutable, + `#!${process.execPath}\nrequire("node:fs").appendFileSync(${JSON.stringify(executableMarker)}, "bash\n");`, + ); + writeFileSync( + ghExecutable, + `#!${process.execPath}\nrequire("node:fs").appendFileSync(${JSON.stringify(executableMarker)}, "gh\n");`, + ); + chmodSync(bashExecutable, 0o755); + chmodSync(ghExecutable, 0o755); + const bashHook = join(item.root, "bash-env"); + writeFileSync(bashHook, `printf injected >> ${JSON.stringify(bashMarker)}`); + const nodeHook = join(item.root, "node-options.mjs"); + writeFileSync( + nodeHook, + `import { appendFileSync } from "node:fs"; appendFileSync(${JSON.stringify(nodeMarker)}, ${JSON.stringify("loaded\n")});`, + ); + item.env.PATH = attackerBin; + item.env.BASH_ENV = bashHook; + item.env.ENV = bashHook; + item.env.NODE_OPTIONS = `--import=${nodeHook}`; + const result = run(item.env); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout).result).toBe("classified"); + expect(existsSync(executableMarker)).toBe(false); + expect(existsSync(bashMarker)).toBe(false); + expect(readFileSync(nodeMarker, "utf8").trim().split("\n")).toEqual(["loaded"]); + }); + + test("binds GitHub API reads to github.com without forwarding enterprise host credentials", () => { + const item = fixture("AssertionError: expected true"); + item.env.GH_HOST = "attacker.invalid"; + item.env.GH_ENTERPRISE_TOKEN = "enterprise-secret"; + item.env.GH_TOKEN = "github-token"; + item.env.GITHUB_TOKEN = "github-actions-token"; + + const result = run(item.env, ["--artifact-name", "results"]); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(item.env.GH_CALLS!, "utf8").trim().split("\n")).toEqual([ + "api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123", + "api --hostname github.com repos/NVIDIA/NemoClaw/actions/jobs/123/logs", + "api --hostname github.com --include repos/NVIDIA/NemoClaw/actions/runs/456/artifacts?per_page=100&page=1", + "api --hostname github.com repos/NVIDIA/NemoClaw/actions/artifacts/789/zip", + ]); + const environments = readFileSync(item.env.GH_ENV!, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(environments).toHaveLength(4); + expect(environments).toEqual( + environments.map(() => ({ + GH_TOKEN: "github-token", + GITHUB_TOKEN: "github-actions-token", + })), + ); + }); + + test("classifies an AssertionError as a test failure", () => { + const item = fixture("AssertionError: expected true"); + const result = run(item.env); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.result).toBe("classified"); + expect(value.categories).toContain("test-failure"); + }); + test("returns unclassified when output has no failure signature", () => { + const item = fixture("ordinary unrelated output"); + const result = run(item.env); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout).result).toBe("unclassified"); + }); + test.each(REDACTION_CASES)( + "redacts a standalone %s from returned process logs", + (_name, secret, exposed) => { + const item = fixture( + ["diagnostic before", secret, "AssertionError: expected true", "diagnostic after"].join( + "\n", + ), + ); + const result = run(item.env); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("[REDACTED]"); + expect(result.stdout).not.toContain(exposed ?? secret); + }, + ); + test("redacts signed URL credentials in selected logs without changing other query fields", () => { + const signedUrl = + "https://storage.test/object?keep=visible&X-Amz-Credential=aws-credential&X-Amz-Signature=aws-signature&X-Amz-Security-Token=aws-session&X-Goog-Credential=google-credential&X-Goog-Signature=google-signature&sig=azure-signature&access_token=oauth-token&token=common-token&tail=retained#fragment"; + const item = fixture(`${signedUrl}\nAssertionError: expected true`); + const result = run(item.env); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.log.stdout).toContain( + "?keep=visible&X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&X-Amz-Security-Token=[REDACTED]&X-Goog-Credential=[REDACTED]&X-Goog-Signature=[REDACTED]&sig=[REDACTED]&access_token=[REDACTED]&token=[REDACTED]&tail=retained#fragment", + ); + expect(result.stdout).not.toMatch( + /aws-credential|aws-signature|aws-session|google-credential|google-signature|azure-signature|oauth-token|common-token/, + ); + }); + test("redacts artifact errors and omits ignored command data", () => { + const item = fixture("ordinary output", { + exitCode: 1, + error: + "> X-API-Key: artifact-error-key\n< Set-Cookie: artifact-response-cookie-secret=opaque; HttpOnly\nrequest: Set-Cookie: artifact-error-cookie=opaque; HttpOnly\nunrelated artifact error\ndownload failed https://storage.test/object?X-Amz-Credential=error-credential&X-Amz-Signature=error-signature&X-Amz-Security-Token=error-session&keep=error-visible", + command: + "request: Api-Key: artifact-command-key\n> COOKIE: artifact-command-cookie=opaque\nunrelated artifact command\ncurl 'https://storage.test/object?X-Goog-Credential=command-credential&X-Goog-Signature=command-google-signature&sig=command-signature&access_token=command-access&token=command-token&keep=command-visible'", + }); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const failure = JSON.parse(result.stdout).artifact.failures[0]; + expect(failure.error).toContain("> X-API-Key: [REDACTED]"); + expect(failure.error).toContain("< Set-Cookie: [REDACTED]"); + expect(failure.error).toContain("request: Set-Cookie: [REDACTED]"); + expect(failure.error).toContain("unrelated artifact error"); + expect(failure).not.toHaveProperty("command"); + expect(failure.error).toContain( + "?X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&X-Amz-Security-Token=[REDACTED]&keep=error-visible", + ); + expect(result.stdout).not.toMatch( + /artifact-error-key|artifact-response-cookie-secret|artifact-error-cookie|artifact-command-key|artifact-command-cookie|error-credential|error-signature|error-session|command-credential|command-google-signature|command-signature|command-access|command-token/, + ); + }); + test("redacts and bounds dynamic GitHub job metadata", () => { + const item = fixture("ordinary output"); + const nameSecret = "metadata-name-secret"; + const urlSecret = "metadata-url-secret"; + item.env.JOB_NAME = `${"n".repeat(600)} BUILD_TOKEN=${nameSecret}`; + item.env.JOB_URL = `https://example.test/job/${"u".repeat(2100)}?keep=visible&token=${urlSecret}`; + const result = run(item.env); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.job.name).toContain("[REDACTED]"); + expect(value.job.name.length).toBeLessThanOrEqual(500); + expect(value.job.url).toContain("token=[REDACTED]"); + expect(value.job.url.length).toBeLessThanOrEqual(2000); + expect(result.stdout).not.toContain(nameSecret); + expect(result.stdout).not.toContain(urlSecret); + }); + test("rejects an alternate valid repository before invoking GitHub", () => { + const item = fixture("unused"); + const r = spawnSync( + process.execPath, + importedClassifierArgs(item.env, ["--repo", "NVIDIA/Other"]), + { + encoding: "utf8", + env: item.env, + }, + ); + expect(r.status).toBe(1); + expect(r.stderr).toContain("repo must be NVIDIA/NemoClaw"); + expect(readdirSync(item.root)).not.toContain("calls"); + }); + test("bounds streamed logs before filtering", () => { + const item = fixture("AssertionError: retained tail"); + item.env.LOG_PREFIX_LINES = "500000"; + const r = run(item.env); + expect(r.status, r.stderr).toBe(0); + const value = JSON.parse(r.stdout); + expect(value.log.truncated).toBe(true); + expect(value.log.truncationReasons).toContain("source-log-bounded-before-filtering"); + expect(value.log.stdout).toContain("retained tail"); + expect(value.log.stdout.length).toBeLessThan(40000); + const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; + expect( + existsSync(temporaryRoot) + ? readdirSync(temporaryRoot).filter((name) => name.startsWith("nemoclaw-ci-log.")) + : [], + ).toEqual([]); + }); + test.each([ + [["--unknown", "value"], "Unknown option --unknown"], + [["--repo", "NVIDIA/NemoClaw"], "Unknown option --repo"], + [["--max-lines", "0"], "--max-lines must be an integer from 1 through 500"], + [["--max-lines", "1.5"], "--max-lines must be an integer from 1 through 500"], + [["--max-lines", "501"], "--max-lines must be an integer from 1 through 500"], + ])("rejects invalid flags before invoking GitHub", (extra, message) => { + const item = fixture("unused"); + const r = run(item.env, extra); + expect(r.status).toBe(1); + expect(r.stderr).toContain(message); + expect(readdirSync(item.root)).not.toContain("calls"); + }); + test.each([ + ["authentication", "authentication required BUILD_TOKEN=log-access-secret"], + ["authorization", "HTTP 403 resource not accessible BUILD_TOKEN=log-access-secret"], + ])("fails log acquisition with bounded, redacted %s recovery guidance", (_kind, failure) => { + const item = fixture("unused"); + item.env.FAIL_LOG = failure; + const result = run(item.env); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("GitHub access failed. Run gh auth status"); + expect(result.stderr).toContain( + "ask the user to correct authentication, authorization, SSO", + ); + expect(result.stderr).toContain("BUILD_TOKEN=[REDACTED]"); + expect(result.stderr).not.toContain("log-access-secret"); + expect(result.stderr.length).toBeLessThanOrEqual(2001); + expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); + }); + + test.each([ + [ + "job metadata", + "FAIL_METADATA", + [], + "authentication required BUILD_TOKEN=metadata-access-secret", + ], + [ + "artifact inventory", + "FAIL_INVENTORY", + ["--artifact-name", "results"], + "HTTP 403 resource not accessible BUILD_TOKEN=inventory-access-secret", + ], + ] as const)( + "fails %s access with bounded, redacted recovery guidance", + (_kind, variable, extra, failure) => { + const item = fixture("ordinary output"); + item.env[variable] = failure; + const result = run(item.env, [...extra]); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("GitHub access failed. Run gh auth status"); + expect(result.stderr).toContain( + "ask the user to correct authentication, authorization, SSO, or token scope", + ); + expect(result.stderr).toContain("BUILD_TOKEN=[REDACTED]"); + expect(result.stderr).not.toMatch(/metadata-access-secret|inventory-access-secret/u); + expect(result.stderr.length).toBeLessThanOrEqual(2001); + }, + ); + + test.each([ + [ + "job metadata", + "BLOCK_METADATA", + "TEST_METADATA_TIMEOUT_MS", + [], + "GitHub job metadata read timed out after 30 seconds", + ], + [ + "job log", + "BLOCK_LOG", + "TEST_LOG_TIMEOUT_MS", + [], + "GitHub job log read timed out after 60 seconds", + ], + [ + "artifact", + "BLOCK_ARTIFACT", + "TEST_ARTIFACT_TIMEOUT_MS", + ["--artifact-name", "results"], + "GitHub artifact read timed out after 60 seconds", + ], + ] as const)( + "reports a bounded silent %s timeout and removes temporary data", + (_operation, block, timeoutVariable, extra, diagnostic) => { + const item = fixture("ordinary output"); + item.env[block] = "1"; + item.env.BLOCK_MARKER = join(item.root, "blocked"); + item.env[timeoutVariable] = "50"; + const result = run(item.env, [...extra]); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain(diagnostic); + expect(result.stderr).toContain( + "Check GitHub availability, then retry CI failure classification.", + ); + expect(result.stderr).not.toContain("repos/NVIDIA/NemoClaw/actions"); + expect(result.stderr).not.toContain("--hostname"); + expect(result.stderr.length).toBeLessThanOrEqual(2001); + expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }, + ); + + test("preserves the primary log failure and directly removes its temporary directory", () => { + const item = fixture("AssertionError: retained tail"); + const credential = "cleanup-path-secret"; + const tempRoot = join(item.root, `BUILD_TOKEN=${credential}`); + mkdirSync(tempRoot, { recursive: true }); + item.env.TMPDIR = tempRoot; + item.env.FAIL_LOG = "primary log download failure"; + const result = run(item.env); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("primary log download failure"); + expect(result.stderr).not.toContain(credential); + expect(result.stderr).not.toContain("cleanup-secret"); + expect(result.stderr.length).toBeLessThanOrEqual(2001); + expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); + }); + } + + if (group === "artifacts") { + test.each([ + ["malformed", Buffer.from("not a zip")], + [ + "symlink", + (() => { + const archive = artifactZip([{ name: "sample.result.json", contents: "{}" }]); + archive.writeUInt32LE(0xa0000000, archive.readUInt32LE(archive.length - 6) + 38); + return archive; + })(), + ], + ["traversal", artifactZip([{ name: "../sample.result.json", contents: "{}" }])], + ["option-like", artifactZip([{ name: "-sample.result.json", contents: "{}" }])], + [ + "duplicate", + artifactZip([ + { name: "sample.result.json", contents: "{}" }, + { name: "sample.result.json", contents: "{}" }, + ]), + ], + ])("rejects a %s artifact ZIP", (_name, archive) => { + const item = fixture("SIGKILL", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); + }); + test("allows an artifact stream at the exact compressed-size limit", () => { + const item = fixture("SIGKILL"); + item.env.STREAM_BYTES = "25000000"; + item.env.ZIP_SIZE = "25000000"; + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); + expect(result.stderr).not.toContain("compressed stream exceeds"); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }); + + test("rejects one byte beyond the compressed-size limit", () => { + const item = fixture("SIGKILL"); + item.env.STREAM_BYTES = "25000001"; + item.env.ZIP_SIZE = "25000000"; + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Selected artifact compressed stream exceeds the 25,000,000-byte limit", + ); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }); + + test("rejects the artifact stream when the extra-byte dd probe fails", () => { + const item = fixture("SIGKILL", { exitCode: 1 }); + item.env.FAIL_PROBE_DD = "1"; + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("Could not download selected artifact"); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }); + + test("rejects the artifact stream when the extra-byte wc probe fails", () => { + const item = fixture("SIGKILL", { exitCode: 1 }); + item.env.FAIL_PROBE_WC = "1"; + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("Could not download selected artifact"); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }); + + test("rejects artifact metadata that differs from downloaded bytes", () => { + const item = fixture("SIGKILL"); + item.env.ZIP_SIZE = String(Number(item.env.ZIP_SIZE) + 1); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Could not download selected artifact"); + }); + test("rejects a corrupted unrelated entry beside a valid result", () => { + const archive = artifactZip( + [ + { name: "sample.result.json", contents: JSON.stringify({ exitCode: 1 }) }, + { name: "diagnostics/unrelated.txt", contents: "unrelated" }, + ], + 8, + ); + archive[artifactZipEntryDataOffset(archive, 1)] ^= 0xff; + const item = fixture("SIGKILL", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); + }); + + test("rejects an artifact whose declared expanded size exceeds the aggregate bound", () => { + const archive = artifactZip([{ name: "sample.result.json", contents: "{}" }]); + const centralOffset = archive.readUInt32LE(archive.length - 6); + archive.writeUInt32LE(100_000_001, centralOffset + 24); + const item = fixture("SIGKILL", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); + }); + test("rejects a result entry over the per-file expanded limit", () => { + const archive = artifactZip( + [{ name: "large.result.json", contents: "x".repeat(1_000_001) }], + 8, + ); + const item = fixture("SIGKILL", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("exceeds the 1,000,000-byte limit"); + }); + test("preserves an artifact validation failure and directly removes its temporary directory", () => { + const item = fixture("SIGKILL", undefined, Buffer.from("not a zip")); + const credential = "artifact-cleanup-path-secret"; + const tempRoot = join(item.root, `BUILD_TOKEN=${credential}`); + mkdirSync(tempRoot, { recursive: true }); + item.env.TMPDIR = tempRoot; + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Artifact ZIP is malformed or unsafe"); + expect(result.stderr).not.toContain("cleanup-secret"); + expect(result.stderr).not.toContain(credential); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }); + test("reads a real ZIP artifact and removes private temporary directories", () => { + const item = fixture("SIGKILL", { + exitCode: 137, + signal: "SIGKILL", + }); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.artifact.artifactId).toBe(789); + expect(value.artifact.filesRead).toBe(1); + expect(value.artifact.failures[0].signal).toBe("SIGKILL"); + expect(result.stdout).not.toContain("artifact-secret"); + const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; + expect( + existsSync(temporaryRoot) + ? readdirSync(temporaryRoot).filter((name) => name.startsWith("nemoclaw-ci-")) + : [], + ).toEqual([]); + }); + test.each([ + { + name: "nonzero exit code", + artifactResult: { exitCode: 23 }, + category: "process-exit-code", + nextAction: "Inspect the captured command and its producing step", + }, + { + name: "timeout", + artifactResult: { exitCode: null, timedOut: true }, + category: "process-timeout", + nextAction: "Inspect the captured command and surrounding resource evidence", + }, + { + name: "reported error", + artifactResult: { exitCode: null, error: "runner could not start" }, + category: "artifact-reported-error", + nextAction: "Inspect the reported command error and its producing step", + }, + ])( + "classifies an artifact-reported $name without a known log signature", + ({ artifactResult, category, nextAction }) => { + const item = fixture("ordinary output", artifactResult); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.result).toBe("classified"); + expect(value.categories).toContain(category); + expect(value.nextActions).toContainEqual(expect.stringContaining(nextAction)); + }, + ); + + test.each([ + { + name: "signal", + artifactResult: { exitCode: 137, signal: "SIGKILL" }, + category: "process-signal", + diagnostic: "ended with SIGKILL", + }, + { + name: "timeout", + artifactResult: { timedOut: true }, + category: "process-timeout", + diagnostic: "exceeded its time limit", + }, + { + name: "reported error", + artifactResult: { error: "runner could not start" }, + category: "artifact-reported-error", + diagnostic: "reported: runner could not start", + }, + ])( + "selects a later $name beyond capped failure evidence", + ({ artifactResult, category, diagnostic }) => { + const secret = "winner-path-secret"; + const earlier = Array.from({ length: 20 }, (_, index) => ({ + name: `earlier-${index}.result.json`, + contents: JSON.stringify({ exitCode: 23 }), + })); + const winnerPath = `BUILD_TOKEN=${secret} ${"x".repeat(1100)}.result.json`; + const archive = artifactZip([ + ...earlier, + { name: winnerPath, contents: JSON.stringify(artifactResult) }, + ]); + const item = fixture("ordinary output", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + const finding = value.findings.find((entry: { type: string }) => entry.type === category); + expect(value.categories).toEqual([category]); + expect(value.artifact.failures).toHaveLength(20); + expect(value.artifact.failuresTruncated).toBe(true); + expect(value.artifact.failures).not.toContainEqual( + expect.objectContaining({ path: expect.stringContaining("[REDACTED]") }), + ); + expect(finding.detail).toContain("BUILD_TOKEN=[REDACTED]"); + expect(finding.detail).toContain(diagnostic); + expect(finding.detail.length).toBeLessThanOrEqual(1100); + expect(result.stdout).not.toContain(secret); + }, + ); + + test("reports bounded malformed result evidence while retaining valid artifact failures", () => { + const pathSecret = "malformed-path-secret"; + const malformed = Array.from({ length: 25 }, (_, index) => ({ + name: `results/TOKEN=${pathSecret}-${index}.result.json`, + contents: '{"exitCode":', + })); + const archive = artifactZip([ + { name: `${"a".repeat(1_100)}.result.json`, contents: "null" }, + ...malformed, + { name: "valid.result.json", contents: JSON.stringify({ exitCode: 1 }) }, + ]); + const item = fixture("ordinary output", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.artifact.filesRead).toBe(27); + expect(value.artifact.malformedResultCount).toBe(26); + expect(value.artifact.malformedResultPaths).toHaveLength(20); + expect( + value.artifact.malformedResultPaths.every((path: string) => path.length <= 1_000), + ).toBe(true); + expect(value.artifact.malformedResultPathsTruncated).toBe(true); + expect( + value.artifact.malformedResultPaths.some((path: string) => path.includes("[REDACTED]")), + ).toBe(true); + expect(value.artifact.failures).toHaveLength(1); + expect(value.artifact.failures[0].exitCode).toBe(1); + expect(result.stdout).not.toContain(pathSecret); + }); + + test("reports malformed evidence when every selected artifact result is malformed", () => { + const archive = artifactZip([ + { name: "truncated.result.json", contents: '{"exitCode":' }, + { name: "non-object.result.json", contents: JSON.stringify("invalid result") }, + ]); + const item = fixture("ordinary output", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.result).toBe("unclassified"); + expect(value.artifact).toMatchObject({ + filesRead: 2, + malformedResultCount: 2, + malformedResultPaths: ["truncated.result.json", "non-object.result.json"], + malformedResultPathsTruncated: false, + failures: [], + }); + }); + + test("ignores non-failure and command-only records and retains a true timeout", () => { + const archive = artifactZip([ + { + name: "string-false.result.json", + contents: JSON.stringify({ exitCode: 0, timedOut: "false" }), + }, + { + name: "command-only.result.json", + contents: JSON.stringify({ command: "npm test" }), + }, + { + name: "boolean-true.result.json", + contents: JSON.stringify({ exitCode: 0, timedOut: true }), + }, + ]); + const item = fixture("ordinary output", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const failures = JSON.parse(result.stdout).artifact.failures; + expect(failures).toHaveLength(1); + expect(failures[0]).toMatchObject({ path: "boolean-true.result.json", timedOut: true }); + }); + + test("rejects ambiguous same-name artifacts before downloading a ZIP", () => { + const item = fixture("SIGKILL"); + item.env.DUPLICATE_ARTIFACTS = "1"; + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Artifact results is ambiguous for run 456"); + expect(result.stderr).toContain("789, 790"); + const calls = readFileSync(item.env.GH_CALLS!, "utf8"); + expect(calls).not.toContain("actions/artifacts/789/zip"); + expect(calls).not.toContain("actions/artifacts/790/zip"); + }); + + test("does not return or use a credential-shaped artifact signal", () => { + const secret = "artifact-signal-secret"; + const item = fixture("ordinary output", { + exitCode: 1, + signal: `BUILD_TOKEN=${secret}`, + }); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.artifact.failures[0].signal).toBeNull(); + expect(value.findings).not.toContainEqual( + expect.objectContaining({ type: "process-signal" }), + ); + expect(result.stdout).not.toContain(secret); + }); + } + + if (group === "process-lifecycle") { + test("retains a leader for an ignored-stdio process-group member until timeout", async () => { + const root = mkdtempSync(join(tmpdir(), "classify-ci-timeout-")); + roots.push(root); + const marker = join(root, "descendant-pid"); + const program = [ + "const {spawn}=require('node:child_process');", + "const child=spawn(process.execPath,['-e',\"process.title='member ) name'; process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)\"],{stdio:'ignore'});", + "require('node:fs').writeFileSync(process.argv[1],String(child.pid));", + ].join(""); + const started = Date.now(); + const pending = execute(process.execPath, ["-e", program, marker], root, 200); + await waitForFile(marker); + const descendantPid = Number(readFileSync(marker, "utf8")); + expect(() => process.kill(descendantPid, 0)).not.toThrow(); + const result = await pending; + expect(Date.now() - started).toBeGreaterThanOrEqual(200); + expect(result.exitCode).not.toBe(0); + expect(result.timedOut).toBe(true); + expect(() => process.kill(descendantPid, 0)).toThrow(); + }); + + test("drains a process group whose command exits promptly on SIGTERM", async () => { + const item = fixture("AssertionError: retained tail"); + const marker = join(item.root, "blocked"); + const signals = join(item.root, "process-group-signals"); + const wrapperMarker = join(item.root, "wrapper-pid"); + const commandMarker = join(item.root, "command-pid"); + item.env.BLOCK_LOG = "1"; + item.env.BLOCK_MARKER = marker; + item.env.EXIT_PROMPTLY_ON_TERM = "1"; + item.env.SIGNAL_LOG = signals; + item.env.BLOCK_WRAPPER_MARKER = wrapperMarker; + item.env.BLOCK_COMMAND_MARKER = commandMarker; + const child = spawn(process.execPath, importedClassifierArgs(item.env, []), { + env: item.env, + stdio: ["ignore", "pipe", "pipe"], + }); + await Promise.all([ + waitForFile(marker), + waitForFile(wrapperMarker), + waitForFile(commandMarker), + ]); + const wrapperPid = Number(readFileSync(wrapperMarker, "utf8")); + const commandPid = Number(readFileSync(commandMarker, "utf8")); + child.kill("SIGTERM"); + const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve) => + child.once("close", (code, closeSignal) => resolve({ code, signal: closeSignal })), + ); + expect(result).toEqual({ code: 143, signal: null }); + expect(readFileSync(signals, "utf8").trim().split("\n")).toEqual(["SIGTERM"]); + expect(() => process.kill(wrapperPid, 0)).toThrow(); + expect(() => process.kill(commandPid, 0)).toThrow(); + expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); + expect(classifierTemporaryDirectories("nemoclaw-ci-classify.")).toEqual([]); + }); + + test("reports a redacted fixed-root removal command when cancellation cleanup fails", async () => { + const item = fixture("AssertionError: retained tail"); + const marker = join(item.root, "blocked"); + const instrument = join(item.root, "fail-cancellation-cleanup.mjs"); + writeFileSync( + instrument, + [ + 'import fs from "node:fs";', + 'import { syncBuiltinESMExports } from "node:module";', + 'import path from "node:path";', + "const realRmSync = fs.rmSync.bind(fs);", + "fs.rmSync = (target, options) =>", + " path.basename(String(target)).startsWith('nemoclaw-ci-log.')", + " ? (() => { throw new Error('BUILD_TOKEN=cancellation-cleanup-secret'); })()", + " : realRmSync(target, options);", + "syncBuiltinESMExports();", + ].join("\n"), + ); + item.env.BLOCK_LOG = "1"; + item.env.BLOCK_MARKER = marker; + item.env.NODE_OPTIONS = `--import=${instrument}`; + const child = spawn(process.execPath, importedClassifierArgs(item.env, []), { + env: item.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + await waitForFile(marker); + child.kill("SIGTERM"); + const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve) => + child.once("close", (code, closeSignal) => resolve({ code, signal: closeSignal })), + ); + expect(result).toEqual({ code: 143, signal: null }); + const generatedName = stderr.match( + /Cancellation cleanup failure for (nemoclaw-ci-log\.[A-Za-z0-9]{6})/, + )?.[1]; + expect(generatedName).toBeDefined(); + expect(stderr).toContain( + `Remove it directly with: rm -rf -- /tmp/nemoclaw-ci-classifier-${uid}/${generatedName}`, + ); + expect(stderr).toContain("BUILD_TOKEN=[REDACTED]"); + expect(stderr).not.toContain("cancellation-cleanup-secret"); + expect(stderr.length).toBeLessThanOrEqual(2001); + rmSync(`/tmp/nemoclaw-ci-classifier-${uid}/${generatedName}`, { + recursive: true, + force: true, + }); + }); + + test.each([ + ["metadata", "BLOCK_METADATA", [], "SIGTERM", 143, false], + ["log", "BLOCK_LOG", [], "SIGINT", 130, true], + ["log", "BLOCK_LOG", [], "SIGHUP", 129, true], + ["artifact", "BLOCK_ARTIFACT", ["--artifact-name", "results"], "SIGTERM", 143, true], + ] as const)( + "kills the detached group and its ignoring descendant during %s cancellation", + async (_kind, block, extra, signal, exitCode, exitGroupLeader) => { + const item = fixture("AssertionError: retained tail"); + const marker = join(item.root, "blocked"); + const groupMarker = join(item.root, "group-pid"); + const descendantMarker = join(item.root, "descendant-pid"); + item.env[block] = "1"; + item.env.BLOCK_MARKER = marker; + item.env.BLOCK_GROUP_MARKER = groupMarker; + item.env.BLOCK_DESCENDANT_MARKER = descendantMarker; + Object.assign(item.env, exitGroupLeader ? { EXIT_GROUP_LEADER: "1" } : {}); + const child = spawn(process.execPath, importedClassifierArgs(item.env, [...extra]), { + env: item.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + await Promise.all([ + waitForFile(marker), + waitForFile(groupMarker), + waitForFile(descendantMarker), + ]); + const groupPid = Number(readFileSync(groupMarker, "utf8")); + const descendantPid = Number(readFileSync(descendantMarker, "utf8")); + child.kill(signal); + child.kill(signal); + const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve) => + child.once("close", (code, closeSignal) => resolve({ code, signal: closeSignal })), + ); + expect(result).toEqual({ code: exitCode, signal: null }); + expect(() => process.kill(groupPid, 0)).toThrow(); + expect(() => process.kill(descendantPid, 0)).toThrow(); + expect(stderr.length).toBeLessThanOrEqual(2000); + const temporaryRoot = `/tmp/nemoclaw-ci-classifier-${uid}`; + const remaining = existsSync(temporaryRoot) + ? readdirSync(temporaryRoot).filter((name) => name.startsWith("nemoclaw-ci-")) + : []; + expect(remaining).toEqual([]); + }, + ); + + test("redacts credential assignments in returned artifact paths", () => { + const secret = "artifact-path-secret"; + const archive = artifactZip([ + { + name: `BUILD_TOKEN=${secret}.result.json`, + contents: JSON.stringify({ exitCode: 1 }), + }, + ]); + const item = fixture("SIGKILL", undefined, archive); + const result = run(item.env, ["--artifact-name", "results"]); + expect(result.status, result.stderr).toBe(0); + const value = JSON.parse(result.stdout); + expect(value.artifact.failures[0].path).toContain("[REDACTED]"); + expect(result.stdout).not.toContain(secret); + }); + } + }); +} diff --git a/test/helpers/onboard-fresh-create-identity-suite.ts b/test/helpers/onboard-fresh-create-identity-suite.ts index fc5bf1efbe7..46d05dd7a47 100644 --- a/test/helpers/onboard-fresh-create-identity-suite.ts +++ b/test/helpers/onboard-fresh-create-identity-suite.ts @@ -78,7 +78,7 @@ const freshCreateIdentityCases = [ expectedOutcome: "identity-mismatch-refusal" as const, }, { - group: "recovery" as const, + group: "recovery-state" as const, title: "retains recovery state when the create runner fails after verification (#9833)", apfInterceptorRequested: true, provider: null, @@ -87,7 +87,7 @@ const freshCreateIdentityCases = [ expectedOutcome: "post-create-runner-refusal" as const, }, { - group: "recovery" as const, + group: "recovery-state" as const, title: "retains recovery state when registry publication fails after create (#9833)", apfInterceptorRequested: true, provider: null, @@ -96,7 +96,7 @@ const freshCreateIdentityCases = [ expectedOutcome: "post-create-registration-refusal" as const, }, { - group: "recovery" as const, + group: "recovery-state" as const, title: "blocks every reentry when registry-failure recovery has no durable journal (#9833)", apfInterceptorRequested: true, provider: null, @@ -105,7 +105,7 @@ const freshCreateIdentityCases = [ expectedOutcome: "post-create-registration-recovery-readback-failure" as const, }, { - group: "recovery" as const, + group: "recovery-policy" as const, title: "retries registry-failure recovery from the process-exit owner (#9833)", apfInterceptorRequested: true, provider: null, @@ -114,7 +114,7 @@ const freshCreateIdentityCases = [ expectedOutcome: "post-create-registration-recovery-retry" as const, }, { - group: "recovery" as const, + group: "recovery-policy" as const, title: "accepts an external policy change after registration (#9833)", apfInterceptorRequested: true, provider: null, @@ -167,7 +167,7 @@ export function registerFreshCreateIdentityTests(group: FreshCreateIdentityGroup const caseTimeoutMs = group === "cancellation" ? childProcessTimeoutMs * 6 + 15_000 - : group === "recovery" + : group.startsWith("recovery-") ? childProcessTimeoutMs * 3 + 15_000 : 45_000; describe(`fresh create identity (${group})`, () => { diff --git a/test/helpers/onboard-messaging-suite.ts b/test/helpers/onboard-messaging-suite.ts new file mode 100644 index 00000000000..728091247e2 --- /dev/null +++ b/test/helpers/onboard-messaging-suite.ts @@ -0,0 +1,2020 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { beforeEach, describe, it, vi } from "vitest"; +import YAML from "yaml"; + +import { + activeChannelsFromDockerfile, + encodeMessagingPlanForChannels, + messagingPlanLiteral, + parseMessagingFixturePayload, + writeCustomMessagingDockerfile, +} from "./messaging-plan-fixtures"; +import { runBoundedOnboardScript } from "./onboard-child-process-harness"; +import { writeOkOpenshell } from "./onboard-openshell-fixture"; + +type CommandEntry = { + command: string; + env?: Record; + policyContent?: string; + policyReadError?: string; + dockerfileContent?: string; + dockerfileReadError?: string; + providerRevisions?: Record | null; + rawCredentialInEnv?: boolean; +}; +const parseStdoutJson = parseMessagingFixturePayload; +const repoRoot = path.join(import.meta.dirname, "../.."); +const yamlModulePath = fileURLToPath(import.meta.resolve("yaml")); +const onboardScriptMocksPath = JSON.stringify( + path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), +); +beforeEach(() => { + vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); + vi.stubEnv("NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE", "1"); + vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); +}); +export type OnboardMessagingGroup = "configuration" | "lifecycle" | "provider"; + +export function registerOnboardMessagingTests(group: OnboardMessagingGroup): void { + describe(`onboard messaging (${group})`, () => { + if (group === "provider") { + it( + "creates providers for messaging tokens and attaches them to the sandbox", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-providers-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-provider-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); +const commands = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ sandboxName: "my-assistant" }); createdSandbox.installRuntimeObservation(); +runner.run = fixtureMocks.createStatefulMessagingProviderRunner({ commands, createdSandbox }); +runner.runCapture = (command) => { + const sandboxCapture = createdSandbox.capture(command); + if (sandboxCapture !== null) return sandboxCapture; + if (_n(command).includes("provider get")) return "Provider: discord-bridge"; + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { + defaultCurlOutput: "ok", + }); + if (mockedCapture !== null) return mockedCapture; + } + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = () => true; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); + const entry = { command, env: args[2]?.env || null }; + const policyMatch = command.match(/--policy ([^ ]+)/); + if (policyMatch) { + try { + entry.policyContent = fs.readFileSync(policyMatch[1], "utf-8"); + } catch (error) { + entry.policyReadError = String(error); + } + } + commands.push(entry); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; +const { createSandbox, setupMessagingChannels } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY = "1"; + process.env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; + process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; + process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; + process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token-value"; + process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; + process.env.KUBECONFIG = "/tmp/host-kubeconfig"; + process.env.SSH_AUTH_SOCK = "/tmp/host-ssh-agent.sock"; + await setupMessagingChannels(null, null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], createFixture)); + console.log(JSON.stringify({ + sandboxName, + commands, + messagingPlanEnv: process.env.NEMOCLAW_MESSAGING_PLAN_B64, + })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + const result = runBoundedOnboardScript(scriptPath, { + cwd: repoRoot, + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + const providerCommands = payload.commands.filter((e: CommandEntry) => + e.command.includes("provider create"), + ); + const discordProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-discord-bridge"), + ); + assert.ok( + discordProvider, + "expected my-assistant-discord-bridge provider create command", + ); + assert.match(discordProvider.command, /--credential DISCORD_BOT_TOKEN/); + + const slackProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-slack-bridge"), + ); + assert.ok(slackProvider, "expected my-assistant-slack-bridge provider create command"); + assert.match(slackProvider.command, /--credential SLACK_BOT_TOKEN/); + + const telegramProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-telegram-bridge"), + ); + assert.ok( + telegramProvider, + "expected my-assistant-telegram-bridge provider create command", + ); + assert.match(telegramProvider.command, /--credential TELEGRAM_BOT_TOKEN/); + + const createCommand = payload.commands.find((e: CommandEntry) => + e.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.match(createCommand.command, /--provider my-assistant-discord-bridge/); + assert.match(createCommand.command, /--provider my-assistant-slack-bridge/); + assert.match(createCommand.command, /--provider my-assistant-telegram-bridge/); + assert.match(createCommand.command, /--policy [^ ]*nemoclaw-initial-policy[^ ]*\.yaml/); + assert.equal(createCommand.policyReadError, undefined); + const policyDoc = YAML.parse(createCommand.policyContent || "") || {}; + const slackEndpointHosts = (policyDoc.network_policies?.slack?.endpoints || []).map( + (entry: { host?: string }) => entry.host, + ); + const slackWebsocketHosts = slackEndpointHosts + .filter( + (host: string | undefined) => + host === "wss-primary.slack.com" || host === "wss-backup.slack.com", + ) + .sort(); + assert.deepEqual( + slackWebsocketHosts, + ["wss-backup.slack.com", "wss-primary.slack.com"].sort(), + ); + + assert.doesNotMatch(createCommand.command, /test-discord-token-value/); + assert.doesNotMatch(createCommand.command, /123456:ABC-test-telegram-token/); + assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /TELEGRAM_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /xoxb-test-slack-token-value/); + assert.doesNotMatch(createCommand.command, /xapp-test-slack-app-token-value/); + assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /SLACK_APP_TOKEN=/); + assert.doesNotMatch(createCommand.command, /NEMOCLAW_MESSAGING_PLAN_B64=/); + + assert.ok( + payload.messagingPlanEnv, + "expected serialized messaging plan in host process env", + ); + const messagingPlan = JSON.parse( + Buffer.from(payload.messagingPlanEnv, "base64").toString("utf8"), + ); + assert.equal(messagingPlan.sandboxName, "my-assistant"); + assert.deepEqual( + messagingPlan.channels + .map((channel: { channelId: string }) => channel.channelId) + .sort(), + ["discord", "slack", "telegram"].sort(), + ); + assert.doesNotMatch(JSON.stringify(messagingPlan), /test-discord-token-value/); + assert.doesNotMatch(JSON.stringify(messagingPlan), /123456:ABC-test-telegram-token/); + + assert.ok(createCommand.env, "expected env to be captured from spawn call"); + assert.equal( + createCommand.env.DISCORD_BOT_TOKEN, + undefined, + "DISCORD_BOT_TOKEN must not be in sandbox env", + ); + assert.equal( + createCommand.env.SLACK_BOT_TOKEN, + undefined, + "SLACK_BOT_TOKEN must not be in sandbox env", + ); + assert.equal( + createCommand.env.SLACK_APP_TOKEN, + undefined, + "SLACK_APP_TOKEN must not be in sandbox env", + ); + assert.equal( + createCommand.env.TELEGRAM_BOT_TOKEN, + undefined, + "TELEGRAM_BOT_TOKEN must not be in sandbox env", + ); + assert.equal( + createCommand.env.NVIDIA_INFERENCE_API_KEY, + undefined, + "NVIDIA_INFERENCE_API_KEY must not be in sandbox env", + ); + assert.equal( + createCommand.env.KUBECONFIG, + undefined, + "KUBECONFIG must not be in sandbox env", + ); + assert.equal( + createCommand.env.SSH_AUTH_SOCK, + undefined, + "SSH_AUTH_SOCK must not be in sandbox env", + ); + + const envString = JSON.stringify(createCommand.env); + assert.ok( + !envString.includes("test-discord-token-value"), + "Discord token value must not leak into sandbox env", + ); + assert.ok( + !envString.includes("xoxb-test-slack-token-value"), + "Slack bot token value must not leak into sandbox spawn env", + ); + assert.ok( + !envString.includes("xapp-test-slack-app-token-value"), + "Slack app token value must not leak into sandbox spawn env", + ); + assert.ok( + !envString.includes("123456:ABC-test-telegram-token"), + "Telegram token value must not leak into sandbox env", + ); + }, + ); + } + + if (group === "lifecycle") { + it( + "preserves Hermes Slack policy when Slack is active at sandbox create time", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-hermes-slack-")); + try { + const fakeBin = path.join(tmpDir, "bin"); + const customBuildDir = path.join(tmpDir, "custom-build"); + const customDockerfilePath = path.join(customBuildDir, "Dockerfile"); + const scriptPath = path.join(tmpDir, "hermes-slack-policy.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const agentDefsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "agent", "defs.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const yamlPath = JSON.stringify(yamlModulePath); + const customDockerfileArg = JSON.stringify(customDockerfilePath); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.mkdirSync(customBuildDir, { recursive: true }); + fs.writeFileSync( + customDockerfilePath, + "FROM scratch\nARG NEMOCLAW_MESSAGING_PLAN_B64=\nARG NEMOCLAW_TOOL_DISCLOSURE=progressive\nENV NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE}\n", + ); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); +const YAML = require(${yamlPath}); +const { loadAgent } = require(${agentDefsPath}); +require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "docker-driver-platform.ts"))}).isLinuxDockerDriverGatewayEnabled = () => false; +const nonSlackMessagingEnvKeys = [ + "DISCORD_BOT_TOKEN", + "DISCORD_SERVER_ID", + "DISCORD_SERVER_IDS", + "DISCORD_ALLOWED_IDS", + "DISCORD_USER_ID", + "DISCORD_REQUIRE_MENTION", + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_ALLOWED_IDS", + "TELEGRAM_REQUIRE_MENTION", +]; + +const commands = []; +let registeredSandbox = null; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ sandboxName: "my-assistant" }); createdSandbox.installRuntimeObservation(); +runner.run = fixtureMocks.createStatefulMessagingProviderRunner({ commands, createdSandbox }); +runner.runCapture = (command) => { + const sandboxCapture = createdSandbox.capture(command); + if (sandboxCapture !== null) return sandboxCapture; + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { + defaultCurlOutput: "ok", + }); + if (mockedCapture !== null) return mockedCapture; + } + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { + registeredSandbox = entry; + return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); + const entry = { command, env: args[2]?.env || null }; + const policyMatch = command.match(/--policy ([^ ]+)/); + if (policyMatch) { + entry.policyPath = policyMatch[1]; + try { + entry.policyContent = fs.readFileSync(policyMatch[1], "utf-8"); + } catch (error) { + entry.policyReadError = String(error); + } + } + commands.push(entry); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + for (const key of nonSlackMessagingEnvKeys) delete process.env[key]; + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.NEMOCLAW_AGENT = "hermes"; + process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; + process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token-value"; + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${customDockerfileArg}, loadAgent("hermes"), null, null, null, []], createFixture)); + const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); + const parsed = YAML.parse(createCommand?.policyContent || "") || {}; + const slack = parsed.network_policies?.slack || {}; + console.log(JSON.stringify({ + sandboxName, + createCommand: { + command: createCommand?.command || "", + policyPath: createCommand?.policyPath || "", + policyReadError: createCommand?.policyReadError || null, + }, + registeredPolicyStatePresent: "policies" in (registeredSandbox || {}), + slackBinaryPaths: (slack.binaries || []).map((entry) => entry.path), + slackEndpointHosts: (slack.endpoints || []).map((entry) => entry.host), + })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + assert.ok(payload.createCommand.command.includes("sandbox create")); + assert.match(payload.createCommand.command, /--provider my-assistant-slack-bridge/); + assert.match(payload.createCommand.command, /--provider my-assistant-slack-app/); + assert.match(payload.createCommand.policyPath, /nemoclaw-initial-policy/); + assert.equal(payload.createCommand.policyReadError, null); + assert.equal(payload.registeredPolicyStatePresent, false); + assert.deepEqual(payload.slackBinaryPaths, [ + "/usr/local/bin/hermes", + "/usr/bin/python3*", + "/usr/bin/python3.13", + "/opt/hermes/.venv/bin/python3", + "/opt/hermes/.venv/bin/python", + ]); + assert.ok( + !payload.slackBinaryPaths.includes("/usr/local/bin/node"), + "Hermes Slack policy must not be replaced by the generic Node Slack preset", + ); + const slackWebsocketHosts = payload.slackEndpointHosts + .filter( + (host: string) => + host === "wss-primary.slack.com" || host === "wss-backup.slack.com", + ) + .sort(); + assert.deepEqual( + slackWebsocketHosts, + ["wss-backup.slack.com", "wss-primary.slack.com"].sort(), + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, + ); + + it( + "publishes attached OpenShell provider state before a messaging recreate starts (#9770)", + { timeout: 60_000 }, + async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-messaging-recreate-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-reuse-provider.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify(path.join(repoRoot, "src/lib/state/registry.ts")); + const preflightPath = JSON.stringify(path.join(repoRoot, "src/lib/onboard/preflight.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src/lib/credentials/store.ts"), + ); + const telegramCredentialKeys = [ + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_BOT_TOKEN_AGENT_A", + "TELEGRAM_BOT_TOKEN_AGENT_B", + ]; + const providerCredentialKeys = { + "compatible-endpoint": ["COMPATIBLE_API_KEY"], + "my-assistant-slack-app": ["SLACK_APP_TOKEN"], + "my-assistant-slack-bridge": ["SLACK_BOT_TOKEN"], + "my-assistant-telegram-bridge": telegramCredentialKeys, + }; + const expectedProviders = Object.keys(providerCredentialKeys).sort(); + const rawGatewayCredential = "gateway-only-provider-secret"; + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + const script = String.raw` +const runner = require(${runnerPath}), registry = require(${registryPath}), preflight = require(${preflightPath}), credentials = require(${credentialsPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const childProcess = require("node:child_process"), { EventEmitter } = require("node:events"); +const commands = [], credentialKeys = ${JSON.stringify(providerCredentialKeys)}; let registered = null; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ sandboxName: "my-assistant" }); createdSandbox.installRuntimeObservation(); +const providers = Object.keys(credentialKeys), revisions = new Map(providers.map((name) => [name, 1])), providerGetCounts = new Map(); +const rawGatewayCredential = ${JSON.stringify(rawGatewayCredential)}, gatewaySecrets = new Map(providers.map((name) => [name, rawGatewayCredential])); +registry.registerSandbox({ name: "my-assistant", messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["slack", "telegram", "whatsapp"])} } }); +runner.run = (command, opts = {}) => { + const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); + const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); if (profileResult !== null) return profileResult; + const providerGet = normalized.match(/provider get -g nemoclaw ([^ ]+)$/)?.[1]; if (providerGet) providerGetCounts.set(providerGet, (providerGetCounts.get(providerGet) || 0) + 1); if (providerGet === process.env.NEMOCLAW_TEST_FAIL_PROVIDER && providerGetCounts.get(providerGet) >= 4) return { status: 2, stderr: "transport unavailable" }; + if (providerGet && revisions.has(providerGet)) return { status: 0, stdout: "Name: " + providerGet + "\nType: " + (providerGet === "compatible-endpoint" ? "openai" : "nemoclaw-mcp-v1") + "\nCredential keys: " + credentialKeys[providerGet] + "\nConfig keys: " + (providerGet === "compatible-endpoint" ? "OPENAI_BASE_URL" : "") + "\n" }; + const refresh = normalized.match(/provider update -g nemoclaw ([^ ]+)$/)?.[1]; + if (refresh && gatewaySecrets.has(refresh)) { if (refresh === process.env.NEMOCLAW_TEST_FAIL_PROVIDER) return { status: 1 }; revisions.set(refresh, revisions.get(refresh) + 1); return { status: 0 }; } + if (normalized.includes("provider get")) return { status: 1 }; + return createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { + const sandboxCapture = createdSandbox.capture(command); + if (sandboxCapture !== null) return sandboxCapture; + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { registered = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "compatible-endpoint", + model: "custom/model", + getSandbox: registry.getSandbox, + registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.unref = () => {}; child.pid = 4242; + const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); const attachedProviders = [...command.matchAll(/--provider ([^ ]+)/g)].map((match) => match[1]); + commands.push({ command, providerRevisions: command.includes("sandbox create") ? Object.fromEntries(attachedProviders.map((name) => [name, revisions.get(name)])) : null, rawCredentialInEnv: Object.values(args[2]?.env || {}).includes(rawGatewayCredential) }); + process.nextTick(() => { child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); child.emit("close", 0); }); + return child; +}; +const { createSandbox } = require(${onboardPath}); +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_EXTRA_PLACEHOLDER_KEYS = "TELEGRAM_BOT_TOKEN_AGENT_A,TELEGRAM_BOT_TOKEN_AGENT_B,GITHUB_TOKEN"; + process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["slack", "telegram", "whatsapp"])})).toString("base64"); + Object.values(credentialKeys).forEach((key) => delete process.env[key]); delete process.env.GITHUB_TOKEN; + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], null, null, null, null, null, []], createFixture)); + console.log(JSON.stringify({ sandboxName, commands, registered })); +})().catch((error) => { const temporaryCreateSources = require("node:fs").readdirSync(process.env.TMPDIR).filter((entry) => entry.startsWith("nemoclaw-initial-policy-") || entry.startsWith("nemoclaw-build-")); console.log(JSON.stringify({ commands, registered, error: String(error), providerRevisions: Object.fromEntries(revisions), temporaryCreateSources })); console.error(error); process.exit(1); }); +`; + fs.writeFileSync(scriptPath, script); + const runScenario = (failedProvider?: string) => + spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + TMPDIR: tmpDir, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_FAIL_PROVIDER: failedProvider || "", + ...Object.fromEntries( + [...Object.values(providerCredentialKeys).flat(), "GITHUB_TOKEN"].map((key) => [ + key, + "", + ]), + ), + }, + }); + const result = runScenario(); + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + const commands = payload.commands as CommandEntry[]; + const createIndex = commands.findIndex(({ command }) => + command.includes("sandbox create"), + ); + assert.notEqual(createIndex, -1, "expected sandbox create command"); + const createCommand = commands[createIndex]; + const providerRefreshes = commands + .map((entry, index) => ({ entry, index })) + .filter(({ entry }) => /\bprovider update -g nemoclaw ([^ ]+)$/.test(entry.command)); + const providerName = (command: string) => + command.match(/\bprovider update -g nemoclaw ([^ ]+)$/)?.[1]; + const refreshedProviders = providerRefreshes + .map(({ entry }: { entry: CommandEntry }) => providerName(entry.command)) + .sort(); + const denied = runScenario("my-assistant-telegram-bridge"); + assert.equal(denied.status, 1); + const deniedPayload = parseStdoutJson(denied.stdout); + const deniedCommands = (deniedPayload.commands as CommandEntry[]).map( + ({ command }) => command, + ); + const deniedRefreshes = deniedCommands.map(providerName).filter(Boolean); + const publishedBeforeCreate = providerRefreshes.every( + ({ entry, index }) => index < createIndex && !entry.command.includes("--credential"), + ); + const extraPlaceholderKeys = createCommand.command + .match(/NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=([^ ]+)/)?.[1] + ?.split(",") + .sort(); + const registeredChannels = payload.registered?.messaging?.plan?.channels.map( + (channel: { channelId: string }) => channel.channelId, + ); + assert.deepEqual(refreshedProviders, expectedProviders); + assert.equal( + commands.some(({ command }) => command.includes("provider create")), + false, + ); + assert.equal(publishedBeforeCreate, true); + assert.deepEqual( + [...createCommand.command.matchAll(/--provider ([^ ]+)/g)] + .map((match) => match[1]) + .sort(), + expectedProviders, + ); + assert.deepEqual( + createCommand.providerRevisions, + Object.fromEntries(expectedProviders.map((provider) => [provider, 2])), + ); + assert.deepEqual(extraPlaceholderKeys, [ + "TELEGRAM_BOT_TOKEN_AGENT_A", + "TELEGRAM_BOT_TOKEN_AGENT_B", + ]); + assert.equal(createCommand.command.includes("GITHUB_TOKEN"), false); + assert.equal(createCommand.rawCredentialInEnv, false); + assert.deepEqual(registeredChannels, ["slack", "telegram", "whatsapp"]); + assert.deepEqual(deniedRefreshes, []); + assert.equal( + Object.values(deniedPayload.providerRevisions).filter((revision) => revision === 2) + .length, + 0, + ); + assert.ok(deniedCommands.every((command) => !command.includes("sandbox create"))); + assert.equal(deniedPayload.registered, null); + assert.deepEqual(deniedPayload.temporaryCreateSources, []); + assert.match( + deniedPayload.error, + /Could not inspect messaging provider 'my-assistant-telegram-bridge': OpenShell could not inspect the provider/, + ); + const combinedOutput = result.stdout + result.stderr + denied.stdout + denied.stderr; + assert.equal( + (JSON.stringify([payload, deniedPayload]) + combinedOutput).includes( + rawGatewayCredential, + ), + false, + ); + }, + ); + + it( + "preserves disabled channels in a custom image after a recreate so `channels start` can re-enable them (#3381)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-disabled-channels-preserve-"), + ); + const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "disabled-channels-preserve.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const messagingPlanB64 = encodeMessagingPlanForChannels(["telegram"], ["telegram"]); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); +const commands = []; let dockerfileContent; +const registerCalls = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ + sandboxName: "my-assistant", +}); createdSandbox.installRuntimeObservation(); +registry.registerSandbox({ + name: "my-assistant", + messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["telegram"], ["telegram"])} }, +}); +runner.run = (command, opts = {}) => { + const normalized = _n(command); + commands.push({ command: normalized, env: opts.env || null }); + if (normalized.includes("provider get -g nemoclaw my-assistant-telegram-bridge")) return { status: 0, stdout: "Name: my-assistant-telegram-bridge\nType: nemoclaw-mcp-v1\nCredential keys: TELEGRAM_BOT_TOKEN\nConfig keys: \n" }; + const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; + return createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { + const sandboxCapture = createdSandbox.capture(command); + if (sandboxCapture !== null) return sandboxCapture; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { + registerCalls.push(entry); + return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); + const entry = { command, env: args[2]?.env || null }; + const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); + if (dockerfileMatch) { + try { + entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); + } catch (error) { + entry.dockerfileReadError = String(error); + } + } + commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + delete process.env.TELEGRAM_BOT_TOKEN; + process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["telegram"], ["telegram"])})).toString("base64"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); + console.log(JSON.stringify({ sandboxName, commands, registerCalls })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + const result = runBoundedOnboardScript(scriptPath, { + cwd: repoRoot, + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_GPU: "0", + NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, + TELEGRAM_BOT_TOKEN: "", + }, + }); + assert.equal(result.status, 0, result.stderr || result.error?.message); + const payload = parseStdoutJson(result.stdout); + + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.equal(createCommand.dockerfileReadError, undefined); + + assert.deepEqual( + activeChannelsFromDockerfile(createCommand.dockerfileContent), + [], + "disabled channel must not be active in the image plan", + ); + assert.doesNotMatch( + createCommand.command, + /--provider my-assistant-telegram-bridge/, + "disabled channel's bridge must not be attached to the new sandbox", + ); + + const registeredPlan = payload.registerCalls[0]?.messaging?.plan; + assert.deepEqual( + registeredPlan?.channels.map((channel: { channelId: string }) => channel.channelId), + ["telegram"], + "registry.messaging.plan must keep the disabled-but-configured channel so `channels start` can recover it", + ); + assert.deepEqual( + registeredPlan?.disabledChannels, + ["telegram"], + "registry.messaging.plan.disabledChannels must round-trip through the rebuild", + ); + }, + ); + } + + if (group === "configuration") { + it( + "bakes WhatsApp into a custom sandbox image without bridge providers when no messaging tokens are set", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-tokenless-whatsapp-"), + ); + try { + const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "tokenless-whatsapp.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"]); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); + +const commands = []; let dockerfileContent; +const registerCalls = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = (command, opts = {}) => { + const normalized = _n(command); + commands.push({ command: normalized, env: opts.env || null }); + const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; + return createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { + const createdIdentity = createdSandbox.capture(command); + if (createdIdentity !== null) return createdIdentity; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { + registerCalls.push(entry); + return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); + const entry = { command, env: args[2]?.env || null }; + const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); + if (dockerfileMatch) { + try { + entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); + } catch (error) { + entry.dockerfileReadError = String(error); + } + } + commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + for (const key of Object.keys(process.env)) { + if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { + delete process.env[key]; + } + } + process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); + console.log(JSON.stringify({ sandboxName, commands, registerCalls })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + const providerMutationCommands = payload.commands.filter( + (entry: CommandEntry) => + /\bprovider (create|update)\b/.test(entry.command) && + entry.command.includes("-bridge"), + ); + assert.equal( + providerMutationCommands.length, + 0, + "QR-only channel selection must not create bridge providers", + ); + + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.equal(createCommand.dockerfileReadError, undefined); + assert.doesNotMatch(createCommand.command, /--provider \S+-bridge\b/); + + assert.deepEqual(activeChannelsFromDockerfile(createCommand.dockerfileContent), [ + "whatsapp", + ]); + assert.deepEqual( + payload.registerCalls[0]?.messaging?.plan?.channels.map( + (channel: { channelId: string }) => channel.channelId, + ), + ["whatsapp"], + ); + assert.equal(payload.registerCalls[0]?.messagingChannels, undefined); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, + ); + + it( + "drops WhatsApp from a rebuilt custom image when the registry marks it disabled", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-disabled-whatsapp-"), + ); + try { + const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "disabled-whatsapp.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"], ["whatsapp"]); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); + +registry.registerSandbox({ + name: "my-assistant", + messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["whatsapp"], ["whatsapp"])} }, +}); + +const commands = []; let dockerfileContent; +const registerCalls = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = (command, opts = {}) => { + const normalized = _n(command); + commands.push({ command: normalized, env: opts.env || null }); + const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; + return createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { + const createdIdentity = createdSandbox.capture(command); + if (createdIdentity !== null) return createdIdentity; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { + registerCalls.push(entry); + return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); + const entry = { command, env: args[2]?.env || null }; + const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); + if (dockerfileMatch) { + try { + entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); + } catch (error) { + entry.dockerfileReadError = String(error); + } + } + commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + for (const key of Object.keys(process.env)) { + if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { + delete process.env[key]; + } + } + process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); + console.log(JSON.stringify({ sandboxName, commands, registerCalls })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.equal(createCommand.dockerfileReadError, undefined); + + assert.deepEqual( + activeChannelsFromDockerfile(createCommand.dockerfileContent), + [], + "disabled QR channel must not be active in the image plan", + ); + const registeredPlan = payload.registerCalls[0]?.messaging?.plan; + assert.deepEqual( + registeredPlan?.channels.map((channel: { channelId: string }) => channel.channelId), + ["whatsapp"], + "registry.messaging.plan must keep the disabled QR channel so `channels start` can recover it (mirrors #3381)", + ); + assert.deepEqual( + registeredPlan?.disabledChannels, + ["whatsapp"], + "registry.messaging.plan.disabledChannels must round-trip through the rebuild", + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, + ); + } + + if (group === "provider") { + it("aborts onboard when a messaging provider upsert fails", { timeout: 60_000 }, async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-provider-fail-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "provider-upsert-fail.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); + +runner.run = (command, opts = {}) => { + // Fail all provider create and update calls + if (_n(command).includes("provider")) { + return { status: 1, stdout: "", stderr: "gateway unreachable" }; + } + return { status: 0 }; +}; +runner.runCapture = (command) => { + if (_n(command).includes("sandbox get")) return ""; + if (_n(command).includes("sandbox list")) return ""; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = () => true; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; + await createSandbox(null, "gpt-5.4", "nvidia-prod"); + console.log("ERROR_DID_NOT_EXIT"); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.notEqual(result.status, 0, "expected non-zero exit when provider upsert fails"); + assert.ok( + !result.stdout.includes("ERROR_DID_NOT_EXIT"), + "onboard should have aborted before reaching sandbox create", + ); + }); + + it( + "reuses sandbox without refreshing unselected ambient messaging providers (#10277)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-reuse-providers-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "reuse-with-providers.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); + +const commands = []; +const existingSandbox = fixtureMocks.createCreatedSandboxFixture({ lifecycleState: "created" }); existingSandbox.installRuntimeObservation(); +const messagingProviderRunner = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ + commands, + createdSandbox: existingSandbox, + initialProviders: [ + ["my-assistant-discord-bridge", "nemoclaw-mcp-v1", "DISCORD_BOT_TOKEN"], + ["my-assistant-slack-bridge", "nemoclaw-mcp-v1", "SLACK_BOT_TOKEN"], + ["my-assistant-slack-app", "nemoclaw-mcp-v1", "SLACK_APP_TOKEN"], + ], +}); +runner.run = messagingProviderRunner; +runner.runCapture = (command) => { + const sandboxCapture = existingSandbox.capture(command); + if (sandboxCapture !== null) return sandboxCapture; + // All messaging providers already exist in gateway + if (_n(command).includes("provider get")) return "Provider: exists"; + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; +registry.getSandbox = () => fixtureMocks.sandboxLifecycleFixture( + { name: "my-assistant", toolDisclosure: "progressive" }, + { sandboxId: existingSandbox.state.sandboxId }, +); +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.DISCORD_BOT_TOKEN = "test-discord-token"; + process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; + process.env.SLACK_APP_TOKEN = "xapp-test-slack-token"; + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + console.log(JSON.stringify({ sandboxName, commands })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + assert.equal(payload.sandboxName, "my-assistant", "should reuse existing sandbox"); + assert.ok( + payload.commands.every( + (entry: CommandEntry) => !entry.command.includes("sandbox create"), + ), + "should NOT recreate sandbox when providers already exist in gateway", + ); + assert.ok( + payload.commands.every( + (entry: CommandEntry) => !entry.command.includes("sandbox delete"), + ), + "should NOT delete sandbox when providers already exist in gateway", + ); + + // Existing gateway providers do not select messaging for this onboarding request. + const providerUpserts = payload.commands.filter((entry: CommandEntry) => + entry.command.includes("provider update"), + ); + assert.equal( + providerUpserts.length, + 0, + "should not refresh ambient messaging providers without a selected channel plan", + ); + }, + ); + + it( + "filters messaging providers to only enabledChannels when provided", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-filter-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "enabled-channels-filter.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); + +const commands = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ + commands, + createdSandbox, +}); +runner.runCapture = (command) => { + const createdIdentity = createdSandbox.capture(command); + if (createdIdentity !== null) return createdIdentity; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = () => true; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; + process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; + process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; + // Only enable telegram — discord and slack should be filtered out + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], null, null, null, null, null, []], createFixture)); + console.log(JSON.stringify({ sandboxName, commands })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + // Only telegram provider should be created + const providerCommands = payload.commands.filter((e: CommandEntry) => + e.command.includes("provider create"), + ); + const telegramProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-telegram-bridge"), + ); + assert.ok(telegramProvider, "expected telegram provider to be created"); + + // Discord and slack providers should NOT be created + const discordProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-discord-bridge"), + ); + assert.ok(!discordProvider, "discord provider should be filtered out"); + + const slackProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-slack-bridge"), + ); + assert.ok(!slackProvider, "slack provider should be filtered out"); + + // Sandbox create should only have the telegram --provider flag + const createCommand = payload.commands.find((e: CommandEntry) => + e.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.match(createCommand.command, /--provider my-assistant-telegram-bridge/); + assert.doesNotMatch(createCommand.command, /my-assistant-discord-bridge/); + assert.doesNotMatch(createCommand.command, /my-assistant-slack-bridge/); + }, + ); + } + + if (group === "configuration") { + it( + "does not create messaging providers from ambient credentials without a selected plan (#10277)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-empty-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "enabled-channels-empty.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); + +const commands = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = (command, opts = {}) => { + commands.push({ command: _n(command), env: opts.env || null }); + return fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw") ?? createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { + const createdIdentity = createdSandbox.capture(command); + if (createdIdentity !== null) return createdIdentity; + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; + return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = () => true; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + createdSandbox.create(args.flat()); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; + process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; + process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; + // No selected messaging plan — ambient repository credentials are unrelated to this request. + const sandboxName = await createSandbox( + ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + "gpt-5.4", + "nvidia-prod", + null, + "my-assistant", + null, + null, + null, + null, + null, + null, + null, + [], + ], + createFixture, + ), + ); + console.log(JSON.stringify({ sandboxName, commands })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + // No messaging providers should be created at all + const providerCommands = payload.commands.filter((e: CommandEntry) => + e.command.includes("provider create"), + ); + assert.equal( + providerCommands.length, + 0, + "no providers should be created without a selected messaging plan", + ); + + // Sandbox create should have no --provider flags for messaging bridges + const createCommand = payload.commands.find((e: CommandEntry) => + e.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.doesNotMatch(createCommand.command, /discord-bridge/); + assert.doesNotMatch(createCommand.command, /slack-bridge/); + assert.doesNotMatch(createCommand.command, /telegram-bridge/); + }, + ); + + it( + "non-interactive setupMessagingChannels returns channels with tokens", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-noninteractive-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-noninteractive.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +// Stub the manifest-driven Telegram reachability hook so this test does not +// make a real network call. +global.fetch = async () => ({ + ok: true, + status: 200, + json: async () => ({ ok: true, result: { id: 1, is_bot: true } }), + text: async () => "", +}); + +const { setupMessagingChannels } = require(${onboardPath}); + +(async () => { + // Only set telegram and slack tokens — discord should be absent + process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; + process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; + process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token"; + process.env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; + const result = await setupMessagingChannels(); + console.log(JSON.stringify(result)); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const channels = parseStdoutJson(result.stdout); + + // Should return only the channels that have tokens set + assert.ok(Array.isArray(channels), "expected an array return value"); + assert.ok(channels.includes("telegram"), "expected telegram in returned channels"); + assert.ok(channels.includes("slack"), "expected slack in returned channels"); + assert.ok(!channels.includes("discord"), "discord should not be in returned channels"); + }, + ); + + it( + "non-interactive setupMessagingChannels drops Slack when live Slack API validation rejects the token", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-slack-live-reject-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-slack-live-reject.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const httpProbePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "adapters", "http", "probe.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const httpProbe = require(${httpProbePath}); +httpProbe.runCurlProbe = (argv) => { + const url = argv[argv.length - 1] || ""; + if (String(url).includes("auth.test")) { + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"ok":false,"error":"invalid_auth"}', + stderr: "", + message: "", + }; + } + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"ok":true}', + stderr: "", + message: "", + }; +}; + +const { setupMessagingChannels } = require(${onboardPath}); + +(async () => { + delete process.env.TELEGRAM_BOT_TOKEN; + delete process.env.DISCORD_BOT_TOKEN; + process.env.SLACK_BOT_TOKEN = "xoxb-fake-bot-token"; + process.env.SLACK_APP_TOKEN = "xapp-fake-app-token"; + const result = await setupMessagingChannels(); + console.log(JSON.stringify(result)); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const channels = parseStdoutJson(result.stdout); + + assert.ok(Array.isArray(channels), "expected an array return value"); + assert.ok(!channels.includes("slack"), "Slack should be dropped after API rejection"); + assert.doesNotMatch(result.stdout, /xoxb-fake-bot-token/); + assert.doesNotMatch(result.stderr, /xoxb-fake-bot-token/); + }, + ); + + it( + "non-interactive setupMessagingChannels returns empty array when no tokens set", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-no-tokens-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-no-tokens.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const { setupMessagingChannels } = require(${onboardPath}); + +(async () => { + // No messaging tokens set + delete process.env.TELEGRAM_BOT_TOKEN; + delete process.env.DISCORD_BOT_TOKEN; + delete process.env.SLACK_BOT_TOKEN; + delete process.env.SLACK_APP_TOKEN; + const result = await setupMessagingChannels(); + console.log(JSON.stringify(result)); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + TELEGRAM_BOT_TOKEN: "", + DISCORD_BOT_TOKEN: "", + SLACK_BOT_TOKEN: "", + SLACK_APP_TOKEN: "", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const channels = parseStdoutJson(result.stdout); + + assert.ok(Array.isArray(channels), "expected an array return value"); + assert.equal(channels.length, 0, "expected empty array when no tokens are set"); + }, + ); + + it( + "interactive setupMessagingChannels drops slack when prompted token fails tokenFormat check (#1912)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-slack-format-reject-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "slack-format-reject.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + // Subscript: mocks credentials.prompt to return a bogus Slack token, + // exposes MESSAGING_CHANNELS so the parent can look up the Slack toggle + // digit, and asserts that setupMessagingChannels rejects the invalid + // token without persisting it. Slack is the 3rd channel in insertion + // order today (telegram, discord, slack) but we compute the index + // dynamically to avoid a brittle coupling to that ordering. + const script = String.raw` +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); + +const saveCalls = []; +credentials.saveCredential = (key, value) => { saveCalls.push({ key, value }); }; +credentials.getCredential = () => null; +credentials.prompt = async (message) => { + if (message.includes("Slack Bot Token")) return "abcd"; + return ""; +}; + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); + +(async () => { + delete process.env.TELEGRAM_BOT_TOKEN; + delete process.env.DISCORD_BOT_TOKEN; + delete process.env.SLACK_BOT_TOKEN; + delete process.env.SLACK_APP_TOKEN; + + const result = await setupMessagingChannels(); + console.log(JSON.stringify({ + result, + saveCalls, + slackIndex1Based: MESSAGING_CHANNELS.findIndex((c) => c.name === "slack") + 1, + })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + // Dry run with just Enter — no toggles, empty result — used to read back + // Slack's 1-based index from the same subscript so the real run can + // press the right digit. + const introspect = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + input: "\n", + }); + assert.equal(introspect.status, 0, introspect.stderr); + const introspectOut = JSON.parse(introspect.stdout.trim().split("\n").pop()!); + const slackIdx = introspectOut.slackIndex1Based; + assert.ok(slackIdx >= 1, `unexpected slack index: ${slackIdx}`); + + // Real run: press Slack's digit, Enter. Slack gets toggled on, prompt + // fires, mocked prompt returns "abcd", tokenFormat regex rejects it, + // channel is dropped, saveCredential never runs for SLACK_BOT_TOKEN. + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + input: `${slackIdx}\n`, + }); + + assert.equal(result.status, 0, result.stderr); + const out = JSON.parse(result.stdout.trim().split("\n").pop()!); + + assert.ok( + !out.result.includes("slack"), + `slack should have been dropped after invalid token; got ${JSON.stringify(out.result)}`, + ); + assert.ok( + !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_BOT_TOKEN"), + `SLACK_BOT_TOKEN should NOT have been persisted; saveCalls=${JSON.stringify(out.saveCalls)}`, + ); + assert.ok( + result.stderr.includes("Invalid format") || result.stdout.includes("Invalid format"), + `expected 'Invalid format' warning; stderr=${result.stderr} stdout=${result.stdout}`, + ); + }, + ); + } + }); +} diff --git a/test/onboarding/onboard-sandbox-build.test.ts b/test/helpers/onboard-sandbox-build-suite.ts similarity index 53% rename from test/onboarding/onboard-sandbox-build.test.ts rename to test/helpers/onboard-sandbox-build-suite.ts index 4d267d92510..e6b8f646640 100644 --- a/test/onboarding/onboard-sandbox-build.test.ts +++ b/test/helpers/onboard-sandbox-build-suite.ts @@ -8,13 +8,13 @@ import os from "node:os"; import path from "node:path"; import { beforeEach, describe, it, vi } from "vitest"; -import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; +import { writeOkOpenshell } from "./onboard-openshell-fixture"; import { type CommandEntry, onboardScriptMocksPath, parseStdoutJson, stripMessagingEnv, -} from "../helpers/onboard-split-context"; +} from "./onboard-split-context"; beforeEach(() => { vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); @@ -22,33 +22,37 @@ beforeEach(() => { vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); }); -describe("onboard helpers", () => { - it( - "creates the stock managed sandbox without uploading an external OpenClaw config file", - { - timeout: 90_000, - }, - async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-sandbox-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "create-sandbox-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); +export type OnboardSandboxBuildGroup = "base" | "dashboard"; - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` +export function registerOnboardSandboxBuildTests(group: OnboardSandboxBuildGroup): void { + describe(`onboard helpers (${group})`, () => { + if (group === "base") { + it( + "creates the stock managed sandbox without uploading an external OpenClaw config file", + { + timeout: 90_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-sandbox-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` const runner = require(${runnerPath}); const fixtureMocks = require(${onboardScriptMocksPath}); const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ @@ -125,100 +129,102 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + assert.equal(payload.sandboxName, "my-assistant"); + // createSandbox no longer marks the sandbox default — that is deferred to the + // finalization step so a cancel at policy presets can't leave an unconfigured + // sandbox as default (#4614). + assert.deepEqual(payload.defaultCalls, []); + assert.ok( + payload.registerCalls.some( + (entry: Record) => + entry.name === "my-assistant" && + entry.model === "gpt-5.4" && + Object.prototype.hasOwnProperty.call(entry, "agentVersion"), + ), + "expected registry metadata for created sandbox", + ); + assert.ok( + payload.updateCalls.every( + (call: { name: string; updates: Record }) => + call.name === "my-assistant" && call.updates, + ), + "expected any registry metadata updates to target the created sandbox", + ); + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.match(createCommand.command, /nemoclaw-managed-startup-hold/); + assert.match( + createCommand.command, + /--from ghcr\.io\/nvidia\/nemoclaw\/openclaw-sandbox@sha256:[0-9a-f]{64}/, + ); + assert.doesNotMatch(createCommand.command, /--upload/); + assert.doesNotMatch(createCommand.command, /OPENCLAW_CONFIG_PATH/); + assert.doesNotMatch(createCommand.command, /NVIDIA_INFERENCE_API_KEY=/); + assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); + assert.ok( + !payload.commands.some((entry: CommandEntry) => + entry.command.includes("forward service my-assistant"), + ), + "sandbox creation must defer forwarding until agent setup or final recovery", + ); }, - }); - - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - assert.equal(payload.sandboxName, "my-assistant"); - // createSandbox no longer marks the sandbox default — that is deferred to the - // finalization step so a cancel at policy presets can't leave an unconfigured - // sandbox as default (#4614). - assert.deepEqual(payload.defaultCalls, []); - assert.ok( - payload.registerCalls.some( - (entry: Record) => - entry.name === "my-assistant" && - entry.model === "gpt-5.4" && - Object.prototype.hasOwnProperty.call(entry, "agentVersion"), - ), - "expected registry metadata for created sandbox", - ); - assert.ok( - payload.updateCalls.every( - (call: { name: string; updates: Record }) => - call.name === "my-assistant" && call.updates, - ), - "expected any registry metadata updates to target the created sandbox", - ); - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.match(createCommand.command, /nemoclaw-managed-startup-hold/); - assert.match( - createCommand.command, - /--from ghcr\.io\/nvidia\/nemoclaw\/openclaw-sandbox@sha256:[0-9a-f]{64}/, - ); - assert.doesNotMatch(createCommand.command, /--upload/); - assert.doesNotMatch(createCommand.command, /OPENCLAW_CONFIG_PATH/); - assert.doesNotMatch(createCommand.command, /NVIDIA_INFERENCE_API_KEY=/); - assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); - assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); - assert.ok( - !payload.commands.some((entry: CommandEntry) => - entry.command.includes("forward service my-assistant"), - ), - "sandbox creation must defer forwarding until agent setup or final recovery", ); - }, - ); - it("resolves the sandbox base for an explicit custom Dockerfile", async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-agent-base-skip-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "agent-base-skip.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const hermesPolicyPath = JSON.stringify( - path.join(repoRoot, "agents", "hermes", "policy-additions.yaml"), - ); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const agentOnboardPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "agent", "onboard.ts"), - ); - const sandboxBaseImagePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "sandbox-base-image.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + it("resolves the sandbox base for an explicit custom Dockerfile", async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-agent-base-skip-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "agent-base-skip.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const hermesPolicyPath = JSON.stringify( + path.join(repoRoot, "agents", "hermes", "policy-additions.yaml"), + ); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const agentOnboardPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "agent", "onboard.ts"), + ); + const sandboxBaseImagePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "sandbox-base-image.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); @@ -376,76 +382,81 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...stripMessagingEnv(process.env), - HOME: tmpDir, - NEMOCLAW_HOME: path.join(tmpDir, ".nemoclaw"), - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - commands: CommandEntry[]; - logs: string[]; - warnings: string[]; - baseResolutionCalls: Array<{ imageName?: string; dockerfilePath?: string }>; - }>(result.stdout); - assert.equal(payload.baseResolutionCalls.length, 1); - assert.equal(payload.baseResolutionCalls[0]?.imageName, "ghcr.io/nvidia/nemoclaw/sandbox-base"); - assert.equal( - payload.baseResolutionCalls[0]?.dockerfilePath, - path.join(repoRoot, "Dockerfile.base"), - ); - const createCommand = payload.commands.find((entry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.match(createCommand.command, /--provider hermes-sandbox-hermes-tool-gateway/); - assert.doesNotMatch(createCommand.command, /TOOL_GATEWAY_USER_TOKEN=/); - assert.doesNotMatch(createCommand.command, /NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN=/); - assert.ok( - payload.logs.some((line) => line.includes("Pinning base image to sha256:aaaaaaaaaaaa")), - "explicit custom Dockerfile path should log sandbox-base pinning", - ); - assert.ok( - !payload.warnings.some((line) => line.includes("base image")), - "resolved custom Dockerfile path should not warn about sandbox-base availability", - ); - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...stripMessagingEnv(process.env), + HOME: tmpDir, + NEMOCLAW_HOME: path.join(tmpDir, ".nemoclaw"), + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson<{ + commands: CommandEntry[]; + logs: string[]; + warnings: string[]; + baseResolutionCalls: Array<{ imageName?: string; dockerfilePath?: string }>; + }>(result.stdout); + assert.equal(payload.baseResolutionCalls.length, 1); + assert.equal( + payload.baseResolutionCalls[0]?.imageName, + "ghcr.io/nvidia/nemoclaw/sandbox-base", + ); + assert.equal( + payload.baseResolutionCalls[0]?.dockerfilePath, + path.join(repoRoot, "Dockerfile.base"), + ); + const createCommand = payload.commands.find((entry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.match(createCommand.command, /--provider hermes-sandbox-hermes-tool-gateway/); + assert.doesNotMatch(createCommand.command, /TOOL_GATEWAY_USER_TOKEN=/); + assert.doesNotMatch(createCommand.command, /NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN=/); + assert.ok( + payload.logs.some((line) => line.includes("Pinning base image to sha256:aaaaaaaaaaaa")), + "explicit custom Dockerfile path should log sandbox-base pinning", + ); + assert.ok( + !payload.warnings.some((line) => line.includes("base image")), + "resolved custom Dockerfile path should not warn about sandbox-base availability", + ); + }); - it("skips OpenClaw sandbox-base resolution for the stock managed image path", async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-base-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "openclaw-base-resolve.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const buildContextPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "sandbox", "build-context.ts"), - ); - const sandboxBaseImagePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "sandbox-base-image.ts"), - ); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + it("skips OpenClaw sandbox-base resolution for the stock managed image path", async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-base-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "openclaw-base-resolve.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const buildContextPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "sandbox", "build-context.ts"), + ); + const sandboxBaseImagePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "sandbox-base-image.ts"), + ); + const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); @@ -569,51 +580,55 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...stripMessagingEnv(process.env), - HOME: tmpDir, - NEMOCLAW_HOME: path.join(tmpDir, ".nemoclaw"), - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - logs: string[]; - baseResolutionCalls: Array<{ imageName?: string }>; - }>(result.stdout); - assert.equal(payload.baseResolutionCalls.length, 0); - assert.ok( - !payload.logs.some((line) => line.includes("Pinning base image")), - "stock managed-image onboarding must not enter sandbox-base resolution", - ); - }); - - it("defers a remote dashboard forward until post-create recovery", async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-remote-forward-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "create-sandbox-remote-forward.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...stripMessagingEnv(process.env), + HOME: tmpDir, + NEMOCLAW_HOME: path.join(tmpDir, ".nemoclaw"), + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson<{ + logs: string[]; + baseResolutionCalls: Array<{ imageName?: string }>; + }>(result.stdout); + assert.equal(payload.baseResolutionCalls.length, 0); + assert.ok( + !payload.logs.some((line) => line.includes("Pinning base image")), + "stock managed-image onboarding must not enter sandbox-base resolution", + ); + }); + } + + if (group === "dashboard") { + it("defers a remote dashboard forward until post-create recovery", async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-remote-forward-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-remote-forward.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` const runner = require(${runnerPath}); const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); @@ -685,48 +700,50 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const commands = parseStdoutJson(result.stdout); - assert.ok( - !commands.some((entry: CommandEntry) => - entry.command.includes("forward service my-assistant"), - ), - "sandbox creation must not launch the remote forward before agent setup", - ); - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const commands = parseStdoutJson(result.stdout); + assert.ok( + !commands.some((entry: CommandEntry) => + entry.command.includes("forward service my-assistant"), + ), + "sandbox creation must not launch the remote forward before agent setup", + ); + }); - it("injects NEMOCLAW_DASHBOARD_PORT into sandbox create envArgs when set (#1925)", async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-dashboard-port-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "dashboard-port-envargs.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + it("injects NEMOCLAW_DASHBOARD_PORT into sandbox create envArgs when set (#1925)", async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-dashboard-port-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "dashboard-port-envargs.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` const runner = require(${runnerPath}); const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); @@ -802,82 +819,84 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - // Strip CHAT_UI_URL so createSandbox falls back to http://127.0.0.1:19000. - // Without this, a CHAT_UI_URL set in the developer's shell or CI would be - // inherited, causing chatUiUrl to use the wrong port and making the forward - // command assertion below fail spuriously. - const { - CHAT_UI_URL: _stripped, - HTTP_PROXY: _httpProxy, - HTTPS_PROXY: _httpsProxy, - NO_PROXY: _noProxy, - http_proxy: _lowerHttpProxy, - https_proxy: _lowerHttpsProxy, - no_proxy: _lowerNoProxy, - ...inheritedEnv - } = process.env; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...inheritedEnv, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_DASHBOARD_PORT: "19000", - HTTP_PROXY: "http://127.0.0.1:8888", - HTTPS_PROXY: "http://127.0.0.1:8888", - NO_PROXY: "corp.internal", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - // Part 1 of fix (#1925): NEMOCLAW_DASHBOARD_PORT must be in envArgs so - // nemoclaw-start.sh can unconditionally override CHAT_UI_URL at runtime, - // overriding whatever value the Docker image had baked in. - assert.match(createCommand.command, /NEMOCLAW_DASHBOARD_PORT=19000/); - assert.match(createCommand.command, /HTTP_PROXY=http:\/\/127\.0\.0\.1:8888/); - assert.match(createCommand.command, /HTTPS_PROXY=http:\/\/127\.0\.0\.1:8888/); - // OpenClaw home/state/workspace dirs must be pinned in the sandbox env so - // `openclaw skills install` and `openclaw skills list` resolve the same - // paths. Without this, the upstream skill loader can fall back to a - // hardcoded DEFAULT_AGENT_WORKSPACE_DIR that drifts from the install path - // and hides workspace-installed skills from `skills list`. - assert.match(createCommand.command, /OPENCLAW_HOME=\/sandbox(?:\s|$)/); - assert.match(createCommand.command, /OPENCLAW_STATE_DIR=\/sandbox\/\.openclaw(?:\s|$)/); - assert.match( - createCommand.command, - /OPENCLAW_WORKSPACE_DIR=\/sandbox\/\.openclaw\/workspace(?:\s|$)/, - ); - const noProxyMatch = createCommand.command.match(/(?:^|\s)NO_PROXY=([^\s]+)/); - assert.ok( - noProxyMatch, - `expected NO_PROXY in sandbox create command:\n${createCommand.command}`, - ); - const noProxyEntries = noProxyMatch[1].split(","); - assert.ok(noProxyEntries.includes("corp.internal")); - assert.ok(noProxyEntries.includes("localhost")); - assert.ok(noProxyEntries.includes("127.0.0.1")); - assert.ok(noProxyEntries.includes("host.docker.internal")); - assert.ok( - !payload.commands.some((entry: CommandEntry) => - entry.command.includes("forward service my-assistant"), - ), - "sandbox creation must defer the custom-port forward until agent setup or final recovery", - ); + fs.writeFileSync(scriptPath, script); + + // Strip CHAT_UI_URL so createSandbox falls back to http://127.0.0.1:19000. + // Without this, a CHAT_UI_URL set in the developer's shell or CI would be + // inherited, causing chatUiUrl to use the wrong port and making the forward + // command assertion below fail spuriously. + const { + CHAT_UI_URL: _stripped, + HTTP_PROXY: _httpProxy, + HTTPS_PROXY: _httpsProxy, + NO_PROXY: _noProxy, + http_proxy: _lowerHttpProxy, + https_proxy: _lowerHttpsProxy, + no_proxy: _lowerNoProxy, + ...inheritedEnv + } = process.env; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...inheritedEnv, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_DASHBOARD_PORT: "19000", + HTTP_PROXY: "http://127.0.0.1:8888", + HTTPS_PROXY: "http://127.0.0.1:8888", + NO_PROXY: "corp.internal", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + // Part 1 of fix (#1925): NEMOCLAW_DASHBOARD_PORT must be in envArgs so + // nemoclaw-start.sh can unconditionally override CHAT_UI_URL at runtime, + // overriding whatever value the Docker image had baked in. + assert.match(createCommand.command, /NEMOCLAW_DASHBOARD_PORT=19000/); + assert.match(createCommand.command, /HTTP_PROXY=http:\/\/127\.0\.0\.1:8888/); + assert.match(createCommand.command, /HTTPS_PROXY=http:\/\/127\.0\.0\.1:8888/); + // OpenClaw home/state/workspace dirs must be pinned in the sandbox env so + // `openclaw skills install` and `openclaw skills list` resolve the same + // paths. Without this, the upstream skill loader can fall back to a + // hardcoded DEFAULT_AGENT_WORKSPACE_DIR that drifts from the install path + // and hides workspace-installed skills from `skills list`. + assert.match(createCommand.command, /OPENCLAW_HOME=\/sandbox(?:\s|$)/); + assert.match(createCommand.command, /OPENCLAW_STATE_DIR=\/sandbox\/\.openclaw(?:\s|$)/); + assert.match( + createCommand.command, + /OPENCLAW_WORKSPACE_DIR=\/sandbox\/\.openclaw\/workspace(?:\s|$)/, + ); + const noProxyMatch = createCommand.command.match(/(?:^|\s)NO_PROXY=([^\s]+)/); + assert.ok( + noProxyMatch, + `expected NO_PROXY in sandbox create command:\n${createCommand.command}`, + ); + const noProxyEntries = noProxyMatch[1].split(","); + assert.ok(noProxyEntries.includes("corp.internal")); + assert.ok(noProxyEntries.includes("localhost")); + assert.ok(noProxyEntries.includes("127.0.0.1")); + assert.ok(noProxyEntries.includes("host.docker.internal")); + assert.ok( + !payload.commands.some((entry: CommandEntry) => + entry.command.includes("forward service my-assistant"), + ), + "sandbox creation must defer the custom-port forward until agent setup or final recovery", + ); + }); + } }); -}); +} diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index d06ee84d820..7118e93f8d9 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -56,15 +56,19 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ "test/onboarding/onboard-custom-dockerfile.test.ts", "test/onboarding/onboard-extra-provider-reconciliation.test.ts", "test/onboarding/onboard-fresh-create-provider-identity.test.ts", - "test/onboarding/onboard-fresh-create-recovery-identity.test.ts", + "test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts", + "test/onboarding/onboard-fresh-create-recovery-registration-identity.test.ts", "test/onboarding/onboard-fresh-create-cancellation-identity.test.ts", "test/onboarding/onboard-installer-restore-intent.test.ts", "test/onboarding/onboard-managed-image-buildless-e2e.test.ts", "test/onboarding/onboard-mcp-observability-redirect.test.ts", - "test/onboarding/onboard-messaging.test.ts", + "test/onboarding/onboard-messaging-image-configuration.test.ts", + "test/onboarding/onboard-messaging-provider.test.ts", + "test/onboarding/onboard-messaging-recreation.test.ts", "test/onboarding/onboard-prepared-build-context.test.ts", "test/onboarding/onboard-reservation-recreate.test.ts", - "test/onboarding/onboard-sandbox-build.test.ts", + "test/onboarding/onboard-sandbox-build-base.test.ts", + "test/onboarding/onboard-sandbox-build-dashboard.test.ts", "test/onboarding/onboard-sandbox-recreation.test.ts", "test/onboarding/onboard-script-mocks-contract.test.ts", "test/onboarding/onboard-terminal-dashboard.test.ts", diff --git a/test/onboarding/onboard-fresh-create-recovery-identity.test.ts b/test/onboarding/onboard-fresh-create-recovery-registration-identity.test.ts similarity index 82% rename from test/onboarding/onboard-fresh-create-recovery-identity.test.ts rename to test/onboarding/onboard-fresh-create-recovery-registration-identity.test.ts index 4e0bed4e2a6..ee0696727fc 100644 --- a/test/onboarding/onboard-fresh-create-recovery-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-recovery-registration-identity.test.ts @@ -3,4 +3,4 @@ import { registerFreshCreateIdentityTests } from "../helpers/onboard-fresh-create-identity-suite"; -registerFreshCreateIdentityTests("recovery"); +registerFreshCreateIdentityTests("recovery-state"); diff --git a/test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts b/test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts new file mode 100644 index 00000000000..8ec4ee8183f --- /dev/null +++ b/test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerFreshCreateIdentityTests } from "../helpers/onboard-fresh-create-identity-suite"; + +registerFreshCreateIdentityTests("recovery-policy"); diff --git a/test/onboarding/onboard-messaging-image-configuration.test.ts b/test/onboarding/onboard-messaging-image-configuration.test.ts new file mode 100644 index 00000000000..764dc56c5dd --- /dev/null +++ b/test/onboarding/onboard-messaging-image-configuration.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerOnboardMessagingTests } from "../helpers/onboard-messaging-suite"; + +registerOnboardMessagingTests("configuration"); diff --git a/test/onboarding/onboard-messaging-provider.test.ts b/test/onboarding/onboard-messaging-provider.test.ts new file mode 100644 index 00000000000..b668978fb06 --- /dev/null +++ b/test/onboarding/onboard-messaging-provider.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerOnboardMessagingTests } from "../helpers/onboard-messaging-suite"; + +registerOnboardMessagingTests("provider"); diff --git a/test/onboarding/onboard-messaging-recreation.test.ts b/test/onboarding/onboard-messaging-recreation.test.ts new file mode 100644 index 00000000000..c8bfc2cbbcb --- /dev/null +++ b/test/onboarding/onboard-messaging-recreation.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerOnboardMessagingTests } from "../helpers/onboard-messaging-suite"; + +registerOnboardMessagingTests("lifecycle"); diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts deleted file mode 100644 index d707e8c59e1..00000000000 --- a/test/onboarding/onboard-messaging.test.ts +++ /dev/null @@ -1,1971 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import { createRequire } from "node:module"; -import os from "node:os"; -import path from "node:path"; - -import { beforeEach, describe, it, vi } from "vitest"; -import YAML from "yaml"; - -import { - activeChannelsFromDockerfile, - encodeMessagingPlanForChannels, - messagingPlanLiteral, - parseMessagingFixturePayload, - writeCustomMessagingDockerfile, -} from "../helpers/messaging-plan-fixtures"; -import { runBoundedOnboardScript } from "../helpers/onboard-child-process-harness"; -import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; - -type CommandEntry = { - command: string; - env?: Record; - policyContent?: string; - policyReadError?: string; - dockerfileContent?: string; - dockerfileReadError?: string; - providerRevisions?: Record | null; - rawCredentialInEnv?: boolean; -}; -const parseStdoutJson = parseMessagingFixturePayload; -const repoRoot = path.join(import.meta.dirname, "../.."); -const requireForTest = createRequire(import.meta.url); -const yamlModulePath = requireForTest.resolve("yaml"); -const onboardScriptMocksPath = JSON.stringify( - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), -); -beforeEach(() => { - vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); - vi.stubEnv("NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE", "1"); - vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); -}); -describe("onboard messaging", () => { - it( - "creates providers for messaging tokens and attaches them to the sandbox", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-providers-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-provider-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); -const commands = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ sandboxName: "my-assistant" }); createdSandbox.installRuntimeObservation(); -runner.run = fixtureMocks.createStatefulMessagingProviderRunner({ commands, createdSandbox }); -runner.runCapture = (command) => { - const sandboxCapture = createdSandbox.capture(command); - if (sandboxCapture !== null) return sandboxCapture; - if (_n(command).includes("provider get")) return "Provider: discord-bridge"; - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { - defaultCurlOutput: "ok", - }); - if (mockedCapture !== null) return mockedCapture; - } - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const policyMatch = command.match(/--policy ([^ ]+)/); - if (policyMatch) { - try { - entry.policyContent = fs.readFileSync(policyMatch[1], "utf-8"); - } catch (error) { - entry.policyReadError = String(error); - } - } - commands.push(entry); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; -const { createSandbox, setupMessagingChannels } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY = "1"; - process.env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; - process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; - process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token-value"; - process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - process.env.KUBECONFIG = "/tmp/host-kubeconfig"; - process.env.SSH_AUTH_SOCK = "/tmp/host-ssh-agent.sock"; - await setupMessagingChannels(null, null, "my-assistant"); - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ - sandboxName, - commands, - messagingPlanEnv: process.env.NEMOCLAW_MESSAGING_PLAN_B64, - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - const result = runBoundedOnboardScript(scriptPath, { - cwd: repoRoot, - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - const providerCommands = payload.commands.filter((e: CommandEntry) => - e.command.includes("provider create"), - ); - const discordProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-discord-bridge"), - ); - assert.ok(discordProvider, "expected my-assistant-discord-bridge provider create command"); - assert.match(discordProvider.command, /--credential DISCORD_BOT_TOKEN/); - - const slackProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-slack-bridge"), - ); - assert.ok(slackProvider, "expected my-assistant-slack-bridge provider create command"); - assert.match(slackProvider.command, /--credential SLACK_BOT_TOKEN/); - - const telegramProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-telegram-bridge"), - ); - assert.ok(telegramProvider, "expected my-assistant-telegram-bridge provider create command"); - assert.match(telegramProvider.command, /--credential TELEGRAM_BOT_TOKEN/); - - const createCommand = payload.commands.find((e: CommandEntry) => - e.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.match(createCommand.command, /--provider my-assistant-discord-bridge/); - assert.match(createCommand.command, /--provider my-assistant-slack-bridge/); - assert.match(createCommand.command, /--provider my-assistant-telegram-bridge/); - assert.match(createCommand.command, /--policy [^ ]*nemoclaw-initial-policy[^ ]*\.yaml/); - assert.equal(createCommand.policyReadError, undefined); - const policyDoc = YAML.parse(createCommand.policyContent || "") || {}; - const slackEndpointHosts = (policyDoc.network_policies?.slack?.endpoints || []).map( - (entry: { host?: string }) => entry.host, - ); - const slackWebsocketHosts = slackEndpointHosts - .filter( - (host: string | undefined) => - host === "wss-primary.slack.com" || host === "wss-backup.slack.com", - ) - .sort(); - assert.deepEqual( - slackWebsocketHosts, - ["wss-backup.slack.com", "wss-primary.slack.com"].sort(), - ); - - assert.doesNotMatch(createCommand.command, /test-discord-token-value/); - assert.doesNotMatch(createCommand.command, /123456:ABC-test-telegram-token/); - assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); - assert.doesNotMatch(createCommand.command, /TELEGRAM_BOT_TOKEN=/); - assert.doesNotMatch(createCommand.command, /xoxb-test-slack-token-value/); - assert.doesNotMatch(createCommand.command, /xapp-test-slack-app-token-value/); - assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); - assert.doesNotMatch(createCommand.command, /SLACK_APP_TOKEN=/); - assert.doesNotMatch(createCommand.command, /NEMOCLAW_MESSAGING_PLAN_B64=/); - - assert.ok(payload.messagingPlanEnv, "expected serialized messaging plan in host process env"); - const messagingPlan = JSON.parse( - Buffer.from(payload.messagingPlanEnv, "base64").toString("utf8"), - ); - assert.equal(messagingPlan.sandboxName, "my-assistant"); - assert.deepEqual( - messagingPlan.channels.map((channel: { channelId: string }) => channel.channelId).sort(), - ["discord", "slack", "telegram"].sort(), - ); - assert.doesNotMatch(JSON.stringify(messagingPlan), /test-discord-token-value/); - assert.doesNotMatch(JSON.stringify(messagingPlan), /123456:ABC-test-telegram-token/); - - assert.ok(createCommand.env, "expected env to be captured from spawn call"); - assert.equal( - createCommand.env.DISCORD_BOT_TOKEN, - undefined, - "DISCORD_BOT_TOKEN must not be in sandbox env", - ); - assert.equal( - createCommand.env.SLACK_BOT_TOKEN, - undefined, - "SLACK_BOT_TOKEN must not be in sandbox env", - ); - assert.equal( - createCommand.env.SLACK_APP_TOKEN, - undefined, - "SLACK_APP_TOKEN must not be in sandbox env", - ); - assert.equal( - createCommand.env.TELEGRAM_BOT_TOKEN, - undefined, - "TELEGRAM_BOT_TOKEN must not be in sandbox env", - ); - assert.equal( - createCommand.env.NVIDIA_INFERENCE_API_KEY, - undefined, - "NVIDIA_INFERENCE_API_KEY must not be in sandbox env", - ); - assert.equal( - createCommand.env.KUBECONFIG, - undefined, - "KUBECONFIG must not be in sandbox env", - ); - assert.equal( - createCommand.env.SSH_AUTH_SOCK, - undefined, - "SSH_AUTH_SOCK must not be in sandbox env", - ); - - const envString = JSON.stringify(createCommand.env); - assert.ok( - !envString.includes("test-discord-token-value"), - "Discord token value must not leak into sandbox env", - ); - assert.ok( - !envString.includes("xoxb-test-slack-token-value"), - "Slack bot token value must not leak into sandbox spawn env", - ); - assert.ok( - !envString.includes("xapp-test-slack-app-token-value"), - "Slack app token value must not leak into sandbox spawn env", - ); - assert.ok( - !envString.includes("123456:ABC-test-telegram-token"), - "Telegram token value must not leak into sandbox env", - ); - }, - ); - - it( - "preserves Hermes Slack policy when Slack is active at sandbox create time", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-hermes-slack-")); - try { - const fakeBin = path.join(tmpDir, "bin"); - const customBuildDir = path.join(tmpDir, "custom-build"); - const customDockerfilePath = path.join(customBuildDir, "Dockerfile"); - const scriptPath = path.join(tmpDir, "hermes-slack-policy.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const agentDefsPath = JSON.stringify(path.join(repoRoot, "src", "lib", "agent", "defs.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const yamlPath = JSON.stringify(yamlModulePath); - const customDockerfileArg = JSON.stringify(customDockerfilePath); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.mkdirSync(customBuildDir, { recursive: true }); - fs.writeFileSync( - customDockerfilePath, - "FROM scratch\nARG NEMOCLAW_MESSAGING_PLAN_B64=\nARG NEMOCLAW_TOOL_DISCLOSURE=progressive\nENV NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE}\n", - ); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); -const YAML = require(${yamlPath}); -const { loadAgent } = require(${agentDefsPath}); -require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "docker-driver-platform.ts"))}).isLinuxDockerDriverGatewayEnabled = () => false; -const nonSlackMessagingEnvKeys = [ - "DISCORD_BOT_TOKEN", - "DISCORD_SERVER_ID", - "DISCORD_SERVER_IDS", - "DISCORD_ALLOWED_IDS", - "DISCORD_USER_ID", - "DISCORD_REQUIRE_MENTION", - "TELEGRAM_BOT_TOKEN", - "TELEGRAM_ALLOWED_IDS", - "TELEGRAM_REQUIRE_MENTION", -]; - -const commands = []; -let registeredSandbox = null; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ sandboxName: "my-assistant" }); createdSandbox.installRuntimeObservation(); -runner.run = fixtureMocks.createStatefulMessagingProviderRunner({ commands, createdSandbox }); -runner.runCapture = (command) => { - const sandboxCapture = createdSandbox.capture(command); - if (sandboxCapture !== null) return sandboxCapture; - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { - defaultCurlOutput: "ok", - }); - if (mockedCapture !== null) return mockedCapture; - } - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { - registeredSandbox = entry; - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const policyMatch = command.match(/--policy ([^ ]+)/); - if (policyMatch) { - entry.policyPath = policyMatch[1]; - try { - entry.policyContent = fs.readFileSync(policyMatch[1], "utf-8"); - } catch (error) { - entry.policyReadError = String(error); - } - } - commands.push(entry); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -const { createSandbox } = require(${onboardPath}); - -(async () => { - for (const key of nonSlackMessagingEnvKeys) delete process.env[key]; - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.NEMOCLAW_AGENT = "hermes"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; - process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token-value"; - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${customDockerfileArg}, loadAgent("hermes"), null, null, null, []], createFixture)); - const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); - const parsed = YAML.parse(createCommand?.policyContent || "") || {}; - const slack = parsed.network_policies?.slack || {}; - console.log(JSON.stringify({ - sandboxName, - createCommand: { - command: createCommand?.command || "", - policyPath: createCommand?.policyPath || "", - policyReadError: createCommand?.policyReadError || null, - }, - registeredPolicyStatePresent: "policies" in (registeredSandbox || {}), - slackBinaryPaths: (slack.binaries || []).map((entry) => entry.path), - slackEndpointHosts: (slack.endpoints || []).map((entry) => entry.host), - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - assert.ok(payload.createCommand.command.includes("sandbox create")); - assert.match(payload.createCommand.command, /--provider my-assistant-slack-bridge/); - assert.match(payload.createCommand.command, /--provider my-assistant-slack-app/); - assert.match(payload.createCommand.policyPath, /nemoclaw-initial-policy/); - assert.equal(payload.createCommand.policyReadError, null); - assert.equal(payload.registeredPolicyStatePresent, false); - assert.deepEqual(payload.slackBinaryPaths, [ - "/usr/local/bin/hermes", - "/usr/bin/python3*", - "/usr/bin/python3.13", - "/opt/hermes/.venv/bin/python3", - "/opt/hermes/.venv/bin/python", - ]); - assert.ok( - !payload.slackBinaryPaths.includes("/usr/local/bin/node"), - "Hermes Slack policy must not be replaced by the generic Node Slack preset", - ); - const slackWebsocketHosts = payload.slackEndpointHosts - .filter( - (host: string) => host === "wss-primary.slack.com" || host === "wss-backup.slack.com", - ) - .sort(); - assert.deepEqual( - slackWebsocketHosts, - ["wss-backup.slack.com", "wss-primary.slack.com"].sort(), - ); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); - - it( - "publishes attached OpenShell provider state before a messaging recreate starts (#9770)", - { timeout: 60_000 }, - async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-messaging-recreate-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-reuse-provider.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src/lib/state/registry.ts")); - const preflightPath = JSON.stringify(path.join(repoRoot, "src/lib/onboard/preflight.ts")); - const credentialsPath = JSON.stringify(path.join(repoRoot, "src/lib/credentials/store.ts")); - const telegramCredentialKeys = [ - "TELEGRAM_BOT_TOKEN", - "TELEGRAM_BOT_TOKEN_AGENT_A", - "TELEGRAM_BOT_TOKEN_AGENT_B", - ]; - const providerCredentialKeys = { - "compatible-endpoint": ["COMPATIBLE_API_KEY"], - "my-assistant-slack-app": ["SLACK_APP_TOKEN"], - "my-assistant-slack-bridge": ["SLACK_BOT_TOKEN"], - "my-assistant-telegram-bridge": telegramCredentialKeys, - }; - const expectedProviders = Object.keys(providerCredentialKeys).sort(); - const rawGatewayCredential = "gateway-only-provider-secret"; - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - const script = String.raw` -const runner = require(${runnerPath}), registry = require(${registryPath}), preflight = require(${preflightPath}), credentials = require(${credentialsPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const childProcess = require("node:child_process"), { EventEmitter } = require("node:events"); -const commands = [], credentialKeys = ${JSON.stringify(providerCredentialKeys)}; let registered = null; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ sandboxName: "my-assistant" }); createdSandbox.installRuntimeObservation(); -const providers = Object.keys(credentialKeys), revisions = new Map(providers.map((name) => [name, 1])), providerGetCounts = new Map(); -const rawGatewayCredential = ${JSON.stringify(rawGatewayCredential)}, gatewaySecrets = new Map(providers.map((name) => [name, rawGatewayCredential])); -registry.registerSandbox({ name: "my-assistant", messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["slack", "telegram", "whatsapp"])} } }); -runner.run = (command, opts = {}) => { - const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); if (profileResult !== null) return profileResult; - const providerGet = normalized.match(/provider get -g nemoclaw ([^ ]+)$/)?.[1]; if (providerGet) providerGetCounts.set(providerGet, (providerGetCounts.get(providerGet) || 0) + 1); if (providerGet === process.env.NEMOCLAW_TEST_FAIL_PROVIDER && providerGetCounts.get(providerGet) >= 4) return { status: 2, stderr: "transport unavailable" }; - if (providerGet && revisions.has(providerGet)) return { status: 0, stdout: "Name: " + providerGet + "\nType: " + (providerGet === "compatible-endpoint" ? "openai" : "nemoclaw-mcp-v1") + "\nCredential keys: " + credentialKeys[providerGet] + "\nConfig keys: " + (providerGet === "compatible-endpoint" ? "OPENAI_BASE_URL" : "") + "\n" }; - const refresh = normalized.match(/provider update -g nemoclaw ([^ ]+)$/)?.[1]; - if (refresh && gatewaySecrets.has(refresh)) { if (refresh === process.env.NEMOCLAW_TEST_FAIL_PROVIDER) return { status: 1 }; revisions.set(refresh, revisions.get(refresh) + 1); return { status: 0 }; } - if (normalized.includes("provider get")) return { status: 1 }; - return createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const sandboxCapture = createdSandbox.capture(command); - if (sandboxCapture !== null) return sandboxCapture; - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { registered = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "compatible-endpoint", - model: "custom/model", - getSandbox: registry.getSandbox, - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.unref = () => {}; child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); const attachedProviders = [...command.matchAll(/--provider ([^ ]+)/g)].map((match) => match[1]); - commands.push({ command, providerRevisions: command.includes("sandbox create") ? Object.fromEntries(attachedProviders.map((name) => [name, revisions.get(name)])) : null, rawCredentialInEnv: Object.values(args[2]?.env || {}).includes(rawGatewayCredential) }); - process.nextTick(() => { child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); child.emit("close", 0); }); - return child; -}; -const { createSandbox } = require(${onboardPath}); -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_EXTRA_PLACEHOLDER_KEYS = "TELEGRAM_BOT_TOKEN_AGENT_A,TELEGRAM_BOT_TOKEN_AGENT_B,GITHUB_TOKEN"; - process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["slack", "telegram", "whatsapp"])})).toString("base64"); - Object.values(credentialKeys).forEach((key) => delete process.env[key]); delete process.env.GITHUB_TOKEN; - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], null, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands, registered })); -})().catch((error) => { const temporaryCreateSources = require("node:fs").readdirSync(process.env.TMPDIR).filter((entry) => entry.startsWith("nemoclaw-initial-policy-") || entry.startsWith("nemoclaw-build-")); console.log(JSON.stringify({ commands, registered, error: String(error), providerRevisions: Object.fromEntries(revisions), temporaryCreateSources })); console.error(error); process.exit(1); }); -`; - fs.writeFileSync(scriptPath, script); - const runScenario = (failedProvider?: string) => - spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - TMPDIR: tmpDir, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_FAIL_PROVIDER: failedProvider || "", - ...Object.fromEntries( - [...Object.values(providerCredentialKeys).flat(), "GITHUB_TOKEN"].map((key) => [ - key, - "", - ]), - ), - }, - }); - const result = runScenario(); - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - const commands = payload.commands as CommandEntry[]; - const createIndex = commands.findIndex(({ command }) => command.includes("sandbox create")); - assert.notEqual(createIndex, -1, "expected sandbox create command"); - const createCommand = commands[createIndex]; - const providerRefreshes = commands - .map((entry, index) => ({ entry, index })) - .filter(({ entry }) => /\bprovider update -g nemoclaw ([^ ]+)$/.test(entry.command)); - const providerName = (command: string) => - command.match(/\bprovider update -g nemoclaw ([^ ]+)$/)?.[1]; - const refreshedProviders = providerRefreshes - .map(({ entry }: { entry: CommandEntry }) => providerName(entry.command)) - .sort(); - const denied = runScenario("my-assistant-telegram-bridge"); - assert.equal(denied.status, 1); - const deniedPayload = parseStdoutJson(denied.stdout); - const deniedCommands = (deniedPayload.commands as CommandEntry[]).map( - ({ command }) => command, - ); - const deniedRefreshes = deniedCommands.map(providerName).filter(Boolean); - const publishedBeforeCreate = providerRefreshes.every( - ({ entry, index }) => index < createIndex && !entry.command.includes("--credential"), - ); - const extraPlaceholderKeys = createCommand.command - .match(/NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=([^ ]+)/)?.[1] - ?.split(",") - .sort(); - const registeredChannels = payload.registered?.messaging?.plan?.channels.map( - (channel: { channelId: string }) => channel.channelId, - ); - assert.deepEqual(refreshedProviders, expectedProviders); - assert.equal( - commands.some(({ command }) => command.includes("provider create")), - false, - ); - assert.equal(publishedBeforeCreate, true); - assert.deepEqual( - [...createCommand.command.matchAll(/--provider ([^ ]+)/g)].map((match) => match[1]).sort(), - expectedProviders, - ); - assert.deepEqual( - createCommand.providerRevisions, - Object.fromEntries(expectedProviders.map((provider) => [provider, 2])), - ); - assert.deepEqual(extraPlaceholderKeys, [ - "TELEGRAM_BOT_TOKEN_AGENT_A", - "TELEGRAM_BOT_TOKEN_AGENT_B", - ]); - assert.equal(createCommand.command.includes("GITHUB_TOKEN"), false); - assert.equal(createCommand.rawCredentialInEnv, false); - assert.deepEqual(registeredChannels, ["slack", "telegram", "whatsapp"]); - assert.deepEqual(deniedRefreshes, []); - assert.equal( - Object.values(deniedPayload.providerRevisions).filter((revision) => revision === 2).length, - 0, - ); - assert.ok(deniedCommands.every((command) => !command.includes("sandbox create"))); - assert.equal(deniedPayload.registered, null); - assert.deepEqual(deniedPayload.temporaryCreateSources, []); - assert.match( - deniedPayload.error, - /Could not inspect messaging provider 'my-assistant-telegram-bridge': OpenShell could not inspect the provider/, - ); - const combinedOutput = result.stdout + result.stderr + denied.stdout + denied.stderr; - assert.equal( - (JSON.stringify([payload, deniedPayload]) + combinedOutput).includes(rawGatewayCredential), - false, - ); - }, - ); - - it( - "preserves disabled channels in a custom image after a recreate so `channels start` can re-enable them (#3381)", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-disabled-channels-preserve-"), - ); - const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "disabled-channels-preserve.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const messagingPlanB64 = encodeMessagingPlanForChannels(["telegram"], ["telegram"]); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); -const commands = []; let dockerfileContent; -const registerCalls = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture({ - sandboxName: "my-assistant", -}); createdSandbox.installRuntimeObservation(); -registry.registerSandbox({ - name: "my-assistant", - messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["telegram"], ["telegram"])} }, -}); -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - if (normalized.includes("provider get -g nemoclaw my-assistant-telegram-bridge")) return { status: 0, stdout: "Name: my-assistant-telegram-bridge\nType: nemoclaw-mcp-v1\nCredential keys: TELEGRAM_BOT_TOKEN\nConfig keys: \n" }; - const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; - return createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const sandboxCapture = createdSandbox.capture(command); - if (sandboxCapture !== null) return sandboxCapture; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", - getSandbox: registry.getSandbox, - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); - if (dockerfileMatch) { - try { - entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); - } catch (error) { - entry.dockerfileReadError = String(error); - } - } - commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - delete process.env.TELEGRAM_BOT_TOKEN; - process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["telegram"], ["telegram"])})).toString("base64"); - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands, registerCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - const result = runBoundedOnboardScript(scriptPath, { - cwd: repoRoot, - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_SANDBOX_GPU: "0", - NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, - TELEGRAM_BOT_TOKEN: "", - }, - }); - assert.equal(result.status, 0, result.stderr || result.error?.message); - const payload = parseStdoutJson(result.stdout); - - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.equal(createCommand.dockerfileReadError, undefined); - - assert.deepEqual( - activeChannelsFromDockerfile(createCommand.dockerfileContent), - [], - "disabled channel must not be active in the image plan", - ); - assert.doesNotMatch( - createCommand.command, - /--provider my-assistant-telegram-bridge/, - "disabled channel's bridge must not be attached to the new sandbox", - ); - - const registeredPlan = payload.registerCalls[0]?.messaging?.plan; - assert.deepEqual( - registeredPlan?.channels.map((channel: { channelId: string }) => channel.channelId), - ["telegram"], - "registry.messaging.plan must keep the disabled-but-configured channel so `channels start` can recover it", - ); - assert.deepEqual( - registeredPlan?.disabledChannels, - ["telegram"], - "registry.messaging.plan.disabledChannels must round-trip through the rebuild", - ); - }, - ); - - it( - "bakes WhatsApp into a custom sandbox image without bridge providers when no messaging tokens are set", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-tokenless-whatsapp-")); - try { - const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "tokenless-whatsapp.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"]); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); - -const commands = []; let dockerfileContent; -const registerCalls = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; - return createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); - if (dockerfileMatch) { - try { - entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); - } catch (error) { - entry.dockerfileReadError = String(error); - } - } - commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - for (const key of Object.keys(process.env)) { - if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { - delete process.env[key]; - } - } - process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands, registerCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - const providerMutationCommands = payload.commands.filter( - (entry: CommandEntry) => - /\bprovider (create|update)\b/.test(entry.command) && entry.command.includes("-bridge"), - ); - assert.equal( - providerMutationCommands.length, - 0, - "QR-only channel selection must not create bridge providers", - ); - - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.equal(createCommand.dockerfileReadError, undefined); - assert.doesNotMatch(createCommand.command, /--provider \S+-bridge\b/); - - assert.deepEqual(activeChannelsFromDockerfile(createCommand.dockerfileContent), [ - "whatsapp", - ]); - assert.deepEqual( - payload.registerCalls[0]?.messaging?.plan?.channels.map( - (channel: { channelId: string }) => channel.channelId, - ), - ["whatsapp"], - ); - assert.equal(payload.registerCalls[0]?.messagingChannels, undefined); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); - - it( - "drops WhatsApp from a rebuilt custom image when the registry marks it disabled", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-disabled-whatsapp-")); - try { - const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "disabled-whatsapp.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"], ["whatsapp"]); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); - -registry.registerSandbox({ - name: "my-assistant", - messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["whatsapp"], ["whatsapp"])} }, -}); - -const commands = []; let dockerfileContent; -const registerCalls = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; - return createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", - getSandbox: registry.getSandbox, - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); - if (dockerfileMatch) { - try { - entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); - } catch (error) { - entry.dockerfileReadError = String(error); - } - } - commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - for (const key of Object.keys(process.env)) { - if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { - delete process.env[key]; - } - } - process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands, registerCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.equal(createCommand.dockerfileReadError, undefined); - - assert.deepEqual( - activeChannelsFromDockerfile(createCommand.dockerfileContent), - [], - "disabled QR channel must not be active in the image plan", - ); - const registeredPlan = payload.registerCalls[0]?.messaging?.plan; - assert.deepEqual( - registeredPlan?.channels.map((channel: { channelId: string }) => channel.channelId), - ["whatsapp"], - "registry.messaging.plan must keep the disabled QR channel so `channels start` can recover it (mirrors #3381)", - ); - assert.deepEqual( - registeredPlan?.disabledChannels, - ["whatsapp"], - "registry.messaging.plan.disabledChannels must round-trip through the rebuild", - ); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); - - it("aborts onboard when a messaging provider upsert fails", { timeout: 60_000 }, async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-provider-fail-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "provider-upsert-fail.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); - -runner.run = (command, opts = {}) => { - // Fail all provider create and update calls - if (_n(command).includes("provider")) { - return { status: 1, stdout: "", stderr: "gateway unreachable" }; - } - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("sandbox get")) return ""; - if (_n(command).includes("sandbox list")) return ""; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - await createSandbox(null, "gpt-5.4", "nvidia-prod"); - console.log("ERROR_DID_NOT_EXIT"); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.notEqual(result.status, 0, "expected non-zero exit when provider upsert fails"); - assert.ok( - !result.stdout.includes("ERROR_DID_NOT_EXIT"), - "onboard should have aborted before reaching sandbox create", - ); - }); - - it( - "reuses sandbox without refreshing unselected ambient messaging providers (#10277)", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-reuse-providers-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "reuse-with-providers.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); - -const commands = []; -const existingSandbox = fixtureMocks.createCreatedSandboxFixture({ lifecycleState: "created" }); existingSandbox.installRuntimeObservation(); -const messagingProviderRunner = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ - commands, - createdSandbox: existingSandbox, - initialProviders: [ - ["my-assistant-discord-bridge", "nemoclaw-mcp-v1", "DISCORD_BOT_TOKEN"], - ["my-assistant-slack-bridge", "nemoclaw-mcp-v1", "SLACK_BOT_TOKEN"], - ["my-assistant-slack-app", "nemoclaw-mcp-v1", "SLACK_APP_TOKEN"], - ], -}); -runner.run = messagingProviderRunner; -runner.runCapture = (command) => { - const sandboxCapture = existingSandbox.capture(command); - if (sandboxCapture !== null) return sandboxCapture; - // All messaging providers already exist in gateway - if (_n(command).includes("provider get")) return "Provider: exists"; - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; -registry.getSandbox = () => fixtureMocks.sandboxLifecycleFixture( - { name: "my-assistant", toolDisclosure: "progressive" }, - { sandboxId: existingSandbox.state.sandboxId }, -); -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.DISCORD_BOT_TOKEN = "test-discord-token"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; - process.env.SLACK_APP_TOKEN = "xapp-test-slack-token"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); - console.log(JSON.stringify({ sandboxName, commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - assert.equal(payload.sandboxName, "my-assistant", "should reuse existing sandbox"); - assert.ok( - payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox create")), - "should NOT recreate sandbox when providers already exist in gateway", - ); - assert.ok( - payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox delete")), - "should NOT delete sandbox when providers already exist in gateway", - ); - - // Existing gateway providers do not select messaging for this onboarding request. - const providerUpserts = payload.commands.filter((entry: CommandEntry) => - entry.command.includes("provider update"), - ); - assert.equal( - providerUpserts.length, - 0, - "should not refresh ambient messaging providers without a selected channel plan", - ); - }, - ); - - it( - "filters messaging providers to only enabledChannels when provided", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-filter-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "enabled-channels-filter.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); - -const commands = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ - commands, - createdSandbox, -}); -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; - process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - // Only enable telegram — discord and slack should be filtered out - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], null, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - // Only telegram provider should be created - const providerCommands = payload.commands.filter((e: CommandEntry) => - e.command.includes("provider create"), - ); - const telegramProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-telegram-bridge"), - ); - assert.ok(telegramProvider, "expected telegram provider to be created"); - - // Discord and slack providers should NOT be created - const discordProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-discord-bridge"), - ); - assert.ok(!discordProvider, "discord provider should be filtered out"); - - const slackProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-slack-bridge"), - ); - assert.ok(!slackProvider, "slack provider should be filtered out"); - - // Sandbox create should only have the telegram --provider flag - const createCommand = payload.commands.find((e: CommandEntry) => - e.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.match(createCommand.command, /--provider my-assistant-telegram-bridge/); - assert.doesNotMatch(createCommand.command, /my-assistant-discord-bridge/); - assert.doesNotMatch(createCommand.command, /my-assistant-slack-bridge/); - }, - ); - - it( - "does not create messaging providers from ambient credentials without a selected plan (#10277)", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-empty-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "enabled-channels-empty.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); - -const commands = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw") ?? createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; - process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - // No selected messaging plan — ambient repository credentials are unrelated to this request. - const sandboxName = await createSandbox( - ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( - [ - null, - "gpt-5.4", - "nvidia-prod", - null, - "my-assistant", - null, - null, - null, - null, - null, - null, - null, - [], - ], - createFixture, - ), - ); - console.log(JSON.stringify({ sandboxName, commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - // No messaging providers should be created at all - const providerCommands = payload.commands.filter((e: CommandEntry) => - e.command.includes("provider create"), - ); - assert.equal( - providerCommands.length, - 0, - "no providers should be created without a selected messaging plan", - ); - - // Sandbox create should have no --provider flags for messaging bridges - const createCommand = payload.commands.find((e: CommandEntry) => - e.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.doesNotMatch(createCommand.command, /discord-bridge/); - assert.doesNotMatch(createCommand.command, /slack-bridge/); - assert.doesNotMatch(createCommand.command, /telegram-bridge/); - }, - ); - - it( - "non-interactive setupMessagingChannels returns channels with tokens", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-noninteractive-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-noninteractive.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -// Stub the manifest-driven Telegram reachability hook so this test does not -// make a real network call. -global.fetch = async () => ({ - ok: true, - status: 200, - json: async () => ({ ok: true, result: { id: 1, is_bot: true } }), - text: async () => "", -}); - -const { setupMessagingChannels } = require(${onboardPath}); - -(async () => { - // Only set telegram and slack tokens — discord should be absent - process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; - process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token"; - process.env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; - const result = await setupMessagingChannels(); - console.log(JSON.stringify(result)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const channels = parseStdoutJson(result.stdout); - - // Should return only the channels that have tokens set - assert.ok(Array.isArray(channels), "expected an array return value"); - assert.ok(channels.includes("telegram"), "expected telegram in returned channels"); - assert.ok(channels.includes("slack"), "expected slack in returned channels"); - assert.ok(!channels.includes("discord"), "discord should not be in returned channels"); - }, - ); - - it( - "non-interactive setupMessagingChannels drops Slack when live Slack API validation rejects the token", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-slack-live-reject-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-slack-live-reject.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const httpProbePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "adapters", "http", "probe.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const httpProbe = require(${httpProbePath}); -httpProbe.runCurlProbe = (argv) => { - const url = argv[argv.length - 1] || ""; - if (String(url).includes("auth.test")) { - return { - ok: true, - httpStatus: 200, - curlStatus: 0, - body: '{"ok":false,"error":"invalid_auth"}', - stderr: "", - message: "", - }; - } - return { - ok: true, - httpStatus: 200, - curlStatus: 0, - body: '{"ok":true}', - stderr: "", - message: "", - }; -}; - -const { setupMessagingChannels } = require(${onboardPath}); - -(async () => { - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - process.env.SLACK_BOT_TOKEN = "xoxb-fake-bot-token"; - process.env.SLACK_APP_TOKEN = "xapp-fake-app-token"; - const result = await setupMessagingChannels(); - console.log(JSON.stringify(result)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const channels = parseStdoutJson(result.stdout); - - assert.ok(Array.isArray(channels), "expected an array return value"); - assert.ok(!channels.includes("slack"), "Slack should be dropped after API rejection"); - assert.doesNotMatch(result.stdout, /xoxb-fake-bot-token/); - assert.doesNotMatch(result.stderr, /xoxb-fake-bot-token/); - }, - ); - - it( - "non-interactive setupMessagingChannels returns empty array when no tokens set", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-no-tokens-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-no-tokens.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const { setupMessagingChannels } = require(${onboardPath}); - -(async () => { - // No messaging tokens set - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - delete process.env.SLACK_BOT_TOKEN; - delete process.env.SLACK_APP_TOKEN; - const result = await setupMessagingChannels(); - console.log(JSON.stringify(result)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - TELEGRAM_BOT_TOKEN: "", - DISCORD_BOT_TOKEN: "", - SLACK_BOT_TOKEN: "", - SLACK_APP_TOKEN: "", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const channels = parseStdoutJson(result.stdout); - - assert.ok(Array.isArray(channels), "expected an array return value"); - assert.equal(channels.length, 0, "expected empty array when no tokens are set"); - }, - ); - - it( - "interactive setupMessagingChannels drops slack when prompted token fails tokenFormat check (#1912)", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-slack-format-reject-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "slack-format-reject.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - // Subscript: mocks credentials.prompt to return a bogus Slack token, - // exposes MESSAGING_CHANNELS so the parent can look up the Slack toggle - // digit, and asserts that setupMessagingChannels rejects the invalid - // token without persisting it. Slack is the 3rd channel in insertion - // order today (telegram, discord, slack) but we compute the index - // dynamically to avoid a brittle coupling to that ordering. - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const saveCalls = []; -credentials.saveCredential = (key, value) => { saveCalls.push({ key, value }); }; -credentials.getCredential = () => null; -credentials.prompt = async (message) => { - if (message.includes("Slack Bot Token")) return "abcd"; - return ""; -}; - -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); - -(async () => { - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - delete process.env.SLACK_BOT_TOKEN; - delete process.env.SLACK_APP_TOKEN; - - const result = await setupMessagingChannels(); - console.log(JSON.stringify({ - result, - saveCalls, - slackIndex1Based: MESSAGING_CHANNELS.findIndex((c) => c.name === "slack") + 1, - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - // Dry run with just Enter — no toggles, empty result — used to read back - // Slack's 1-based index from the same subscript so the real run can - // press the right digit. - const introspect = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - input: "\n", - }); - assert.equal(introspect.status, 0, introspect.stderr); - const introspectOut = JSON.parse(introspect.stdout.trim().split("\n").pop()!); - const slackIdx = introspectOut.slackIndex1Based; - assert.ok(slackIdx >= 1, `unexpected slack index: ${slackIdx}`); - - // Real run: press Slack's digit, Enter. Slack gets toggled on, prompt - // fires, mocked prompt returns "abcd", tokenFormat regex rejects it, - // channel is dropped, saveCredential never runs for SLACK_BOT_TOKEN. - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - input: `${slackIdx}\n`, - }); - - assert.equal(result.status, 0, result.stderr); - const out = JSON.parse(result.stdout.trim().split("\n").pop()!); - - assert.ok( - !out.result.includes("slack"), - `slack should have been dropped after invalid token; got ${JSON.stringify(out.result)}`, - ); - assert.ok( - !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_BOT_TOKEN"), - `SLACK_BOT_TOKEN should NOT have been persisted; saveCalls=${JSON.stringify(out.saveCalls)}`, - ); - assert.ok( - result.stderr.includes("Invalid format") || result.stdout.includes("Invalid format"), - `expected 'Invalid format' warning; stderr=${result.stderr} stdout=${result.stdout}`, - ); - }, - ); -}); diff --git a/test/onboarding/onboard-sandbox-build-base.test.ts b/test/onboarding/onboard-sandbox-build-base.test.ts new file mode 100644 index 00000000000..8c893699c58 --- /dev/null +++ b/test/onboarding/onboard-sandbox-build-base.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerOnboardSandboxBuildTests } from "../helpers/onboard-sandbox-build-suite"; + +registerOnboardSandboxBuildTests("base"); diff --git a/test/onboarding/onboard-sandbox-build-dashboard.test.ts b/test/onboarding/onboard-sandbox-build-dashboard.test.ts new file mode 100644 index 00000000000..e0c11ee0c42 --- /dev/null +++ b/test/onboarding/onboard-sandbox-build-dashboard.test.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { registerOnboardSandboxBuildTests } from "../helpers/onboard-sandbox-build-suite"; + +registerOnboardSandboxBuildTests("dashboard"); diff --git a/test/repository/cli-coverage-sequencer.test.ts b/test/repository/cli-coverage-sequencer.test.ts index 4240b9b1aae..8049976f0e6 100644 --- a/test/repository/cli-coverage-sequencer.test.ts +++ b/test/repository/cli-coverage-sequencer.test.ts @@ -211,6 +211,12 @@ describe("stable CLI coverage sharding", () => { headSha: "5b74336486a8492ad87dd7eaba3d1bc06abbadf6", recordedAt: "2026-09-04T23:53:49Z", }, + { + runId: 34013605812, + artifactId: 9983530478, + headSha: "be1452092af48f439531502870e826215b0d76f9", + recordedAt: "2026-09-06T05:43:35Z", + }, ]); expect(files).toEqual([...files].sort()); expect(files.length).toBeGreaterThan(50); diff --git a/test/repository/cli-test-timing-drift.test.ts b/test/repository/cli-test-timing-drift.test.ts new file mode 100644 index 00000000000..e7faa1c17b9 --- /dev/null +++ b/test/repository/cli-test-timing-drift.test.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + findCliTestTimingDrift, + formatCliTestTimingDriftSummary, +} from "../../scripts/checks/report-cli-test-timing-drift.mts"; + +const repoRoot = path.resolve("/workspace/repository"); + +function report(files: Record): object { + return { + testResults: Object.entries(files).map(([file, duration]) => ({ + name: path.join(repoRoot, file), + startTime: 1_000, + endTime: 1_000 + duration, + })), + }; +} + +const hints = { + defaultDurationMs: 5_000, + files: { + "test/faster.test.ts": 50_000, + "test/slower.test.ts": 10_000, + "test/stable.test.ts": 20_000, + }, +}; + +describe("CLI test timing drift", () => { + it("finds material slow, fast, and unprofiled files without flagging normal noise", () => { + expect( + findCliTestTimingDrift( + report({ + "test/faster.test.ts": 20_000, + "test/slower.test.ts": 25_000, + "test/stable.test.ts": 25_000, + "test/unprofiled.test.ts": 15_000, + "test/ordinary.test.ts": 14_999, + }), + hints, + repoRoot, + ), + ).toEqual([ + { file: "test/slower.test.ts", hintMs: 10_000, kind: "slower", observedMs: 25_000 }, + { file: "test/faster.test.ts", hintMs: 50_000, kind: "faster", observedMs: 20_000 }, + { file: "test/unprofiled.test.ts", kind: "unprofiled", observedMs: 15_000 }, + ]); + }); + + it("ignores report entries outside the repository and keeps the largest duplicate duration", () => { + const value = report({ "test/slower.test.ts": 22_000 }) as { + testResults: Array<{ name: string; startTime: number; endTime: number }>; + }; + value.testResults.push( + { name: path.join(repoRoot, "test/slower.test.ts"), startTime: 1_000, endTime: 31_000 }, + { name: "/outside/repository.test.ts", startTime: 1_000, endTime: 101_000 }, + ); + + expect(findCliTestTimingDrift(value, hints, repoRoot)).toEqual([ + { file: "test/slower.test.ts", hintMs: 10_000, kind: "slower", observedMs: 30_000 }, + ]); + }); + + it("renders a concise advisory summary", () => { + const summary = formatCliTestTimingDriftSummary([ + { file: "test/new.test.ts", kind: "unprofiled", observedMs: 16_000 }, + ]); + + expect(summary).toContain("1 file(s) need a timing-hint refresh"); + expect(summary).toContain("`test/new.test.ts`"); + expect(summary).toContain("16.0s"); + expect(formatCliTestTimingDriftSummary([])).toContain("No material timing-hint drift"); + }); +}); diff --git a/test/repository/vitest-watch-triggers.test.ts b/test/repository/vitest-watch-triggers.test.ts index 6796c5bee64..13841e2801a 100644 --- a/test/repository/vitest-watch-triggers.test.ts +++ b/test/repository/vitest-watch-triggers.test.ts @@ -124,15 +124,19 @@ describe("Vitest opaque-input watch triggers", () => { "test/onboarding/onboard-custom-dockerfile.test.ts", "test/onboarding/onboard-extra-provider-reconciliation.test.ts", "test/onboarding/onboard-fresh-create-provider-identity.test.ts", - "test/onboarding/onboard-fresh-create-recovery-identity.test.ts", + "test/onboarding/onboard-fresh-create-recovery-resume-identity.test.ts", + "test/onboarding/onboard-fresh-create-recovery-registration-identity.test.ts", "test/onboarding/onboard-fresh-create-cancellation-identity.test.ts", "test/onboarding/onboard-installer-restore-intent.test.ts", "test/onboarding/onboard-managed-image-buildless-e2e.test.ts", "test/onboarding/onboard-mcp-observability-redirect.test.ts", - "test/onboarding/onboard-messaging.test.ts", + "test/onboarding/onboard-messaging-image-configuration.test.ts", + "test/onboarding/onboard-messaging-provider.test.ts", + "test/onboarding/onboard-messaging-recreation.test.ts", "test/onboarding/onboard-prepared-build-context.test.ts", "test/onboarding/onboard-reservation-recreate.test.ts", - "test/onboarding/onboard-sandbox-build.test.ts", + "test/onboarding/onboard-sandbox-build-base.test.ts", + "test/onboarding/onboard-sandbox-build-dashboard.test.ts", "test/onboarding/onboard-sandbox-recreation.test.ts", "test/onboarding/onboard-script-mocks-contract.test.ts", "test/onboarding/onboard-terminal-dashboard.test.ts", From 804ce7b3e18a18e72011b2dd2439be77b709e192 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 8 Sep 2026 16:39:52 -0700 Subject: [PATCH 10/20] test(ci): avoid credential-shaped redaction literal Signed-off-by: Charan Jagwani --- test/helpers/classify-ci-failure-suite.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/helpers/classify-ci-failure-suite.ts b/test/helpers/classify-ci-failure-suite.ts index 1326b420662..4c5731178c8 100644 --- a/test/helpers/classify-ci-failure-suite.ts +++ b/test/helpers/classify-ci-failure-suite.ts @@ -27,6 +27,7 @@ const script = resolve( ); const roots: string[] = []; const uid = process.getuid?.() ?? "unknown"; +const OPENAI_API_KEY = ["sk", "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN"].join("-"); const REDACTION_CASES = [ [ "Slack bot token", @@ -38,11 +39,7 @@ const REDACTION_CASES = [ ["xapp", "1", "A1234567890", "1234567890123", "abcdefghijklmnopqrstuvwx"].join("-"), ["xapp", "1", "A1234567890", "1234567890123", "abcdefghijklmnopqrstuvwx"].join("-"), ], - [ - "OpenAI API key", - "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", - "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", - ], + ["OpenAI API key", OPENAI_API_KEY, OPENAI_API_KEY], [ "OpenAI project API key", "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMN", From 42946d4f4c0b5e21bf94b3cc5d0a472a14f30a57 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 8 Sep 2026 17:05:32 -0700 Subject: [PATCH 11/20] perf(ci): rebalance serialized coverage lane Signed-off-by: Charan Jagwani --- test/helpers/cli-coverage-sequencer.ts | 2 +- test/repository/cli-coverage-sequencer.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index 6ecef5d403d..4f57cf15079 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -42,7 +42,7 @@ const cliCoverageProjects = new Set(["cli", "integration", "e2e-support"]); // Integration coverage is serialized, so it needs an independent salt instead // of relying on combined weight from the parallel CLI and E2E-support lanes. const stableShardSalt = "43695"; -const integrationShardSalt = "45487"; +const integrationShardSalt = "78136"; const e2eSupportShardSalt = "41846"; // Only measured outliers are stored; new and ordinary files share the // conservative fallback used to estimate each stable shard's load. diff --git a/test/repository/cli-coverage-sequencer.test.ts b/test/repository/cli-coverage-sequencer.test.ts index 8049976f0e6..49fec2ff6bf 100644 --- a/test/repository/cli-coverage-sequencer.test.ts +++ b/test/repository/cli-coverage-sequencer.test.ts @@ -117,9 +117,9 @@ describe("stable CLI coverage sharding", () => { expect(Object.fromEntries(owners)).toEqual({ "cli:src/lib/example.test.ts": 1, "e2e-support:test/e2e/support/example.test.ts": 8, - "integration:test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts": 2, - "integration:test/credentials/local-credential-helper-fields.test.ts": 6, - "integration:test/regular-0.test.ts": 6, + "integration:test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts": 8, + "integration:test/credentials/local-credential-helper-fields.test.ts": 7, + "integration:test/regular-0.test.ts": 8, }); }); From 5c401c6593e090b418f79a7207ff3ae2d074dcd4 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 8 Sep 2026 17:25:43 -0700 Subject: [PATCH 12/20] perf(ci): retain twelve coverage shards Signed-off-by: Charan Jagwani --- .github/workflows/main.yaml | 10 +++++----- .github/workflows/pr.yaml | 10 +++++----- .../pull-requests/pr-workflow-contract.test.ts | 4 ++-- test/helpers/cli-coverage-sequencer.ts | 6 +++--- test/repository/cli-coverage-sequencer.test.ts | 18 +++++++++--------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9c54b9a1486..4aa2c944f4c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -193,7 +193,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -219,7 +219,7 @@ jobs: uses: ./.github/actions/ci-cli-coverage-shard with: shard: ${{ matrix.shard }} - shard-count: "10" + shard-count: "12" cli-tests: needs: [build-typecheck, cli-test-shards] @@ -258,10 +258,10 @@ jobs: else [.jobs[] | select((.name | type) == "string") | - select(.name | test("^cli-test-shards \\(([1-9]|10)\\)$")) | + select(.name | test("^cli-test-shards \\(([1-9]|1[0-2])\\)$")) | select(.conclusion != "success")] as $failed | if ($failed | length) == 0 or - ($failed | length) > 10 or + ($failed | length) > 12 or (($failed | map(.name) | unique | length) != ($failed | length)) then error("invalid failed CLI shard listing") else @@ -297,7 +297,7 @@ jobs: - name: Merge CLI coverage uses: ./.github/actions/ci-cli-coverage-merge with: - shard-count: "10" + shard-count: "12" plugin-tests: permissions: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 590c5e5ac85..1fb2ffa0a89 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -573,7 +573,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -631,7 +631,7 @@ jobs: uses: ./.trusted-ci-actions/.github/actions/ci-cli-coverage-shard with: shard: ${{ matrix.shard }} - shard-count: "10" + shard-count: "12" cli-tests: needs: @@ -673,10 +673,10 @@ jobs: else [.jobs[] | select((.name | type) == "string") | - select(.name | test("^cli-test-shards \\(([1-9]|10)\\)$")) | + select(.name | test("^cli-test-shards \\(([1-9]|1[0-2])\\)$")) | select(.conclusion != "success")] as $failed | if ($failed | length) == 0 or - ($failed | length) > 10 or + ($failed | length) > 12 or (($failed | map(.name) | unique | length) != ($failed | length)) then error("invalid failed CLI shard listing") else @@ -744,7 +744,7 @@ jobs: - name: Merge CLI coverage uses: ./.trusted-ci-actions/.github/actions/ci-cli-coverage-merge with: - shard-count: "10" + shard-count: "12" plugin-tests: needs: [changes, openshell-sdk-package] diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index c194f58b57f..839d275fd7c 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -91,7 +91,7 @@ const trustedActionDirs = [ ".github/actions/ci-installer-integration", ] as const; -const cliShardCount = "10"; +const cliShardCount = "12"; const cliShardTimeoutMinutes = 30; const dependencyInstallJobs = [ "build-typecheck", @@ -1067,7 +1067,7 @@ describe("pull request and main workflow contracts", () => { GITHUB_OUTPUT: output, }); const invalidRange = runWorkflowShellStep(shardValidationStep, { - CLI_SHARD: "11", + CLI_SHARD: String(Number(cliShardCount) + 1), CLI_SHARD_COUNT: cliShardCount, GITHUB_OUTPUT: join(temp, "github-output"), }); diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index 4f57cf15079..50213ea397c 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -41,9 +41,9 @@ const cliCoverageProjects = new Set(["cli", "integration", "e2e-support"]); // ordinary roster changes preserve ownership between profile refreshes. // Integration coverage is serialized, so it needs an independent salt instead // of relying on combined weight from the parallel CLI and E2E-support lanes. -const stableShardSalt = "43695"; -const integrationShardSalt = "78136"; -const e2eSupportShardSalt = "41846"; +const stableShardSalt = "45603"; +const integrationShardSalt = "7876"; +const e2eSupportShardSalt = "96995"; // Only measured outliers are stored; new and ordinary files share the // conservative fallback used to estimate each stable shard's load. const timingHintsUrl = new URL("../../ci/cli-test-timing-hints.json", import.meta.url); diff --git a/test/repository/cli-coverage-sequencer.test.ts b/test/repository/cli-coverage-sequencer.test.ts index 49fec2ff6bf..60636732ad8 100644 --- a/test/repository/cli-coverage-sequencer.test.ts +++ b/test/repository/cli-coverage-sequencer.test.ts @@ -116,28 +116,28 @@ describe("stable CLI coverage sharding", () => { expect(Object.fromEntries(owners)).toEqual({ "cli:src/lib/example.test.ts": 1, - "e2e-support:test/e2e/support/example.test.ts": 8, - "integration:test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts": 8, - "integration:test/credentials/local-credential-helper-fields.test.ts": 7, - "integration:test/regular-0.test.ts": 8, + "e2e-support:test/e2e/support/example.test.ts": 7, + "integration:test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts": 1, + "integration:test/credentials/local-credential-helper-fields.test.ts": 1, + "integration:test/regular-0.test.ts": 5, }); }); - it("keeps the current test roster balanced across the ten CI shards (#6237)", () => { - const shards = assignStableShards(currentCliCoverageEntries(), 10); + it("keeps the current test roster balanced across the twelve CI shards (#6237)", () => { + const shards = assignStableShards(currentCliCoverageEntries(), 12); const weights = shards.map((shard) => shard.totalWeightMs); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; - expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.06); + expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.05); }); - it("balances the serialized integration lane across the ten CI shards (#6237)", () => { + it("balances the serialized integration lane across the twelve CI shards (#6237)", () => { const integrationEntries = currentCliCoverageEntries().filter((entry) => entry.key.startsWith("integration:"), ); expect(integrationEntries.length).toBeGreaterThan(0); - const weights = assignStableShards(integrationEntries, 10).map( + const weights = assignStableShards(integrationEntries, 12).map( (shard) => shard.totalWeightMs, ); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; From acffdc4f778a7db149613224f3e846c77cadca54 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 8 Sep 2026 19:03:48 -0700 Subject: [PATCH 13/20] test(ci): isolate ARM64 GPU prover fixtures Signed-off-by: Charan Jagwani --- src/lib/onboard/wsl-docker-desktop-gpu.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/onboard/wsl-docker-desktop-gpu.test.ts b/src/lib/onboard/wsl-docker-desktop-gpu.test.ts index 99f31552fac..4d666a0458b 100644 --- a/src/lib/onboard/wsl-docker-desktop-gpu.test.ts +++ b/src/lib/onboard/wsl-docker-desktop-gpu.test.ts @@ -173,6 +173,7 @@ describe("createArm64ContainerGpuProver (#4565)", () => { platform: "linux", arch: "arm64", env: {}, + resolveRuntimeProvider: () => proofProvider("docker"), runProof, log: () => undefined, }); @@ -195,6 +196,7 @@ describe("createArm64ContainerGpuProver (#4565)", () => { platform: "linux", arch: "arm64", env: {}, + resolveRuntimeProvider: () => proofProvider("docker"), runProof, log: () => undefined, }); From 3a3fbfe85eca22e38f86db4474cbe16326530bfe Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 8 Sep 2026 19:23:04 -0700 Subject: [PATCH 14/20] test(ci): remove hidden shard setup Signed-off-by: Charan Jagwani --- .../onboard/wsl-docker-desktop-gpu.test.ts | 3 + test/credentials/credential-rotation.test.ts | 67 ++----------------- 2 files changed, 7 insertions(+), 63 deletions(-) diff --git a/src/lib/onboard/wsl-docker-desktop-gpu.test.ts b/src/lib/onboard/wsl-docker-desktop-gpu.test.ts index 4d666a0458b..5204d06b4e2 100644 --- a/src/lib/onboard/wsl-docker-desktop-gpu.test.ts +++ b/src/lib/onboard/wsl-docker-desktop-gpu.test.ts @@ -711,6 +711,7 @@ describe("createArm64ContainerGpuProver (#4565)", () => { platform: "linux", arch: "arm64", env: {}, + resolveRuntimeProvider: () => proofProvider("docker"), runProof: () => passingProof, log: (message) => logs.push(message), }); @@ -745,6 +746,7 @@ describe("createArm64ContainerGpuProver (#4565)", () => { const prover = createArm64ContainerGpuProver({ platform: "linux", arch: "arm64", + resolveRuntimeProvider: () => proofProvider("docker"), runProof: () => failing, log: () => undefined, }); @@ -763,6 +765,7 @@ describe("createArm64ContainerGpuProver (#4565)", () => { const prover = createArm64ContainerGpuProver({ platform: "linux", arch: "arm64", + resolveRuntimeProvider: () => proofProvider("docker"), runProof: () => execFormatFailure, log: (message) => logs.push(message), }); diff --git a/test/credentials/credential-rotation.test.ts b/test/credentials/credential-rotation.test.ts index 1344b88f019..e76830e794a 100644 --- a/test/credentials/credential-rotation.test.ts +++ b/test/credentials/credential-rotation.test.ts @@ -1,71 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createRequire } from "node:module"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const require = createRequire(import.meta.url); - -type ModuleProperty = string | number | boolean | Function | object | null | undefined; -type ModuleRecord = { [key: string]: ModuleProperty }; - -type MessagingProvider = { - name: string; - envKey: string; - token: string | null; -}; - -type CredentialRotationInternals = { - hashCredential: (value: string | null | undefined) => string | null; - detectMessagingCredentialRotation: ( - sandboxName: string, - providers: MessagingProvider[], - ) => { changed: boolean; changedProviders: string[] }; -}; - -function isObjectRecord(value: unknown): value is ModuleRecord { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function isCredentialRotationInternals(value: unknown): value is CredentialRotationInternals { - return ( - isObjectRecord(value) && - typeof value.hashCredential === "function" && - typeof value.detectMessagingCredentialRotation === "function" - ); -} - -function isRegistryModule(value: unknown): value is typeof import("../../src/lib/state/registry.js") { - return isObjectRecord(value) && typeof value.getSandbox === "function"; -} - -function loadCredentialRotationInternals(): CredentialRotationInternals { - const loaded: unknown = require("../../src/lib/onboard.js"); - if (!isCredentialRotationInternals(loaded)) { - throw new Error("Expected onboard internals to expose credential rotation helpers"); - } - return loaded; -} - -function loadRegistryModule(): typeof import("../../src/lib/state/registry.js") { - const loaded: unknown = require("../../src/lib/state/registry.js"); - if (!isRegistryModule(loaded)) { - throw new Error("Expected registry module to expose getSandbox"); - } - return loaded; -} +import { describe, expect, it, vi } from "vitest"; +import { detectMessagingCredentialRotation } from "../../src/lib/onboard/messaging-credentials"; +import { hashCredential } from "../../src/lib/security/credential-hash"; +import * as registry from "../../src/lib/state/registry"; describe("credential rotation detection", () => { - let hashCredential: CredentialRotationInternals["hashCredential"]; - let detectMessagingCredentialRotation: CredentialRotationInternals["detectMessagingCredentialRotation"]; - let registry: typeof import("../../src/lib/state/registry.js"); - - beforeEach(() => { - // Fresh imports to avoid cross-test contamination - ({ hashCredential, detectMessagingCredentialRotation } = loadCredentialRotationInternals()); - registry = loadRegistryModule(); - }); - function hashCredentialOrThrow(value: string): string { const hash = hashCredential(value); expect(hash).not.toBeNull(); From fab7f5079741f31294f96f1925a60da9d700ebfa Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 8 Sep 2026 22:37:15 -0700 Subject: [PATCH 15/20] docs(messaging): update split onboarding test command --- src/lib/messaging/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/messaging/AGENTS.md b/src/lib/messaging/AGENTS.md index 219281eed76..4b266fcdb37 100644 --- a/src/lib/messaging/AGENTS.md +++ b/src/lib/messaging/AGENTS.md @@ -94,7 +94,7 @@ Use the narrowest test that covers the changed surface: - Hook behavior: `npx vitest run src/lib/messaging/hooks src/lib/messaging/channels//hooks` - Host/OpenShell application: `npx vitest run src/lib/messaging/applier` - Build-time render/install behavior: `npx vitest run test/runtime/messaging/messaging-build-applier.test.ts` -- Onboard/channel CLI integration: `npx vitest run test/onboarding/onboard-messaging.test.ts test/channels/channels-add-preset.test.ts src/lib/onboard/messaging-channel-setup.test.ts` +- Onboard/channel CLI integration: `npx vitest run test/onboarding/onboard-messaging-image-configuration.test.ts test/onboarding/onboard-messaging-provider.test.ts test/onboarding/onboard-messaging-recreation.test.ts test/channels/channels-add-preset.test.ts src/lib/onboard/messaging-channel-setup.test.ts` Add focused negative tests for invalid credentials, unauthorized senders, denied network access, malformed configuration, and cleanup when those behaviors are in scope. From 851d7d9ae70a43c73efe062ea7f3728dff147bab Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 8 Sep 2026 23:22:28 -0700 Subject: [PATCH 16/20] test(ci): govern extracted suite size --- .../pull-requests/growth-guardrails.test.ts | 15 + .../e2e/support/launch-agent-turn-fixtures.ts | 225 +++++ test/e2e/support/launch-agent-turn-suite.ts | 212 +--- test/helpers/growth-guardrail-checks.ts | 4 +- .../onboard-messaging-configuration-suite.ts | 876 +++++++++++++++++ test/helpers/onboard-messaging-suite.ts | 902 +----------------- ...oard-messaging-image-configuration.test.ts | 4 +- 7 files changed, 1165 insertions(+), 1073 deletions(-) create mode 100644 test/e2e/support/launch-agent-turn-fixtures.ts create mode 100644 test/helpers/onboard-messaging-configuration-suite.ts diff --git a/test/automation/pull-requests/growth-guardrails.test.ts b/test/automation/pull-requests/growth-guardrails.test.ts index 725751c3b83..fd0decacb07 100644 --- a/test/automation/pull-requests/growth-guardrails.test.ts +++ b/test/automation/pull-requests/growth-guardrails.test.ts @@ -368,6 +368,21 @@ function defineCodebaseGrowthGuardrailTestSupport(): void { ); }); + it("applies the test size budget to executable suite modules", async () => { + const diff = fixtureDiff( + [{ filename: "test/helpers/example-suite.ts", status: "added" }], + { "ci/test-file-size-budget.json": '{"defaultMaxLines":1}' }, + { + "ci/test-file-size-budget.json": '{"defaultMaxLines":1}', + "test/helpers/example-suite.ts": "import { it } from 'vitest';\nit('works', () => {});\n", + }, + ); + + expect(await testSizeViolations(diff)).toEqual([ + "test/helpers/example-suite.ts has 2 lines, above its budget 1", + ]); + }); + it("asks for a stale legacy budget to be removed when its test is deleted", async () => { const budget = '{"defaultMaxLines":1500,"legacyMaxLines":{"test/legacy.test.ts":1}}'; const diff = fixtureDiff( diff --git a/test/e2e/support/launch-agent-turn-fixtures.ts b/test/e2e/support/launch-agent-turn-fixtures.ts new file mode 100644 index 00000000000..f696e757996 --- /dev/null +++ b/test/e2e/support/launch-agent-turn-fixtures.ts @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + appendFileSync, + closeSync, + constants, + fstatSync, + fsyncSync, + ftruncateSync, + mkdtempSync, + mkdirSync, + openSync, + readFileSync, + rmSync, + type Stats, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect } from "vitest"; +import { OPENCLAW_SESSION_EVIDENCE_SCRIPT } from "../live/launch-agent-turn.ts"; + +export const PROCESS_EXIT_WAIT = new Int32Array( + new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT), +); +export type SessionRecords = Record; +export type FixtureMode = + | "cleanup-failure" + | "delayed-input-attachment" + | "delayed-recording" + | "delayed-tui-ready" + | "input-mode-timeout" + | "invalid-order" + | "late-extra" + | "nonzero" + | "nonzero-pty-cleanup-failure" + | "pty-cleanup-failure" + | "pty-cleanup-unknown-entry" + | "pty-socket-invalid" + | "pty-socket-permission" + | "pty-response-identity" + | "pty-socket-timeout" + | "pty-path-unreadable" + | "pty-termios-unavailable" + | "recording-timeout" + | "restored-canonical-timeout" + | "valid"; + +export function message(role: "assistant" | "user", content = "nonempty"): string { + return JSON.stringify({ + message: { content: [{ text: content, type: "text" }], role }, + type: "message", + }); +} + +export function emptyMessage(role: "assistant" | "user"): string { + return JSON.stringify({ message: { content: [], role }, type: "message" }); +} + +export function writeSessionRecords( + root: string, + sessions: SessionRecords, + append: boolean, + finalNewline = true, +): void { + for (const [sessionId, records] of Object.entries(sessions)) { + const filePath = join(root, `${sessionId}.jsonl`); + const body = records.length > 0 ? `${records.join("\n")}${finalNewline ? "\n" : ""}` : ""; + const writeRecords = append ? appendFileSync : writeFileSync; + writeRecords(filePath, body); + } +} + +export function withOwnedFixtureFile( + filePath: string, + flags: number, + action: (descriptor: number, stats: Stats) => T, +): T { + const descriptor = openSync(filePath, flags | constants.O_NOFOLLOW, 0o600); + try { + const stats = fstatSync(descriptor); + expect([stats.isFile(), stats.uid, stats.mode & 0o777, stats.nlink]).toEqual([ + true, + process.getuid?.(), + 0o600, + 1, + ]); + return action(descriptor, stats); + } finally { + closeSync(descriptor); + } +} + +export function runEvidenceFixture(input: { + after: SessionRecords; + afterFinalNewline?: boolean; + before?: SessionRecords; + expectedTurns: number; +}) { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-evidence-")); + const runId = randomUUID().replaceAll("-", ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + const sessionRoot = join(fixtureRoot, "sessions"); + mkdirSync(sessionRoot); + try { + writeSessionRecords(sessionRoot, input.before ?? {}, false); + const baseline = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "baseline", + sessionRoot, + baselinePath, + "", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + writeSessionRecords(sessionRoot, input.after, true, input.afterFinalNewline ?? true); + const qualification = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "qualify", + sessionRoot, + baselinePath, + String(input.expectedTurns), + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + const baselineFile = withOwnedFixtureFile( + baselinePath, + constants.O_RDONLY, + (descriptor, stats) => ({ body: readFileSync(descriptor, "utf8"), stats }), + ); + return { + baseline, + baselineKeys: Object.keys(JSON.parse(baselineFile.body)).sort(), + baselineMode: baselineFile.stats.mode & 0o777, + baselineNlink: baselineFile.stats.nlink, + baselineUid: baselineFile.stats.uid, + qualification, + }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(baselinePath, { force: true }); + rmSync(`${baselinePath}.tmp`, { force: true }); + rmSync(ptyMonitorRoot, { force: true, recursive: true }); + } +} + +export function runBaselineMutationFixture( + mutation: "invalid" | "removed" | "rewritten" | "truncated", +) { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-baseline-")); + const runId = randomUUID().replaceAll("-", ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; + const sessionRoot = join(fixtureRoot, "sessions"); + const sessionPath = join(sessionRoot, "session-a.jsonl"); + mkdirSync(sessionRoot); + writeSessionRecords(sessionRoot, { "session-a": [message("user"), message("assistant")] }, false); + try { + const baseline = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "baseline", + sessionRoot, + baselinePath, + "", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + const applyMutation: Record void> = { + invalid: () => + withOwnedFixtureFile(baselinePath, constants.O_WRONLY, (descriptor) => { + ftruncateSync(descriptor, 0); + writeFileSync(descriptor, "{}"); + fsyncSync(descriptor); + }), + removed: () => rmSync(sessionPath), + rewritten: () => + writeFileSync( + sessionPath, + readFileSync(sessionPath, "utf8").replace("nonempty", "changed!"), + ), + truncated: () => writeFileSync(sessionPath, ""), + }; + applyMutation[mutation](); + const qualification = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "qualify", + sessionRoot, + baselinePath, + "1", + ptyMonitorRoot, + runId, + ], + { encoding: "utf8" }, + ); + return { baseline, qualification }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(baselinePath, { force: true }); + rmSync(`${baselinePath}.tmp`, { force: true }); + rmSync(ptyMonitorRoot, { force: true, recursive: true }); + } +} diff --git a/test/e2e/support/launch-agent-turn-suite.ts b/test/e2e/support/launch-agent-turn-suite.ts index e8f8443316c..2b4007f6e97 100644 --- a/test/e2e/support/launch-agent-turn-suite.ts +++ b/test/e2e/support/launch-agent-turn-suite.ts @@ -4,21 +4,13 @@ import { spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { - appendFileSync, chmodSync, - closeSync, - constants, existsSync, - fstatSync, - fsyncSync, - ftruncateSync, mkdtempSync, mkdirSync, - openSync, readFileSync, readdirSync, rmSync, - type Stats, statSync, writeFileSync, } from "node:fs"; @@ -42,201 +34,15 @@ import { runOpenClawLaunchSession, runOpenClawLaunchReadinessLeaseTurns, } from "../live/launch-agent-turn.ts"; -const PROCESS_EXIT_WAIT = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); -type SessionRecords = Record; -type FixtureMode = - | "cleanup-failure" - | "delayed-input-attachment" - | "delayed-recording" - | "delayed-tui-ready" - | "input-mode-timeout" - | "invalid-order" - | "late-extra" - | "nonzero" - | "nonzero-pty-cleanup-failure" - | "pty-cleanup-failure" - | "pty-cleanup-unknown-entry" - | "pty-socket-invalid" - | "pty-socket-permission" - | "pty-response-identity" - | "pty-socket-timeout" - | "pty-path-unreadable" - | "pty-termios-unavailable" - | "recording-timeout" - | "restored-canonical-timeout" - | "valid"; - -function message(role: "assistant" | "user", content = "nonempty"): string { - return JSON.stringify({ - message: { content: [{ text: content, type: "text" }], role }, - type: "message", - }); -} - -function emptyMessage(role: "assistant" | "user"): string { - return JSON.stringify({ message: { content: [], role }, type: "message" }); -} - -function writeSessionRecords( - root: string, - sessions: SessionRecords, - append: boolean, - finalNewline = true, -): void { - for (const [sessionId, records] of Object.entries(sessions)) { - const filePath = join(root, `${sessionId}.jsonl`); - const body = records.length > 0 ? `${records.join("\n")}${finalNewline ? "\n" : ""}` : ""; - const writeRecords = append ? appendFileSync : writeFileSync; - writeRecords(filePath, body); - } -} - -function withOwnedFixtureFile( - filePath: string, - flags: number, - action: (descriptor: number, stats: Stats) => T, -): T { - const descriptor = openSync(filePath, flags | constants.O_NOFOLLOW, 0o600); - try { - const stats = fstatSync(descriptor); - expect([stats.isFile(), stats.uid, stats.mode & 0o777, stats.nlink]).toEqual([ - true, - process.getuid?.(), - 0o600, - 1, - ]); - return action(descriptor, stats); - } finally { - closeSync(descriptor); - } -} - -function runEvidenceFixture(input: { - after: SessionRecords; - afterFinalNewline?: boolean; - before?: SessionRecords; - expectedTurns: number; -}) { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-evidence-")); - const runId = randomUUID().replaceAll("-", ""); - const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - const sessionRoot = join(fixtureRoot, "sessions"); - mkdirSync(sessionRoot); - try { - writeSessionRecords(sessionRoot, input.before ?? {}, false); - const baseline = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "baseline", - sessionRoot, - baselinePath, - "", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - writeSessionRecords(sessionRoot, input.after, true, input.afterFinalNewline ?? true); - const qualification = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "qualify", - sessionRoot, - baselinePath, - String(input.expectedTurns), - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - const baselineFile = withOwnedFixtureFile( - baselinePath, - constants.O_RDONLY, - (descriptor, stats) => ({ body: readFileSync(descriptor, "utf8"), stats }), - ); - return { - baseline, - baselineKeys: Object.keys(JSON.parse(baselineFile.body)).sort(), - baselineMode: baselineFile.stats.mode & 0o777, - baselineNlink: baselineFile.stats.nlink, - baselineUid: baselineFile.stats.uid, - qualification, - }; - } finally { - rmSync(fixtureRoot, { force: true, recursive: true }); - rmSync(baselinePath, { force: true }); - rmSync(`${baselinePath}.tmp`, { force: true }); - rmSync(ptyMonitorRoot, { force: true, recursive: true }); - } -} - -function runBaselineMutationFixture(mutation: "invalid" | "removed" | "rewritten" | "truncated") { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-baseline-")); - const runId = randomUUID().replaceAll("-", ""); - const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - const ptyMonitorRoot = `/tmp/nemoclaw-launch-turn-${runId}`; - const sessionRoot = join(fixtureRoot, "sessions"); - const sessionPath = join(sessionRoot, "session-a.jsonl"); - mkdirSync(sessionRoot); - writeSessionRecords(sessionRoot, { "session-a": [message("user"), message("assistant")] }, false); - try { - const baseline = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "baseline", - sessionRoot, - baselinePath, - "", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - const applyMutation: Record void> = { - invalid: () => - withOwnedFixtureFile(baselinePath, constants.O_WRONLY, (descriptor) => { - ftruncateSync(descriptor, 0); - writeFileSync(descriptor, "{}"); - fsyncSync(descriptor); - }), - removed: () => rmSync(sessionPath), - rewritten: () => - writeFileSync( - sessionPath, - readFileSync(sessionPath, "utf8").replace("nonempty", "changed!"), - ), - truncated: () => writeFileSync(sessionPath, ""), - }; - applyMutation[mutation](); - const qualification = spawnSync( - process.execPath, - [ - "-e", - OPENCLAW_SESSION_EVIDENCE_SCRIPT, - "qualify", - sessionRoot, - baselinePath, - "1", - ptyMonitorRoot, - runId, - ], - { encoding: "utf8" }, - ); - return { baseline, qualification }; - } finally { - rmSync(fixtureRoot, { force: true, recursive: true }); - rmSync(baselinePath, { force: true }); - rmSync(`${baselinePath}.tmp`, { force: true }); - rmSync(ptyMonitorRoot, { force: true, recursive: true }); - } -} +import { + emptyMessage, + message, + PROCESS_EXIT_WAIT, + runBaselineMutationFixture, + runEvidenceFixture, + type FixtureMode, + type SessionRecords, +} from "./launch-agent-turn-fixtures"; export type LaunchAgentTurnGroup = "evidence" | "failure-paths" | "readiness" | "termios"; diff --git a/test/helpers/growth-guardrail-checks.ts b/test/helpers/growth-guardrail-checks.ts index a94cd413cba..2201ebc81a1 100644 --- a/test/helpers/growth-guardrail-checks.ts +++ b/test/helpers/growth-guardrail-checks.ts @@ -11,6 +11,8 @@ const E2E_ASSERTION_BUDGET_FILE = "ci/e2e-assertion-budget.json"; const FALLBACK_BUDGET = '{"defaultMaxLines":1500,"legacyMaxLines":{}}'; const JAVASCRIPT_FILE_RE = /\.(?:cjs|js|mjs)$/; const TEST_FILE_RE = /^(?:test|src|nemoclaw\/src)\/.*\.(?:test|spec)\.(?:[cm]?[jt]s)$/; +const TEST_SIZE_FILE_RE = + /^(?:test|src|nemoclaw\/src)\/.*(?:\.(?:test|spec)|-suite)\.(?:[cm]?[jt]s)$/; const LIVE_E2E_TEST_RE = /^test\/e2e\/live\/.*\.(?:test|spec)\.(?:[cm]?[jt]s|[jt]sx)$/; const ONBOARD_ENTRY = "src/lib/onboard.ts"; const STOCK_DOCKERFILE = "Dockerfile"; @@ -345,7 +347,7 @@ export async function testSizeViolations(diff: GrowthGuardrailDiff): Promise status !== "removed" && TEST_FILE_RE.test(filename)) + .filter(({ filename, status }) => status !== "removed" && TEST_SIZE_FILE_RE.test(filename)) .map(({ filename }) => filename); const baseBudgetBlob = await diff.readBase([BUDGET_FILE]); const baseBudget = parseBudget(baseBudgetBlob.get(BUDGET_FILE) ?? FALLBACK_BUDGET, "base budget"); diff --git a/test/helpers/onboard-messaging-configuration-suite.ts b/test/helpers/onboard-messaging-configuration-suite.ts new file mode 100644 index 00000000000..88c8888546a --- /dev/null +++ b/test/helpers/onboard-messaging-configuration-suite.ts @@ -0,0 +1,876 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { beforeEach, describe, it, vi } from "vitest"; +import YAML from "yaml"; + +import { + activeChannelsFromDockerfile, + encodeMessagingPlanForChannels, + messagingPlanLiteral, + parseMessagingFixturePayload, + writeCustomMessagingDockerfile, +} from "./messaging-plan-fixtures"; +import { runBoundedOnboardScript } from "./onboard-child-process-harness"; +import { writeOkOpenshell } from "./onboard-openshell-fixture"; + +type CommandEntry = { + command: string; + env?: Record; + policyContent?: string; + policyReadError?: string; + dockerfileContent?: string; + dockerfileReadError?: string; + providerRevisions?: Record | null; + rawCredentialInEnv?: boolean; +}; +const parseStdoutJson = parseMessagingFixturePayload; +const repoRoot = path.join(import.meta.dirname, "../.."); +const yamlModulePath = fileURLToPath(import.meta.resolve("yaml")); +const onboardScriptMocksPath = JSON.stringify( + path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), +); +beforeEach(() => { + vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); + vi.stubEnv("NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE", "1"); + vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); +}); + +export function registerOnboardMessagingConfigurationTests(): void { + describe("onboard messaging (configuration)", () => { + it( + "bakes WhatsApp into a custom sandbox image without bridge providers when no messaging tokens are set", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-tokenless-whatsapp-"), + ); + try { + const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "tokenless-whatsapp.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"]); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); + +const commands = []; let dockerfileContent; +const registerCalls = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = (command, opts = {}) => { +const normalized = _n(command); +commands.push({ command: normalized, env: opts.env || null }); +const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; +return createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { +const createdIdentity = createdSandbox.capture(command); +if (createdIdentity !== null) return createdIdentity; +{ + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; +} +if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; +return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { +registerCalls.push(entry); +return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { +sandboxName: "my-assistant", +provider: "nvidia-prod", +model: "gpt-5.4", +registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { +createdSandbox.create(args.flat()); +const child = new EventEmitter(); +child.stdout = new EventEmitter(); +child.stderr = new EventEmitter(); +child.unref = () => {}; +child.pid = 4242; +const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); +const entry = { command, env: args[2]?.env || null }; +const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); +if (dockerfileMatch) { + try { + entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); + } catch (error) { + entry.dockerfileReadError = String(error); + } +} +commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); +process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); +}); +return child; +}; + +require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); +const { createSandbox } = require(${onboardPath}); + +(async () => { +process.env.OPENSHELL_GATEWAY = "nemoclaw"; +for (const key of Object.keys(process.env)) { + if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { + delete process.env[key]; + } +} +process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); +const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); +console.log(JSON.stringify({ sandboxName, commands, registerCalls })); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + const providerMutationCommands = payload.commands.filter( + (entry: CommandEntry) => + /\bprovider (create|update)\b/.test(entry.command) && + entry.command.includes("-bridge"), + ); + assert.equal( + providerMutationCommands.length, + 0, + "QR-only channel selection must not create bridge providers", + ); + + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.equal(createCommand.dockerfileReadError, undefined); + assert.doesNotMatch(createCommand.command, /--provider \S+-bridge\b/); + + assert.deepEqual(activeChannelsFromDockerfile(createCommand.dockerfileContent), [ + "whatsapp", + ]); + assert.deepEqual( + payload.registerCalls[0]?.messaging?.plan?.channels.map( + (channel: { channelId: string }) => channel.channelId, + ), + ["whatsapp"], + ); + assert.equal(payload.registerCalls[0]?.messagingChannels, undefined); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, + ); + + it( + "drops WhatsApp from a rebuilt custom image when the registry marks it disabled", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-disabled-whatsapp-"), + ); + try { + const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "disabled-whatsapp.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"], ["whatsapp"]); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); + +registry.registerSandbox({ +name: "my-assistant", +messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["whatsapp"], ["whatsapp"])} }, +}); + +const commands = []; let dockerfileContent; +const registerCalls = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = (command, opts = {}) => { +const normalized = _n(command); +commands.push({ command: normalized, env: opts.env || null }); +const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; +return createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { +const createdIdentity = createdSandbox.capture(command); +if (createdIdentity !== null) return createdIdentity; +{ + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; +} +if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; +return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = (entry) => { +registerCalls.push(entry); +return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { +sandboxName: "my-assistant", +provider: "nvidia-prod", +model: "gpt-5.4", +getSandbox: registry.getSandbox, +registerSandbox: registry.registerSandbox, +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { +createdSandbox.create(args.flat()); +const child = new EventEmitter(); +child.stdout = new EventEmitter(); +child.stderr = new EventEmitter(); +child.unref = () => {}; +child.pid = 4242; +const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); +const entry = { command, env: args[2]?.env || null }; +const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); +if (dockerfileMatch) { + try { + entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); + } catch (error) { + entry.dockerfileReadError = String(error); + } +} +commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); +process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); +}); +return child; +}; + +require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); +const { createSandbox } = require(${onboardPath}); + +(async () => { +process.env.OPENSHELL_GATEWAY = "nemoclaw"; +for (const key of Object.keys(process.env)) { + if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { + delete process.env[key]; + } +} +process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); +const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); +console.log(JSON.stringify({ sandboxName, commands, registerCalls })); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.equal(createCommand.dockerfileReadError, undefined); + + assert.deepEqual( + activeChannelsFromDockerfile(createCommand.dockerfileContent), + [], + "disabled QR channel must not be active in the image plan", + ); + const registeredPlan = payload.registerCalls[0]?.messaging?.plan; + assert.deepEqual( + registeredPlan?.channels.map((channel: { channelId: string }) => channel.channelId), + ["whatsapp"], + "registry.messaging.plan must keep the disabled QR channel so `channels start` can recover it (mirrors #3381)", + ); + assert.deepEqual( + registeredPlan?.disabledChannels, + ["whatsapp"], + "registry.messaging.plan.disabledChannels must round-trip through the rebuild", + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, + ); + + it( + "does not create messaging providers from ambient credentials without a selected plan (#10277)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-empty-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "enabled-channels-empty.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); + +const commands = []; +const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); +runner.run = (command, opts = {}) => { +commands.push({ command: _n(command), env: opts.env || null }); +return fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw") ?? createdSandbox.run(command) ?? { status: 0 }; +}; +runner.runCapture = (command) => { +const createdIdentity = createdSandbox.capture(command); +if (createdIdentity !== null) return createdIdentity; +{ + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; +} +if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; +return ""; +}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); +registry.registerSandbox = () => true; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { +sandboxName: "my-assistant", +provider: "nvidia-prod", +model: "gpt-5.4", +}); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { +createdSandbox.create(args.flat()); +const child = new EventEmitter(); +child.stdout = new EventEmitter(); +child.stderr = new EventEmitter(); +child.unref = () => {}; +child.pid = 4242; +commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null }); +process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); +}); +return child; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { +process.env.OPENSHELL_GATEWAY = "nemoclaw"; +process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; +process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; +process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; +// No selected messaging plan — ambient repository credentials are unrelated to this request. +const sandboxName = await createSandbox( + ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + "gpt-5.4", + "nvidia-prod", + null, + "my-assistant", + null, + null, + null, + null, + null, + null, + null, + [], + ], + createFixture, + ), +); +console.log(JSON.stringify({ sandboxName, commands })); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson(result.stdout); + + // No messaging providers should be created at all + const providerCommands = payload.commands.filter((e: CommandEntry) => + e.command.includes("provider create"), + ); + assert.equal( + providerCommands.length, + 0, + "no providers should be created without a selected messaging plan", + ); + + // Sandbox create should have no --provider flags for messaging bridges + const createCommand = payload.commands.find((e: CommandEntry) => + e.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.doesNotMatch(createCommand.command, /discord-bridge/); + assert.doesNotMatch(createCommand.command, /slack-bridge/); + assert.doesNotMatch(createCommand.command, /telegram-bridge/); + }, + ); + + it( + "non-interactive setupMessagingChannels returns channels with tokens", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-noninteractive-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-noninteractive.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +// Stub the manifest-driven Telegram reachability hook so this test does not +// make a real network call. +global.fetch = async () => ({ +ok: true, +status: 200, +json: async () => ({ ok: true, result: { id: 1, is_bot: true } }), +text: async () => "", +}); + +const { setupMessagingChannels } = require(${onboardPath}); + +(async () => { +// Only set telegram and slack tokens — discord should be absent +process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; +process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; +process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token"; +process.env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; +const result = await setupMessagingChannels(); +console.log(JSON.stringify(result)); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const channels = parseStdoutJson(result.stdout); + + // Should return only the channels that have tokens set + assert.ok(Array.isArray(channels), "expected an array return value"); + assert.ok(channels.includes("telegram"), "expected telegram in returned channels"); + assert.ok(channels.includes("slack"), "expected slack in returned channels"); + assert.ok(!channels.includes("discord"), "discord should not be in returned channels"); + }, + ); + + it( + "non-interactive setupMessagingChannels drops Slack when live Slack API validation rejects the token", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-slack-live-reject-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-slack-live-reject.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const httpProbePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "adapters", "http", "probe.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const httpProbe = require(${httpProbePath}); +httpProbe.runCurlProbe = (argv) => { +const url = argv[argv.length - 1] || ""; +if (String(url).includes("auth.test")) { + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"ok":false,"error":"invalid_auth"}', + stderr: "", + message: "", + }; +} +return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"ok":true}', + stderr: "", + message: "", +}; +}; + +const { setupMessagingChannels } = require(${onboardPath}); + +(async () => { +delete process.env.TELEGRAM_BOT_TOKEN; +delete process.env.DISCORD_BOT_TOKEN; +process.env.SLACK_BOT_TOKEN = "xoxb-fake-bot-token"; +process.env.SLACK_APP_TOKEN = "xapp-fake-app-token"; +const result = await setupMessagingChannels(); +console.log(JSON.stringify(result)); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const channels = parseStdoutJson(result.stdout); + + assert.ok(Array.isArray(channels), "expected an array return value"); + assert.ok(!channels.includes("slack"), "Slack should be dropped after API rejection"); + assert.doesNotMatch(result.stdout, /xoxb-fake-bot-token/); + assert.doesNotMatch(result.stderr, /xoxb-fake-bot-token/); + }, + ); + + it( + "non-interactive setupMessagingChannels returns empty array when no tokens set", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-messaging-no-tokens-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "messaging-no-tokens.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const { setupMessagingChannels } = require(${onboardPath}); + +(async () => { +// No messaging tokens set +delete process.env.TELEGRAM_BOT_TOKEN; +delete process.env.DISCORD_BOT_TOKEN; +delete process.env.SLACK_BOT_TOKEN; +delete process.env.SLACK_APP_TOKEN; +const result = await setupMessagingChannels(); +console.log(JSON.stringify(result)); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + TELEGRAM_BOT_TOKEN: "", + DISCORD_BOT_TOKEN: "", + SLACK_BOT_TOKEN: "", + SLACK_APP_TOKEN: "", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const channels = parseStdoutJson(result.stdout); + + assert.ok(Array.isArray(channels), "expected an array return value"); + assert.equal(channels.length, 0, "expected empty array when no tokens are set"); + }, + ); + + it( + "interactive setupMessagingChannels drops slack when prompted token fails tokenFormat check (#1912)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-slack-format-reject-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "slack-format-reject.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + // Subscript: mocks credentials.prompt to return a bogus Slack token, + // exposes MESSAGING_CHANNELS so the parent can look up the Slack toggle + // digit, and asserts that setupMessagingChannels rejects the invalid + // token without persisting it. Slack is the 3rd channel in insertion + // order today (telegram, discord, slack) but we compute the index + // dynamically to avoid a brittle coupling to that ordering. + const script = String.raw` +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); + +const saveCalls = []; +credentials.saveCredential = (key, value) => { saveCalls.push({ key, value }); }; +credentials.getCredential = () => null; +credentials.prompt = async (message) => { +if (message.includes("Slack Bot Token")) return "abcd"; +return ""; +}; + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); + +(async () => { +delete process.env.TELEGRAM_BOT_TOKEN; +delete process.env.DISCORD_BOT_TOKEN; +delete process.env.SLACK_BOT_TOKEN; +delete process.env.SLACK_APP_TOKEN; + +const result = await setupMessagingChannels(); +console.log(JSON.stringify({ + result, + saveCalls, + slackIndex1Based: MESSAGING_CHANNELS.findIndex((c) => c.name === "slack") + 1, +})); +})().catch((error) => { +console.error(error); +process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + // Dry run with just Enter — no toggles, empty result — used to read back + // Slack's 1-based index from the same subscript so the real run can + // press the right digit. + const introspect = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + input: "\n", + }); + assert.equal(introspect.status, 0, introspect.stderr); + const introspectOut = JSON.parse(introspect.stdout.trim().split("\n").pop()!); + const slackIdx = introspectOut.slackIndex1Based; + assert.ok(slackIdx >= 1, `unexpected slack index: ${slackIdx}`); + + // Real run: press Slack's digit, Enter. Slack gets toggled on, prompt + // fires, mocked prompt returns "abcd", tokenFormat regex rejects it, + // channel is dropped, saveCredential never runs for SLACK_BOT_TOKEN. + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + input: `${slackIdx}\n`, + }); + + assert.equal(result.status, 0, result.stderr); + const out = JSON.parse(result.stdout.trim().split("\n").pop()!); + + assert.ok( + !out.result.includes("slack"), + `slack should have been dropped after invalid token; got ${JSON.stringify(out.result)}`, + ); + assert.ok( + !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_BOT_TOKEN"), + `SLACK_BOT_TOKEN should NOT have been persisted; saveCalls=${JSON.stringify(out.saveCalls)}`, + ); + assert.ok( + result.stderr.includes("Invalid format") || result.stdout.includes("Invalid format"), + `expected 'Invalid format' warning; stderr=${result.stderr} stdout=${result.stdout}`, + ); + }, + ); + }); +} diff --git a/test/helpers/onboard-messaging-suite.ts b/test/helpers/onboard-messaging-suite.ts index 728091247e2..ae05fbd4028 100644 --- a/test/helpers/onboard-messaging-suite.ts +++ b/test/helpers/onboard-messaging-suite.ts @@ -42,7 +42,7 @@ beforeEach(() => { vi.stubEnv("NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE", "1"); vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); }); -export type OnboardMessagingGroup = "configuration" | "lifecycle" | "provider"; +export type OnboardMessagingGroup = "lifecycle" | "provider"; export function registerOnboardMessagingTests(group: OnboardMessagingGroup): void { describe(`onboard messaging (${group})`, () => { @@ -855,346 +855,6 @@ const { createSandbox } = require(${onboardPath}); ); } - if (group === "configuration") { - it( - "bakes WhatsApp into a custom sandbox image without bridge providers when no messaging tokens are set", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-tokenless-whatsapp-"), - ); - try { - const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "tokenless-whatsapp.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"]); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); - -const commands = []; let dockerfileContent; -const registerCalls = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; - return createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); - if (dockerfileMatch) { - try { - entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); - } catch (error) { - entry.dockerfileReadError = String(error); - } - } - commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - for (const key of Object.keys(process.env)) { - if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { - delete process.env[key]; - } - } - process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands, registerCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - const providerMutationCommands = payload.commands.filter( - (entry: CommandEntry) => - /\bprovider (create|update)\b/.test(entry.command) && - entry.command.includes("-bridge"), - ); - assert.equal( - providerMutationCommands.length, - 0, - "QR-only channel selection must not create bridge providers", - ); - - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.equal(createCommand.dockerfileReadError, undefined); - assert.doesNotMatch(createCommand.command, /--provider \S+-bridge\b/); - - assert.deepEqual(activeChannelsFromDockerfile(createCommand.dockerfileContent), [ - "whatsapp", - ]); - assert.deepEqual( - payload.registerCalls[0]?.messaging?.plan?.channels.map( - (channel: { channelId: string }) => channel.channelId, - ), - ["whatsapp"], - ); - assert.equal(payload.registerCalls[0]?.messagingChannels, undefined); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); - - it( - "drops WhatsApp from a rebuilt custom image when the registry marks it disabled", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-disabled-whatsapp-"), - ); - try { - const customDockerfileArg = JSON.stringify(writeCustomMessagingDockerfile(tmpDir)); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "disabled-whatsapp.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const messagingPlanB64 = encodeMessagingPlanForChannels(["whatsapp"], ["whatsapp"]); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const fs = require("node:fs"); - -registry.registerSandbox({ - name: "my-assistant", - messaging: { schemaVersion: 1, plan: ${messagingPlanLiteral(["whatsapp"], ["whatsapp"])} }, -}); - -const commands = []; let dockerfileContent; -const registerCalls = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - const providerGetResult = fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw"); if (providerGetResult !== null) return providerGetResult; - return createdSandbox.run(command) ?? { status: 0 }; -}; -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", - getSandbox: registry.getSandbox, - registerSandbox: registry.registerSandbox, -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - const command = _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]); - const entry = { command, env: args[2]?.env || null }; - const dockerfileMatch = command.match(/(?:--from|-f) ([^ ]+Dockerfile)/); - if (dockerfileMatch) { - try { - entry.dockerfileContent = dockerfileContent = fs.readFileSync(dockerfileMatch[1], "utf-8"); - } catch (error) { - entry.dockerfileReadError = String(error); - } - } - commands.push({ ...entry, dockerfileContent: entry.dockerfileContent ?? dockerfileContent }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -require(${onboardScriptMocksPath}).mockFreshOpenClawPluginDiscovery(); -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - for (const key of Object.keys(process.env)) { - if (key.startsWith("DISCORD_") || key.startsWith("SLACK_") || key.startsWith("TELEGRAM_")) { - delete process.env[key]; - } - } - process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands, registerCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_MESSAGING_PLAN_B64: messagingPlanB64, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.equal(createCommand.dockerfileReadError, undefined); - - assert.deepEqual( - activeChannelsFromDockerfile(createCommand.dockerfileContent), - [], - "disabled QR channel must not be active in the image plan", - ); - const registeredPlan = payload.registerCalls[0]?.messaging?.plan; - assert.deepEqual( - registeredPlan?.channels.map((channel: { channelId: string }) => channel.channelId), - ["whatsapp"], - "registry.messaging.plan must keep the disabled QR channel so `channels start` can recover it (mirrors #3381)", - ); - assert.deepEqual( - registeredPlan?.disabledChannels, - ["whatsapp"], - "registry.messaging.plan.disabledChannels must round-trip through the rebuild", - ); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); - } - if (group === "provider") { it("aborts onboard when a messaging provider upsert fails", { timeout: 60_000 }, async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-provider-fail-")); @@ -1371,172 +1031,29 @@ const { createSandbox } = require(${onboardPath}); "should NOT delete sandbox when providers already exist in gateway", ); - // Existing gateway providers do not select messaging for this onboarding request. - const providerUpserts = payload.commands.filter((entry: CommandEntry) => - entry.command.includes("provider update"), - ); - assert.equal( - providerUpserts.length, - 0, - "should not refresh ambient messaging providers without a selected channel plan", - ); - }, - ); - - it( - "filters messaging providers to only enabledChannels when provided", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-filter-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "enabled-channels-filter.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fixtureMocks = require(${onboardScriptMocksPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); - -const commands = []; -const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ - commands, - createdSandbox, -}); -runner.runCapture = (command) => { - const createdIdentity = createdSandbox.capture(command); - if (createdIdentity !== null) return createdIdentity; - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "SANDBOX BIND PORT PID STATUS"; - return ""; -}; require(${onboardScriptMocksPath}).mockIsolatedDockerSandboxLifecycleFromRunner(); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { - sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", -}); -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -childProcess.spawn = (...args) => { - createdSandbox.create(args.flat()); - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.emit("close", 0); - }); - return child; -}; - -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; - process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - // Only enable telegram — discord and slack should be filtered out - const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], null, null, null, null, null, []], createFixture)); - console.log(JSON.stringify({ sandboxName, commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson(result.stdout); - - // Only telegram provider should be created - const providerCommands = payload.commands.filter((e: CommandEntry) => - e.command.includes("provider create"), - ); - const telegramProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-telegram-bridge"), - ); - assert.ok(telegramProvider, "expected telegram provider to be created"); - - // Discord and slack providers should NOT be created - const discordProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-discord-bridge"), - ); - assert.ok(!discordProvider, "discord provider should be filtered out"); - - const slackProvider = providerCommands.find((e: CommandEntry) => - e.command.includes("my-assistant-slack-bridge"), + // Existing gateway providers do not select messaging for this onboarding request. + const providerUpserts = payload.commands.filter((entry: CommandEntry) => + entry.command.includes("provider update"), ); - assert.ok(!slackProvider, "slack provider should be filtered out"); - - // Sandbox create should only have the telegram --provider flag - const createCommand = payload.commands.find((e: CommandEntry) => - e.command.includes("sandbox create"), + assert.equal( + providerUpserts.length, + 0, + "should not refresh ambient messaging providers without a selected channel plan", ); - assert.ok(createCommand, "expected sandbox create command"); - assert.match(createCommand.command, /--provider my-assistant-telegram-bridge/); - assert.doesNotMatch(createCommand.command, /my-assistant-discord-bridge/); - assert.doesNotMatch(createCommand.command, /my-assistant-slack-bridge/); }, ); - } - if (group === "configuration") { it( - "does not create messaging providers from ambient credentials without a selected plan (#10277)", + "filters messaging providers to only enabledChannels when provided", { timeout: 60_000, }, async () => { const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-empty-"), + path.join(os.tmpdir(), "nemoclaw-onboard-enabled-channels-filter-"), ); const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "enabled-channels-empty.js"); + const scriptPath = path.join(tmpDir, "enabled-channels-filter.js"); const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); const registryPath = JSON.stringify( @@ -1564,10 +1081,10 @@ const { EventEmitter } = require("node:events"); const commands = []; const createdSandbox = fixtureMocks.createCreatedSandboxFixture(); createdSandbox.installRuntimeObservation(); -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return fixtureMocks.mockNvidiaOrMissingProviderGetRun(command, "nemoclaw") ?? createdSandbox.run(command) ?? { status: 0 }; -}; +runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ + commands, + createdSandbox, +}); runner.runCapture = (command) => { const createdIdentity = createdSandbox.capture(command); if (createdIdentity !== null) return createdIdentity; @@ -1612,27 +1129,8 @@ const { createSandbox } = require(${onboardPath}); process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - // No selected messaging plan — ambient repository credentials are unrelated to this request. - const sandboxName = await createSandbox( - ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( - [ - null, - "gpt-5.4", - "nvidia-prod", - null, - "my-assistant", - null, - null, - null, - null, - null, - null, - null, - [], - ], - createFixture, - ), - ); + // Only enable telegram — discord and slack should be filtered out + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], null, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1655,364 +1153,34 @@ const { createSandbox } = require(${onboardPath}); assert.equal(result.status, 0, result.stderr); const payload = parseStdoutJson(result.stdout); - // No messaging providers should be created at all + // Only telegram provider should be created const providerCommands = payload.commands.filter((e: CommandEntry) => e.command.includes("provider create"), ); - assert.equal( - providerCommands.length, - 0, - "no providers should be created without a selected messaging plan", - ); - - // Sandbox create should have no --provider flags for messaging bridges - const createCommand = payload.commands.find((e: CommandEntry) => - e.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.doesNotMatch(createCommand.command, /discord-bridge/); - assert.doesNotMatch(createCommand.command, /slack-bridge/); - assert.doesNotMatch(createCommand.command, /telegram-bridge/); - }, - ); - - it( - "non-interactive setupMessagingChannels returns channels with tokens", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-noninteractive-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-noninteractive.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -// Stub the manifest-driven Telegram reachability hook so this test does not -// make a real network call. -global.fetch = async () => ({ - ok: true, - status: 200, - json: async () => ({ ok: true, result: { id: 1, is_bot: true } }), - text: async () => "", -}); - -const { setupMessagingChannels } = require(${onboardPath}); - -(async () => { - // Only set telegram and slack tokens — discord should be absent - process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; - process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; - process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token"; - process.env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; - const result = await setupMessagingChannels(); - console.log(JSON.stringify(result)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const channels = parseStdoutJson(result.stdout); - - // Should return only the channels that have tokens set - assert.ok(Array.isArray(channels), "expected an array return value"); - assert.ok(channels.includes("telegram"), "expected telegram in returned channels"); - assert.ok(channels.includes("slack"), "expected slack in returned channels"); - assert.ok(!channels.includes("discord"), "discord should not be in returned channels"); - }, - ); - - it( - "non-interactive setupMessagingChannels drops Slack when live Slack API validation rejects the token", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-slack-live-reject-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-slack-live-reject.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const httpProbePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "adapters", "http", "probe.ts"), + const telegramProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-telegram-bridge"), ); + assert.ok(telegramProvider, "expected telegram provider to be created"); - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const httpProbe = require(${httpProbePath}); -httpProbe.runCurlProbe = (argv) => { - const url = argv[argv.length - 1] || ""; - if (String(url).includes("auth.test")) { - return { - ok: true, - httpStatus: 200, - curlStatus: 0, - body: '{"ok":false,"error":"invalid_auth"}', - stderr: "", - message: "", - }; - } - return { - ok: true, - httpStatus: 200, - curlStatus: 0, - body: '{"ok":true}', - stderr: "", - message: "", - }; -}; - -const { setupMessagingChannels } = require(${onboardPath}); - -(async () => { - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - process.env.SLACK_BOT_TOKEN = "xoxb-fake-bot-token"; - process.env.SLACK_APP_TOKEN = "xapp-fake-app-token"; - const result = await setupMessagingChannels(); - console.log(JSON.stringify(result)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const channels = parseStdoutJson(result.stdout); - - assert.ok(Array.isArray(channels), "expected an array return value"); - assert.ok(!channels.includes("slack"), "Slack should be dropped after API rejection"); - assert.doesNotMatch(result.stdout, /xoxb-fake-bot-token/); - assert.doesNotMatch(result.stderr, /xoxb-fake-bot-token/); - }, - ); - - it( - "non-interactive setupMessagingChannels returns empty array when no tokens set", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-messaging-no-tokens-"), + // Discord and slack providers should NOT be created + const discordProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-discord-bridge"), ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "messaging-no-tokens.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const { setupMessagingChannels } = require(${onboardPath}); - -(async () => { - // No messaging tokens set - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - delete process.env.SLACK_BOT_TOKEN; - delete process.env.SLACK_APP_TOKEN; - const result = await setupMessagingChannels(); - console.log(JSON.stringify(result)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - TELEGRAM_BOT_TOKEN: "", - DISCORD_BOT_TOKEN: "", - SLACK_BOT_TOKEN: "", - SLACK_APP_TOKEN: "", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const channels = parseStdoutJson(result.stdout); - - assert.ok(Array.isArray(channels), "expected an array return value"); - assert.equal(channels.length, 0, "expected empty array when no tokens are set"); - }, - ); + assert.ok(!discordProvider, "discord provider should be filtered out"); - it( - "interactive setupMessagingChannels drops slack when prompted token fails tokenFormat check (#1912)", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-slack-format-reject-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "slack-format-reject.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + const slackProvider = providerCommands.find((e: CommandEntry) => + e.command.includes("my-assistant-slack-bridge"), ); + assert.ok(!slackProvider, "slack provider should be filtered out"); - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - // Subscript: mocks credentials.prompt to return a bogus Slack token, - // exposes MESSAGING_CHANNELS so the parent can look up the Slack toggle - // digit, and asserts that setupMessagingChannels rejects the invalid - // token without persisting it. Slack is the 3rd channel in insertion - // order today (telegram, discord, slack) but we compute the index - // dynamically to avoid a brittle coupling to that ordering. - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const saveCalls = []; -credentials.saveCredential = (key, value) => { saveCalls.push({ key, value }); }; -credentials.getCredential = () => null; -credentials.prompt = async (message) => { - if (message.includes("Slack Bot Token")) return "abcd"; - return ""; -}; - -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); - -(async () => { - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - delete process.env.SLACK_BOT_TOKEN; - delete process.env.SLACK_APP_TOKEN; - - const result = await setupMessagingChannels(); - console.log(JSON.stringify({ - result, - saveCalls, - slackIndex1Based: MESSAGING_CHANNELS.findIndex((c) => c.name === "slack") + 1, - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - // Dry run with just Enter — no toggles, empty result — used to read back - // Slack's 1-based index from the same subscript so the real run can - // press the right digit. - const introspect = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - input: "\n", - }); - assert.equal(introspect.status, 0, introspect.stderr); - const introspectOut = JSON.parse(introspect.stdout.trim().split("\n").pop()!); - const slackIdx = introspectOut.slackIndex1Based; - assert.ok(slackIdx >= 1, `unexpected slack index: ${slackIdx}`); - - // Real run: press Slack's digit, Enter. Slack gets toggled on, prompt - // fires, mocked prompt returns "abcd", tokenFormat regex rejects it, - // channel is dropped, saveCredential never runs for SLACK_BOT_TOKEN. - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - input: `${slackIdx}\n`, - }); - - assert.equal(result.status, 0, result.stderr); - const out = JSON.parse(result.stdout.trim().split("\n").pop()!); - - assert.ok( - !out.result.includes("slack"), - `slack should have been dropped after invalid token; got ${JSON.stringify(out.result)}`, - ); - assert.ok( - !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_BOT_TOKEN"), - `SLACK_BOT_TOKEN should NOT have been persisted; saveCalls=${JSON.stringify(out.saveCalls)}`, - ); - assert.ok( - result.stderr.includes("Invalid format") || result.stdout.includes("Invalid format"), - `expected 'Invalid format' warning; stderr=${result.stderr} stdout=${result.stdout}`, + // Sandbox create should only have the telegram --provider flag + const createCommand = payload.commands.find((e: CommandEntry) => + e.command.includes("sandbox create"), ); + assert.ok(createCommand, "expected sandbox create command"); + assert.match(createCommand.command, /--provider my-assistant-telegram-bridge/); + assert.doesNotMatch(createCommand.command, /my-assistant-discord-bridge/); + assert.doesNotMatch(createCommand.command, /my-assistant-slack-bridge/); }, ); } diff --git a/test/onboarding/onboard-messaging-image-configuration.test.ts b/test/onboarding/onboard-messaging-image-configuration.test.ts index 764dc56c5dd..baeaeae34a9 100644 --- a/test/onboarding/onboard-messaging-image-configuration.test.ts +++ b/test/onboarding/onboard-messaging-image-configuration.test.ts @@ -1,6 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { registerOnboardMessagingTests } from "../helpers/onboard-messaging-suite"; +import { registerOnboardMessagingConfigurationTests } from "../helpers/onboard-messaging-configuration-suite"; -registerOnboardMessagingTests("configuration"); +registerOnboardMessagingConfigurationTests(); From 933cb4ef3f59defc5c872475fcd38bc893399add Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 9 Sep 2026 00:29:14 -0700 Subject: [PATCH 17/20] test(messaging): remove unused configuration fixtures --- test/helpers/onboard-messaging-configuration-suite.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/helpers/onboard-messaging-configuration-suite.ts b/test/helpers/onboard-messaging-configuration-suite.ts index 88c8888546a..84a179dc052 100644 --- a/test/helpers/onboard-messaging-configuration-suite.ts +++ b/test/helpers/onboard-messaging-configuration-suite.ts @@ -6,10 +6,8 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { beforeEach, describe, it, vi } from "vitest"; -import YAML from "yaml"; import { activeChannelsFromDockerfile, @@ -18,7 +16,6 @@ import { parseMessagingFixturePayload, writeCustomMessagingDockerfile, } from "./messaging-plan-fixtures"; -import { runBoundedOnboardScript } from "./onboard-child-process-harness"; import { writeOkOpenshell } from "./onboard-openshell-fixture"; type CommandEntry = { @@ -33,7 +30,6 @@ type CommandEntry = { }; const parseStdoutJson = parseMessagingFixturePayload; const repoRoot = path.join(import.meta.dirname, "../.."); -const yamlModulePath = fileURLToPath(import.meta.resolve("yaml")); const onboardScriptMocksPath = JSON.stringify( path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), ); From 824af6fbd6706d3553fc7f024c221d4f9d69ee8b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 9 Sep 2026 01:46:24 -0700 Subject: [PATCH 18/20] test(ci): unify timing hint validation --- ci/cli-test-timing-hints.json | 6 ++ scripts/checks/cli-test-timing-hints.mts | 76 +++++++++++++++++++ .../checks/report-cli-test-timing-drift.mts | 30 +------- test/helpers/cli-coverage-sequencer.ts | 75 +----------------- .../repository/cli-coverage-sequencer.test.ts | 41 +++++++--- test/repository/cli-test-timing-drift.test.ts | 20 +++++ 6 files changed, 136 insertions(+), 112 deletions(-) create mode 100644 scripts/checks/cli-test-timing-hints.mts diff --git a/ci/cli-test-timing-hints.json b/ci/cli-test-timing-hints.json index f432b5bd7cf..af55bc5b50b 100644 --- a/ci/cli-test-timing-hints.json +++ b/ci/cli-test-timing-hints.json @@ -37,6 +37,12 @@ "artifactId": 9983530478, "headSha": "be1452092af48f439531502870e826215b0d76f9", "recordedAt": "2026-09-06T05:43:35Z" + }, + { + "runId": 34324134466, + "artifactId": 10093500400, + "headSha": "933cb4ef3f59defc5c872475fcd38bc893399add", + "recordedAt": "2026-09-09T07:43:34Z" } ], "files": { diff --git a/scripts/checks/cli-test-timing-hints.mts b/scripts/checks/cli-test-timing-hints.mts new file mode 100644 index 00000000000..be0f11103d7 --- /dev/null +++ b/scripts/checks/cli-test-timing-hints.mts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +interface TimingHintSource { + runId: number; + artifactId: number; + headSha: string; + recordedAt: string; +} + +export interface CliTestTimingHints { + schemaVersion: 2; + defaultDurationMs: number; + sources: readonly TimingHintSource[]; + files: Readonly>; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function parseCliTestTimingHints(value: unknown): CliTestTimingHints { + if (!isRecord(value) || value.schemaVersion !== 2) { + throw new Error("CLI test timing hints must use schemaVersion 2"); + } + if (!Number.isSafeInteger(value.defaultDurationMs) || Number(value.defaultDurationMs) <= 0) { + throw new Error("CLI test timing hints require a positive integer defaultDurationMs"); + } + if (!Array.isArray(value.sources) || value.sources.length === 0) { + throw new Error("CLI test timing hints require source metadata"); + } + const sources = value.sources.map((source) => { + if (!isRecord(source)) throw new Error("Invalid CLI test timing hint source"); + const { runId, artifactId, headSha, recordedAt } = source; + if ( + !Number.isSafeInteger(runId) || + Number(runId) <= 0 || + !Number.isSafeInteger(artifactId) || + Number(artifactId) <= 0 || + typeof headSha !== "string" || + !/^[0-9a-f]{40}$/u.test(headSha) || + typeof recordedAt !== "string" || + Number.isNaN(Date.parse(recordedAt)) + ) { + throw new Error("Invalid CLI test timing hint source"); + } + return { runId: Number(runId), artifactId: Number(artifactId), headSha, recordedAt }; + }); + if (!isRecord(value.files)) { + throw new Error("CLI test timing hints require a files map"); + } + + const defaultDurationMs = Number(value.defaultDurationMs); + const files: Record = {}; + for (const [file, durationMs] of Object.entries(value.files)) { + const segments = file.split("/"); + if ( + file.length === 0 || + file.startsWith("/") || + file.includes("\\") || + segments.includes("..") || + !Number.isSafeInteger(durationMs) || + Number(durationMs) <= defaultDurationMs + ) { + throw new Error(`Invalid CLI test timing hint: ${file}`); + } + files[file] = Number(durationMs); + } + + return { + schemaVersion: 2, + defaultDurationMs, + sources, + files, + }; +} diff --git a/scripts/checks/report-cli-test-timing-drift.mts b/scripts/checks/report-cli-test-timing-drift.mts index 74780305921..8e56d22f622 100755 --- a/scripts/checks/report-cli-test-timing-drift.mts +++ b/scripts/checks/report-cli-test-timing-drift.mts @@ -6,10 +6,7 @@ import { appendFileSync, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -type TimingHints = { - defaultDurationMs: number; - files: Record; -}; +import { parseCliTestTimingHints } from "./cli-test-timing-hints.mts"; type VitestFileResult = { name?: unknown; @@ -45,29 +42,6 @@ function normalizeReportPath(name: string, repoRoot: string): string | undefined return relativeName; } -function parseHints(value: unknown): TimingHints { - if (!isRecord(value) || !Number.isSafeInteger(value.defaultDurationMs)) { - throw new Error("Invalid CLI timing hints"); - } - if (!isRecord(value.files)) { - throw new Error("Invalid CLI timing hint file map"); - } - - const defaultDurationMs = Number(value.defaultDurationMs); - if (defaultDurationMs <= 0) { - throw new Error("Invalid CLI timing hint default duration"); - } - - const files: Record = {}; - for (const [file, duration] of Object.entries(value.files)) { - if (!Number.isSafeInteger(duration) || Number(duration) <= defaultDurationMs) { - throw new Error(`Invalid CLI timing hint: ${file}`); - } - files[file] = Number(duration); - } - return { defaultDurationMs, files }; -} - function parseFileResults(value: VitestReport, repoRoot: string): Map { if (!Array.isArray(value.testResults)) { throw new Error("Invalid Vitest timing report"); @@ -99,7 +73,7 @@ export function findCliTestTimingDrift( hintsValue: unknown, repoRoot: string, ): TimingDrift[] { - const hints = parseHints(hintsValue); + const hints = parseCliTestTimingHints(hintsValue); const durations = parseFileResults(reportValue, path.resolve(repoRoot)); const drift: TimingDrift[] = []; diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index 50213ea397c..3de2ea5d273 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -6,20 +6,7 @@ import { readFileSync } from "node:fs"; import path from "node:path"; import { BaseSequencer, type TestSpecification } from "vitest/node"; - -interface TimingHintSource { - runId: number; - artifactId: number; - headSha: string; - recordedAt: string; -} - -export interface CliTestTimingHints { - schemaVersion: 2; - defaultDurationMs: number; - sources: readonly TimingHintSource[]; - files: Readonly>; -} +import { parseCliTestTimingHints } from "../../scripts/checks/cli-test-timing-hints.mts"; export interface WeightedShardEntry { key: string; @@ -48,66 +35,6 @@ const e2eSupportShardSalt = "96995"; // conservative fallback used to estimate each stable shard's load. const timingHintsUrl = new URL("../../ci/cli-test-timing-hints.json", import.meta.url); -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -export function parseCliTestTimingHints(value: unknown): CliTestTimingHints { - if (!isRecord(value) || value.schemaVersion !== 2) { - throw new Error("CLI test timing hints must use schemaVersion 2"); - } - if (!Number.isSafeInteger(value.defaultDurationMs) || Number(value.defaultDurationMs) <= 0) { - throw new Error("CLI test timing hints require a positive integer defaultDurationMs"); - } - if (!Array.isArray(value.sources) || value.sources.length === 0) { - throw new Error("CLI test timing hints require source metadata"); - } - const sources = value.sources.map((source) => { - if (!isRecord(source)) throw new Error("Invalid CLI test timing hint source"); - const { runId, artifactId, headSha, recordedAt } = source; - if ( - !Number.isSafeInteger(runId) || - Number(runId) <= 0 || - !Number.isSafeInteger(artifactId) || - Number(artifactId) <= 0 || - typeof headSha !== "string" || - !/^[0-9a-f]{40}$/u.test(headSha) || - typeof recordedAt !== "string" || - Number.isNaN(Date.parse(recordedAt)) - ) { - throw new Error("Invalid CLI test timing hint source"); - } - return { runId: Number(runId), artifactId: Number(artifactId), headSha, recordedAt }; - }); - if (!isRecord(value.files)) { - throw new Error("CLI test timing hints require a files map"); - } - - const defaultDurationMs = Number(value.defaultDurationMs); - const files: Record = {}; - for (const [file, durationMs] of Object.entries(value.files)) { - const segments = file.split("/"); - if ( - file.length === 0 || - file.startsWith("/") || - file.includes("\\") || - segments.includes("..") || - !Number.isSafeInteger(durationMs) || - Number(durationMs) <= defaultDurationMs - ) { - throw new Error(`Invalid CLI test timing hint: ${file}`); - } - files[file] = Number(durationMs); - } - - return { - schemaVersion: 2, - defaultDurationMs, - sources, - files, - }; -} - export const cliTestTimingHints = parseCliTestTimingHints( JSON.parse(readFileSync(timingHintsUrl, "utf8")), ); diff --git a/test/repository/cli-coverage-sequencer.test.ts b/test/repository/cli-coverage-sequencer.test.ts index 60636732ad8..791a53ab718 100644 --- a/test/repository/cli-coverage-sequencer.test.ts +++ b/test/repository/cli-coverage-sequencer.test.ts @@ -11,11 +11,11 @@ import { discoverVitestCandidates, expectedProjectForTestPath, } from "../../scripts/checks/vitest-project-overlap.mts"; +import { parseCliTestTimingHints } from "../../scripts/checks/cli-test-timing-hints.mts"; import { assignStableShards, CliCoverageSequencer, cliTestTimingHints, - parseCliTestTimingHints, shouldUseCliCoverageSharding, timingWeightForPath, type WeightedShardEntry, @@ -95,10 +95,14 @@ describe("stable CLI coverage sharding", () => { ]); const withRemoval = assignmentOwners(entries.slice(1)); - expect(entries.every((entry) => - Object.is(withAddition.get(entry.key), baseline.get(entry.key)))).toBe(true); - expect(entries.slice(1).every((entry) => - Object.is(withRemoval.get(entry.key), baseline.get(entry.key)))).toBe(true); + expect( + entries.every((entry) => Object.is(withAddition.get(entry.key), baseline.get(entry.key))), + ).toBe(true); + expect( + entries + .slice(1) + .every((entry) => Object.is(withRemoval.get(entry.key), baseline.get(entry.key))), + ).toBe(true); }); it("keeps recorded project and path keys on their stable shards", () => { @@ -137,9 +141,7 @@ describe("stable CLI coverage sharding", () => { ); expect(integrationEntries.length).toBeGreaterThan(0); - const weights = assignStableShards(integrationEntries, 12).map( - (shard) => shard.totalWeightMs, - ); + const weights = assignStableShards(integrationEntries, 12).map((shard) => shard.totalWeightMs); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.1); @@ -155,8 +157,14 @@ describe("stable CLI coverage sharding", () => { it("wires stable project and path ownership into the Vitest sequencer", async () => { const specifications = [ - testSpecification("test/credentials/local-credential-helper-fields.test.ts", "local-credentials"), - testSpecification("test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts", "hermes-config"), + testSpecification( + "test/credentials/local-credential-helper-fields.test.ts", + "local-credentials", + ), + testSpecification( + "test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts", + "hermes-config", + ), ...Array.from({ length: 8 }, (_, index) => testSpecification(`test/regular-${index}.test.ts`, `regular-${index}`), ), @@ -217,6 +225,12 @@ describe("stable CLI coverage sharding", () => { headSha: "be1452092af48f439531502870e826215b0d76f9", recordedAt: "2026-09-06T05:43:35Z", }, + { + runId: 34324134466, + artifactId: 10093500400, + headSha: "933cb4ef3f59defc5c872475fcd38bc893399add", + recordedAt: "2026-09-09T07:43:34Z", + }, ]); expect(files).toEqual([...files].sort()); expect(files.length).toBeGreaterThan(50); @@ -227,6 +241,13 @@ describe("stable CLI coverage sharding", () => { expect(timingWeightForPath("test/new-unprofiled-test.test.ts")).toBe(5_000); }); + it("uses the conservative fallback for the measured sub-default session-evidence split", () => { + const file = "test/e2e/support/launch-agent-turn-session-evidence.test.ts"; + + expect(cliTestTimingHints.files[file]).toBeUndefined(); + expect(timingWeightForPath(file)).toBe(cliTestTimingHints.defaultDurationMs); + }); + it("rejects malformed timing hint manifests", () => { expect(() => parseCliTestTimingHints({ schemaVersion: 1 })).toThrow(/schemaVersion 2/u); expect(() => diff --git a/test/repository/cli-test-timing-drift.test.ts b/test/repository/cli-test-timing-drift.test.ts index e7faa1c17b9..f0ccb1da01d 100644 --- a/test/repository/cli-test-timing-drift.test.ts +++ b/test/repository/cli-test-timing-drift.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { parseCliTestTimingHints } from "../../scripts/checks/cli-test-timing-hints.mts"; import { findCliTestTimingDrift, formatCliTestTimingDriftSummary, @@ -23,7 +24,16 @@ function report(files: Record): object { } const hints = { + schemaVersion: 2, defaultDurationMs: 5_000, + sources: [ + { + runId: 1, + artifactId: 1, + headSha: "0123456789abcdef0123456789abcdef01234567", + recordedAt: "2026-09-09T00:00:00Z", + }, + ], files: { "test/faster.test.ts": 50_000, "test/slower.test.ts": 10_000, @@ -76,4 +86,14 @@ describe("CLI test timing drift", () => { expect(summary).toContain("16.0s"); expect(formatCliTestTimingDriftSummary([])).toContain("No material timing-hint drift"); }); + + it.each([ + { name: "schema version", value: { ...hints, schemaVersion: 1 } }, + { name: "source metadata", value: { ...hints, sources: [] } }, + ])("rejects invalid $name through the shared parser and reporter", ({ value }) => { + expect(() => parseCliTestTimingHints(value)).toThrow(/schemaVersion 2|source metadata/u); + expect(() => findCliTestTimingDrift(report({}), value, repoRoot)).toThrow( + /schemaVersion 2|source metadata/u, + ); + }); }); From 6457319fa7e53f9e4af4c1808bd3e8258dfa408d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 9 Sep 2026 02:11:15 -0700 Subject: [PATCH 19/20] test(ci): expose invalid timing records --- scripts/checks/report-cli-test-timing-drift.mts | 8 +++++--- .../pull-requests/pr-workflow-contract.test.ts | 3 +++ test/repository/cli-test-timing-drift.test.ts | 11 +++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/checks/report-cli-test-timing-drift.mts b/scripts/checks/report-cli-test-timing-drift.mts index 8e56d22f622..31ba8eae209 100755 --- a/scripts/checks/report-cli-test-timing-drift.mts +++ b/scripts/checks/report-cli-test-timing-drift.mts @@ -48,8 +48,10 @@ function parseFileResults(value: VitestReport, repoRoot: string): Map(); - for (const candidate of value.testResults) { - if (!isRecord(candidate)) continue; + for (const [index, candidate] of value.testResults.entries()) { + if (!isRecord(candidate)) { + throw new Error(`Invalid Vitest timing report entry at index ${index}`); + } const { name, startTime, endTime } = candidate as VitestFileResult; if ( typeof name !== "string" || @@ -59,7 +61,7 @@ function parseFileResults(value: VitestReport, repoRoot: string): Map { "keeps %s plugin test typechecking after the trusted production build", (_name, workflow) => { expect([workflow.jobs["build-typecheck"].needs].flat()).toContain("compile-artifacts"); + expect( + requiredStep(sharedActions.compileArtifacts, "Compile and verify outputs").run, + ).toContain("npm --prefix nemoclaw run build"); expect(requiredStep(sharedActions.buildTypecheck, "Typecheck plugin tests").run).toBe( "npm --prefix nemoclaw exec -- tsc --noEmit -p nemoclaw/tsconfig.test.json", ); diff --git a/test/repository/cli-test-timing-drift.test.ts b/test/repository/cli-test-timing-drift.test.ts index f0ccb1da01d..e940b822860 100644 --- a/test/repository/cli-test-timing-drift.test.ts +++ b/test/repository/cli-test-timing-drift.test.ts @@ -76,6 +76,17 @@ describe("CLI test timing drift", () => { ]); }); + it("rejects malformed file results instead of reporting a false normal state", () => { + const value = report({ "test/stable.test.ts": 20_000 }) as { + testResults: unknown[]; + }; + value.testResults.push({ name: "test/malformed.test.ts", startTime: 1_000 }); + + expect(() => findCliTestTimingDrift(value, hints, repoRoot)).toThrow( + /Invalid Vitest timing report entry at index 1/u, + ); + }); + it("renders a concise advisory summary", () => { const summary = formatCliTestTimingDriftSummary([ { file: "test/new.test.ts", kind: "unprofiled", observedMs: 16_000 }, From 5d184691371622fff408b0f8bae452bf41e83a39 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 9 Sep 2026 02:44:18 -0700 Subject: [PATCH 20/20] test(ci): govern extracted suite growth --- .../pull-requests/growth-guardrails.test.ts | 21 ++++++++++++++++ test/e2e/support/launch-agent-turn-suite.ts | 25 ++++++++++--------- test/helpers/classify-ci-failure-suite.ts | 17 +++++++------ test/helpers/growth-guardrail-checks.ts | 13 +++++----- test/helpers/onboard-messaging-suite.ts | 13 +++++----- test/helpers/onboard-sandbox-build-suite.ts | 9 ++++--- test/helpers/test-group-registration.ts | 10 ++++++++ 7 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 test/helpers/test-group-registration.ts diff --git a/test/automation/pull-requests/growth-guardrails.test.ts b/test/automation/pull-requests/growth-guardrails.test.ts index fd0decacb07..f952968dcbe 100644 --- a/test/automation/pull-requests/growth-guardrails.test.ts +++ b/test/automation/pull-requests/growth-guardrails.test.ts @@ -441,6 +441,27 @@ function defineCodebaseGrowthGuardrailTestSupport(): void { ]); }); + it("keeps conditional growth attached when a test is renamed to a suite owner", async () => { + const base = "it('works', () => expect(ok).toBe(true));"; + const diff = fixtureDiff( + [ + { + filename: "test/helpers/example-suite.ts", + previous_filename: "test/example.test.ts", + status: "renamed", + }, + ], + { "test/example.test.ts": base }, + { + "test/helpers/example-suite.ts": `${base}\nif (ok) expect(ok).toBe(true);`, + }, + ); + + expect(await conditionalGrowthViolations(diff)).toEqual([ + "test/helpers/example-suite.ts: 1 if statement(s), up from 0", + ]); + }); + it.each([ ["if statements", conditionalGrowthViolations, "if (ok) expect(ok).toBe(true);"], [ diff --git a/test/e2e/support/launch-agent-turn-suite.ts b/test/e2e/support/launch-agent-turn-suite.ts index 2b4007f6e97..05ceac8849b 100644 --- a/test/e2e/support/launch-agent-turn-suite.ts +++ b/test/e2e/support/launch-agent-turn-suite.ts @@ -25,6 +25,7 @@ import { SUBPROCESS_ENV_ALLOWED_PREFIXES, } from "../../../src/lib/subprocess-env"; import { testTimeout } from "../../helpers/timeouts"; +import { registerTestGroup } from "../../helpers/test-group-registration"; import { LAUNCH_TURN_SCRIPT, OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, @@ -47,7 +48,7 @@ import { export type LaunchAgentTurnGroup = "evidence" | "failure-paths" | "readiness" | "termios"; export function registerLaunchAgentTurnTests(group: LaunchAgentTurnGroup): void { - if (group === "evidence") { + registerTestGroup(group, "evidence", () => { it("reports a residual PTY monitor socket without removing it (#9384)", async () => { const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-monitor-cleanup-")); const runId = randomUUID().replaceAll("-", ""); @@ -87,7 +88,7 @@ export function registerLaunchAgentTurnTests(group: LaunchAgentTurnGroup): void rmSync(ptyMonitorRoot, { force: true, recursive: true }); } }); - } + }); function runLaunchSessionFixture( mode: FixtureMode, @@ -669,7 +670,7 @@ require("node:fs").appendFileSync( } } - if (group === "evidence") { + registerTestGroup(group, "evidence", () => { it("qualifies two ordered structured turns without comparing message content (#9160)", () => { const { baseline, baselineKeys, baselineMode, baselineNlink, baselineUid, qualification } = runEvidenceFixture({ @@ -822,9 +823,9 @@ require("node:fs").appendFileSync( } }, ); - } + }); - if (group === "readiness") { + registerTestGroup(group, "readiness", () => { it.runIf(process.platform === "linux").each(["absent", "ansi", "reordered"] as const)( "keeps the monitor alive through SIGTERM, records an auto-message and PTY turn, sends /exit, strips launch authority, and ignores terminal copy evidence [%s] (#9160, #9384)", (terminalCopy) => { @@ -1029,9 +1030,9 @@ require("node:fs").appendFileSync( }, testTimeout(20_000), ); - } + }); - if (group === "termios") { + registerTestGroup(group, "termios", () => { it.runIf(process.platform === "linux")( "requires a current noncanonical observation after the PTY returns to canonical mode (#9384)", () => { @@ -1057,9 +1058,9 @@ require("node:fs").appendFileSync( }, testTimeout(30_000), ); - } + }); - if (group === "failure-paths") { + registerTestGroup(group, "failure-paths", () => { it.runIf(process.platform === "linux")( "fails when the PTY monitor socket remains missing until the session deadline (#9160)", () => { @@ -1207,9 +1208,9 @@ require("node:fs").appendFileSync( expect(result.status).toBe(23); }, ); - } + }); - if (group === "termios") { + registerTestGroup(group, "termios", () => { it.runIf(process.platform === "linux")( "rejects a relative OpenShell command before launching a host command (#9160)", async () => { @@ -1345,5 +1346,5 @@ require("node:fs").appendFileSync( }); }, ); - } + }); } diff --git a/test/helpers/classify-ci-failure-suite.ts b/test/helpers/classify-ci-failure-suite.ts index 4c5731178c8..2e2cd1624e8 100644 --- a/test/helpers/classify-ci-failure-suite.ts +++ b/test/helpers/classify-ci-failure-suite.ts @@ -22,6 +22,7 @@ import { resolveProductionGhExecutableForTest, } from "../../.agents/skills/nemoclaw-maintainer-classify-ci-failure/scripts/classify-ci-failure.mts"; import { artifactZip, artifactZipEntryDataOffset } from "./artifact-zip"; +import { registerTestGroup } from "./test-group-registration"; const script = resolve( ".agents/skills/nemoclaw-maintainer-classify-ci-failure/scripts/classify-ci-failure.mts", ); @@ -251,7 +252,7 @@ function trustedDirectories(home = "/home/tester"): Record { export type ClassifyCiFailureGroup = "artifacts" | "logs" | "process-lifecycle"; export function registerClassifyCiFailureTests(group: ClassifyCiFailureGroup): void { - if (group === "logs") { + registerTestGroup(group, "logs", () => { describe("GitHub CLI production resolver", () => { test("selects a safe user-local executable without consulting PATH", () => { const home = "/home/tester"; @@ -307,10 +308,10 @@ export function registerClassifyCiFailureTests(group: ClassifyCiFailureGroup): v ).toThrow("Could not find a trusted GitHub CLI executable"); }); }); - } + }); describe.skipIf(process.platform !== "linux")("CI failure classifier process", () => { - if (group === "logs") { + registerTestGroup(group, "logs", () => { test("redacts credentials from classified diagnostic output", () => { const secrets = [ "Authorization: Bearer full authorization value with spaces", @@ -652,9 +653,9 @@ export function registerClassifyCiFailureTests(group: ClassifyCiFailureGroup): v expect(result.stderr.length).toBeLessThanOrEqual(2001); expect(classifierTemporaryDirectories("nemoclaw-ci-log.")).toEqual([]); }); - } + }); - if (group === "artifacts") { + registerTestGroup(group, "artifacts", () => { test.each([ ["malformed", Buffer.from("not a zip")], [ @@ -977,9 +978,9 @@ export function registerClassifyCiFailureTests(group: ClassifyCiFailureGroup): v ); expect(result.stdout).not.toContain(secret); }); - } + }); - if (group === "process-lifecycle") { + registerTestGroup(group, "process-lifecycle", () => { test("retains a leader for an ignored-stdio process-group member until timeout", async () => { const root = mkdtempSync(join(tmpdir(), "classify-ci-timeout-")); roots.push(root); @@ -1156,6 +1157,6 @@ export function registerClassifyCiFailureTests(group: ClassifyCiFailureGroup): v expect(value.artifact.failures[0].path).toContain("[REDACTED]"); expect(result.stdout).not.toContain(secret); }); - } + }); }); } diff --git a/test/helpers/growth-guardrail-checks.ts b/test/helpers/growth-guardrail-checks.ts index 2201ebc81a1..eaf640d8240 100644 --- a/test/helpers/growth-guardrail-checks.ts +++ b/test/helpers/growth-guardrail-checks.ts @@ -10,8 +10,7 @@ const BUDGET_FILE = "ci/test-file-size-budget.json"; const E2E_ASSERTION_BUDGET_FILE = "ci/e2e-assertion-budget.json"; const FALLBACK_BUDGET = '{"defaultMaxLines":1500,"legacyMaxLines":{}}'; const JAVASCRIPT_FILE_RE = /\.(?:cjs|js|mjs)$/; -const TEST_FILE_RE = /^(?:test|src|nemoclaw\/src)\/.*\.(?:test|spec)\.(?:[cm]?[jt]s)$/; -const TEST_SIZE_FILE_RE = +const TEST_SOURCE_FILE_RE = /^(?:test|src|nemoclaw\/src)\/.*(?:\.(?:test|spec)|-suite)\.(?:[cm]?[jt]s)$/; const LIVE_E2E_TEST_RE = /^test\/e2e\/live\/.*\.(?:test|spec)\.(?:[cm]?[jt]s|[jt]sx)$/; const ONBOARD_ENTRY = "src/lib/onboard.ts"; @@ -81,14 +80,16 @@ function testChanges(files: readonly PullRequestFile[]): TestChange[] { return files .filter( ({ filename, previous_filename }) => - TEST_FILE_RE.test(filename) || TEST_FILE_RE.test(previous_filename ?? ""), + TEST_SOURCE_FILE_RE.test(filename) || TEST_SOURCE_FILE_RE.test(previous_filename ?? ""), ) .map((file) => ({ - basePath: TEST_FILE_RE.test(file.previous_filename ?? "") + basePath: TEST_SOURCE_FILE_RE.test(file.previous_filename ?? "") ? (file.previous_filename as string) : file.filename, headPath: - file.status === "removed" || !TEST_FILE_RE.test(file.filename) ? null : file.filename, + file.status === "removed" || !TEST_SOURCE_FILE_RE.test(file.filename) + ? null + : file.filename, displayName: file.filename, })); } @@ -347,7 +348,7 @@ export async function testSizeViolations(diff: GrowthGuardrailDiff): Promise status !== "removed" && TEST_SIZE_FILE_RE.test(filename)) + .filter(({ filename, status }) => status !== "removed" && TEST_SOURCE_FILE_RE.test(filename)) .map(({ filename }) => filename); const baseBudgetBlob = await diff.readBase([BUDGET_FILE]); const baseBudget = parseBudget(baseBudgetBlob.get(BUDGET_FILE) ?? FALLBACK_BUDGET, "base budget"); diff --git a/test/helpers/onboard-messaging-suite.ts b/test/helpers/onboard-messaging-suite.ts index ae05fbd4028..b4f7d5785d6 100644 --- a/test/helpers/onboard-messaging-suite.ts +++ b/test/helpers/onboard-messaging-suite.ts @@ -20,6 +20,7 @@ import { } from "./messaging-plan-fixtures"; import { runBoundedOnboardScript } from "./onboard-child-process-harness"; import { writeOkOpenshell } from "./onboard-openshell-fixture"; +import { registerTestGroup } from "./test-group-registration"; type CommandEntry = { command: string; @@ -46,7 +47,7 @@ export type OnboardMessagingGroup = "lifecycle" | "provider"; export function registerOnboardMessagingTests(group: OnboardMessagingGroup): void { describe(`onboard messaging (${group})`, () => { - if (group === "provider") { + registerTestGroup(group, "provider", () => { it( "creates providers for messaging tokens and attaches them to the sandbox", { @@ -306,9 +307,9 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); ); }, ); - } + }); - if (group === "lifecycle") { + registerTestGroup(group, "lifecycle", () => { it( "preserves Hermes Slack policy when Slack is active at sandbox create time", { @@ -853,9 +854,9 @@ const { createSandbox } = require(${onboardPath}); ); }, ); - } + }); - if (group === "provider") { + registerTestGroup(group, "provider", () => { it("aborts onboard when a messaging provider upsert fails", { timeout: 60_000 }, async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-provider-fail-")); const fakeBin = path.join(tmpDir, "bin"); @@ -1183,6 +1184,6 @@ const { createSandbox } = require(${onboardPath}); assert.doesNotMatch(createCommand.command, /my-assistant-slack-bridge/); }, ); - } + }); }); } diff --git a/test/helpers/onboard-sandbox-build-suite.ts b/test/helpers/onboard-sandbox-build-suite.ts index e6b8f646640..78ee9d3a6c7 100644 --- a/test/helpers/onboard-sandbox-build-suite.ts +++ b/test/helpers/onboard-sandbox-build-suite.ts @@ -15,6 +15,7 @@ import { parseStdoutJson, stripMessagingEnv, } from "./onboard-split-context"; +import { registerTestGroup } from "./test-group-registration"; beforeEach(() => { vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); @@ -26,7 +27,7 @@ export type OnboardSandboxBuildGroup = "base" | "dashboard"; export function registerOnboardSandboxBuildTests(group: OnboardSandboxBuildGroup): void { describe(`onboard helpers (${group})`, () => { - if (group === "base") { + registerTestGroup(group, "base", () => { it( "creates the stock managed sandbox without uploading an external OpenClaw config file", { @@ -605,9 +606,9 @@ const { createSandbox } = require(${onboardPath}); "stock managed-image onboarding must not enter sandbox-base resolution", ); }); - } + }); - if (group === "dashboard") { + registerTestGroup(group, "dashboard", () => { it("defers a remote dashboard forward until post-create recovery", async () => { const repoRoot = path.join(import.meta.dirname, "../.."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-remote-forward-")); @@ -897,6 +898,6 @@ const { createSandbox } = require(${onboardPath}); "sandbox creation must defer the custom-port forward until agent setup or final recovery", ); }); - } + }); }); } diff --git a/test/helpers/test-group-registration.ts b/test/helpers/test-group-registration.ts new file mode 100644 index 00000000000..bfbc66d2f68 --- /dev/null +++ b/test/helpers/test-group-registration.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function registerTestGroup( + selected: Group, + expected: Group, + register: () => void, +): void { + if (selected === expected) register(); +}