diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index 6b54aa5e7ec..56962b1681c 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -136,7 +136,7 @@ npm run release:e2e-evidence -- \ The preflight derives every required execution from one empty-selector dispatch. The full run includes every default-selected workflow E2E plus `Exact staging Brev Launchable`. -Accepted release evidence requires `allow_jetson_runner_queue=false` and `allow_dgx_spark_runner_queue=false`. +Accepted release evidence requires `allowJetsonDispatch: false`, `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`. The required denominator excludes `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification`. Each job that declares `RELEASE_E2E_ACTIVATION_PATH` requires that path at the candidate SHA. A missing activation path is a preflight failure. @@ -148,7 +148,7 @@ Monitor the dispatched correlation ID with one bounded status query. Before accepting full-mode exact Brev evidence, require: - the workflow `head_sha` to equal the plan candidate SHA; -- the trusted dispatch receipt to prove empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`; +- the trusted dispatch receipt to prove empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`; a v1 receipt may omit `allowJetsonDispatch` but must set it to `false` when present, while a v2 receipt must include `allowJetsonDispatch: false`; - the workflow conclusion to be `success`; - the `Exact staging Brev Launchable` job conclusion to be `success`; - the job URL and selected successful Launchable job attempt; diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts index 5c3f9dd07d9..0eba947839a 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts @@ -484,6 +484,16 @@ export function buildReleaseE2eLedger( false, `${label}.dispatch.allowJetsonRunnerQueue`, ); + if ( + dispatch.kind === "nemoclaw-e2e-dispatch-v2" || + Object.hasOwn(dispatch, "allowJetsonDispatch") + ) { + requireEqual( + booleanField(dispatch, "allowJetsonDispatch", `${label}.dispatch`), + false, + `${label}.dispatch.allowJetsonDispatch`, + ); + } requireEqual( booleanField(dispatch, "allowDgxSparkRunnerQueue", `${label}.dispatch`), false, diff --git a/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md b/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md index a7227e17e13..6bac644d45d 100644 --- a/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md +++ b/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md @@ -85,7 +85,7 @@ After a failure, inspect the artifacts and remove resources that target cleanup Dispatch the trusted `main` workflow with the current PR number, lowercase 40-character head SHA, head repository, lowercase 40-character base SHA, trusted workflow SHA, and a review reason containing 10 to 500 printable characters. Leave job and target selectors empty and keep Launchable disabled. -Keep `allow_jetson_runner_queue=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. +Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If the DGX Spark flag is `true`, GitHub can pause `llama-cpp-dgx-spark-qualification` for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. The trusted pre-checkout step requires current `maintain` or `admin` access and validates the exact open PR before candidate code runs. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 2cdc2155007..8f5adc3dbe7 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -67,7 +67,8 @@ Choose exactly one mode: - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. - The run skips `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` unless their separate runner-queue flags are `true`. + The run skips `jetson-nvmap-gpu` unless its Colossus dispatch flag is `true`. + It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. - For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. Leave `targets` empty and keep Launchable disabled: @@ -81,7 +82,7 @@ esac REVIEW_REASON='Reviewed the PR head commit for credentialed E2E.' CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" INFERENCE_MODE=mock -ALLOW_JETSON_RUNNER_QUEUE=false +ALLOW_JETSON_DISPATCH=false ALLOW_DGX_SPARK_RUNNER_QUEUE=false gh workflow run .github/workflows/e2e.yaml \ --repo NVIDIA/NemoClaw \ @@ -90,7 +91,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f "jobs=${E2E_JOBS}" \ -f "inference_mode=${INFERENCE_MODE}" \ -f include_staging_brev_launchable=false \ - -f "allow_jetson_runner_queue=${ALLOW_JETSON_RUNNER_QUEUE}" \ + -f "allow_jetson_dispatch=${ALLOW_JETSON_DISPATCH}" \ -f "allow_dgx_spark_runner_queue=${ALLOW_DGX_SPARK_RUNNER_QUEUE}" \ -f "pr_number=${PR_NUMBER}" \ -f "checkout_sha=${HEAD_SHA}" \ @@ -162,7 +163,7 @@ Ask for clarification only when the request contains conflicting mode phrases. Ordinary mode selects every default-selected workflow E2E except `Exact staging Brev Launchable`. Launchable mode runs only `Exact staging Brev Launchable`. Full mode adds `Exact staging Brev Launchable` to the default E2E selection in the same workflow run. -The documented invocations for all three modes keep both hardware runner-queue flags set to `false`. +The documented invocations for all three modes keep the Jetson dispatch and DGX Spark runner-queue flags set to `false`. ## Resolve the Candidate @@ -199,7 +200,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f jobs= \ -f inference_mode=mock \ -f include_staging_brev_launchable=false \ - -f allow_jetson_runner_queue=false \ + -f allow_jetson_dispatch=false \ -f allow_dgx_spark_runner_queue=false \ -f "correlation_id=${CORRELATION_ID}" ``` @@ -214,7 +215,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f jobs=staging-brev-launchable \ -f inference_mode=mock \ -f include_staging_brev_launchable=false \ - -f allow_jetson_runner_queue=false \ + -f allow_jetson_dispatch=false \ -f allow_dgx_spark_runner_queue=false \ -f "correlation_id=${CORRELATION_ID}" ``` @@ -229,7 +230,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f jobs= \ -f inference_mode=mock \ -f include_staging_brev_launchable=true \ - -f allow_jetson_runner_queue=false \ + -f allow_jetson_dispatch=false \ -f allow_dgx_spark_runner_queue=false \ -f "correlation_id=${CORRELATION_ID}" ``` @@ -240,11 +241,14 @@ The `include_staging_brev_launchable` input adds the Launchable E2E job to that The trusted `main` workflow verifies that the dispatching and rerunning actors have repository `maintain` or `admin` permission before the Launchable path's source checkout. That role check is the authorization. -A user permitted to dispatch this workflow may set -`allow_jetson_runner_queue=true` to add `jetson-nvmap-gpu` to an empty-selector -manual run or enable its explicit selection. Set it only after a repository -administrator confirms an online Jetson runner in the authoritative runner -inventory. +A user permitted to dispatch this workflow may set `allow_jetson_dispatch=true` +to add `jetson-nvmap-gpu` to an empty-selector manual run or enable its explicit +selection. Set it only after every deployment check in +[Jetson Dispatch Through Colossus](../../../test/e2e/docs/jetson-colossus-dispatch.md) +passes. +The Jetson lifecycle removes and verifies only its fixed job-owned cleanup +allowlist. Do not treat `cleanup: "succeeded"` as evidence that every possible +candidate host change was reversed. A permitted dispatcher may set `allow_dgx_spark_runner_queue=true` to add `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` to an empty-selector manual run or enable explicit qualification selection. Set it @@ -350,6 +354,8 @@ The validator requires: - the workflow run to succeed for the selected SHA; - `dispatch.json` to bind the same run, empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, `allowDgxSparkRunnerQueue: false`, and the selected successful Launchable job attempt; +- `allowJetsonDispatch: false` in every v2 `dispatch.json` receipt; +- `allowJetsonDispatch` to be absent or `false` in every v1 `dispatch.json` receipt; - `Exact staging Brev Launchable` to conclude `success` in the selected current or earlier attempt of the same workflow run; - `launchable-e2e.json` to identify the selected SHA in the repository and provision records; - the booted repository to be unmodified; @@ -376,7 +382,10 @@ If the release candidate SHA changes, discard the earlier full run and dispatch No release-note-only delta exception is currently defined. When `nemoclaw-maintainer-cut-release-tag` invokes this skill, return the validated fields for its pre-tag E2E evidence ledger. -The trusted `dispatch.json` receipt proves that full mode used empty selectors, included `Exact staging Brev Launchable`, and disabled both optional hardware paths. +The trusted `dispatch.json` receipt proves that full mode used empty selectors and included `Exact staging Brev Launchable`. +For Jetson dispatch, a v2 receipt requires `allowJetsonDispatch: false`. +A v1 receipt may omit `allowJetsonDispatch`, but it must be `false` when present. +Both receipt versions require the optional DGX Spark runner path to be disabled. The release evidence ledger proves the result of each workflow E2E. Do not ask for the release confirmation phrase in this skill. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts index 1bb2a540b1d..f38b36ae847 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts +++ b/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts @@ -30,6 +30,7 @@ export interface FullE2eEvidenceSummary { }; dispatch: { allowDgxSparkRunnerQueue: false; + allowJetsonDispatch: false; allowJetsonRunnerQueue: false; emptySelectors: true; includeStagingBrevLaunchable: true; @@ -166,6 +167,9 @@ export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvi "dispatch.includeStagingBrevLaunchable", ); requireEqual(dispatch.allowJetsonRunnerQueue, false, "dispatch.allowJetsonRunnerQueue"); + if (dispatch.kind === "nemoclaw-e2e-dispatch-v2" || Object.hasOwn(dispatch, "allowJetsonDispatch")) { + requireEqual(dispatch.allowJetsonDispatch, false, "dispatch.allowJetsonDispatch"); + } requireEqual(dispatch.allowDgxSparkRunnerQueue, false, "dispatch.allowDgxSparkRunnerQueue"); requireEqual(dispatch.emptySelectors, true, "dispatch.emptySelectors"); @@ -233,6 +237,7 @@ export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvi }, dispatch: { allowDgxSparkRunnerQueue: false, + allowJetsonDispatch: false, allowJetsonRunnerQueue: false, emptySelectors: true, includeStagingBrevLaunchable: true, diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index 36f05b574cf..0e98df5b46e 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -57,7 +57,7 @@ Before asking for the release confirmation phrase, build and show an evidence le - Require the workflow-produced trusted dispatch receipt to bind the accepted run candidate SHA, run ID, attempt, and selector inputs. - Run `nemoclaw-maintainer-e2e` in full mode when the ledger lacks complete evidence for the candidate SHA. - Require one completed, successful full workflow run for all default-selected workflow E2E jobs and the full-mode additions, including `Exact staging Brev Launchable`. -- Require the trusted dispatch receipt to record `allowJetsonRunnerQueue: false` and `allowDgxSparkRunnerQueue: false`. +- Require the trusted dispatch receipt to record `allowJetsonRunnerQueue: false` and `allowDgxSparkRunnerQueue: false`. A v1 receipt may omit `allowJetsonDispatch` but must record `false` when the field is present. A v2 receipt must record `allowJetsonDispatch: false`. - Exclude `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` from the required denominator. - Require the trusted dispatch receipt to bind the workflow run and an attempt no later than the run's latest attempt. The receipt must record empty selectors and `include_staging_brev_launchable=true`. - Require the Launchable E2E receipt to identify the candidate SHA in the repository and provision records. diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 297e82fa3ce..abe58ecf09a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -15,7 +15,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson and DGX Spark require their queue flags. PR revisions use the trusted controller matrix." + description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix." required: false default: "" type: string @@ -33,8 +33,8 @@ on: - mock - internal-nvidia - public-nvidia - allow_jetson_runner_queue: - description: "Before setting true for jetson-nvmap-gpu, obtain repository administrator confirmation of an online Jetson runner in the authoritative NVIDIA/NemoClaw Settings -> Actions -> Runners inventory; queued jobs do not honor timeout-minutes before assignment." + allow_jetson_dispatch: + description: "Set true for jetson-nvmap-gpu only after the Colossus dispatcher, Cloudflare Tunnel, cleanup helper, and JETSON_DISPATCH_URL repository variable have passed the deployment checks in test/e2e/docs/jetson-colossus-dispatch.md." required: false default: false type: boolean @@ -92,7 +92,7 @@ permissions: concurrency: group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && format('full-{0}', github.run_id)) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }} - cancel-in-progress: ${{ inputs.checkout_sha != '' }} + cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} @@ -179,7 +179,7 @@ jobs: matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]' test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/vllm-docker-storage.test.ts","project":"integration"}]' ;; - inference-routing: | managed-image-protected-runtime:) + inference-routing: | managed-image-protected-runtime: | :jetson-nvmap-gpu) matrix='[]' ;; :ubuntu-repo-cloud-langchain-deepagents-code) @@ -224,6 +224,7 @@ jobs: if: ${{ inputs.checkout_sha != '' }} env: ACTOR: ${{ github.actor }} + ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} @@ -268,10 +269,10 @@ jobs: exit 1 } [[ "$RUN_ATTEMPT" == "1" ]] || { echo "::error::Manual PR E2E cannot be rerun" >&2; exit 1; } - case "${JOBS}:${TARGETS}:${INCLUDE_LAUNCHABLE}" in - ::false | inference-routing::false | managed-image-protected-runtime::false) ;; + case "${JOBS}:${TARGETS}:${INCLUDE_LAUNCHABLE}:${ALLOW_JETSON_DISPATCH}" in + ::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | :jetson-nvmap-gpu:false:true) ;; *) - echo "::error::Manual PR E2E accepts only empty selectors, inference-routing, or managed-image-protected-runtime without target or Launchable selectors" >&2 + echo "::error::Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, or jetson-nvmap-gpu with its dispatch flag" >&2 exit 1 ;; esac @@ -286,7 +287,7 @@ jobs: require_maintainer "$ACTOR" - if [[ "${TRIGGERING_ACTOR,,}" != "${ACTOR,,}" ]]; then + if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then require_maintainer "$TRIGGERING_ACTOR" fi @@ -304,7 +305,8 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} env: ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }} - ALLOW_JETSON_RUNNER_QUEUE: ${{ inputs.allow_jetson_runner_queue && 'true' || 'false' }} + ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} + ALLOW_JETSON_RUNNER_QUEUE: "false" BASE_SHA: ${{ inputs.checkout_sha != '' && inputs.base_sha || github.sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} @@ -334,6 +336,7 @@ jobs: --arg workflowRunId "$RUN_ID" \ --arg workflowSha "$WORKFLOW_SHA" \ --argjson allowDgxSparkRunnerQueue "$ALLOW_DGX_SPARK_RUNNER_QUEUE" \ + --argjson allowJetsonDispatch "$ALLOW_JETSON_DISPATCH" \ --argjson allowJetsonRunnerQueue "$ALLOW_JETSON_RUNNER_QUEUE" \ --argjson includeStagingBrevLaunchable "$INCLUDE_STAGING_BREV_LAUNCHABLE" \ --argjson workflowRunAttempt "$RUN_ATTEMPT" \ @@ -351,6 +354,7 @@ jobs: jobs: $jobs, targets: $targets, allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue, + allowJetsonDispatch: $allowJetsonDispatch, allowJetsonRunnerQueue: $allowJetsonRunnerQueue, includeStagingBrevLaunchable: $includeStagingBrevLaunchable, emptySelectors: ($jobs == "" and $targets == "") @@ -400,7 +404,7 @@ jobs: } require_maintainer "$ACTOR" - if [[ "${TRIGGERING_ACTOR,,}" != "${ACTOR,,}" ]]; then + if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then require_maintainer "$TRIGGERING_ACTOR" fi @@ -4918,70 +4922,42 @@ jobs: jetson-nvmap-gpu: needs: generate-matrix - # The opt-in flag prevents assignment to an unavailable Jetson runner. - if: ${{ inputs.allow_jetson_runner_queue && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && ((github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',jetson-nvmap-gpu,') || contains(format(',{0},', inputs.targets), ',jetson-nvmap-gpu,')) }} - runs-on: ${{ vars.JETSON_E2E_RUNNER_LABEL || 'linux-arm64-gpu-jetson-orin-latest-1' }} + if: ${{ inputs.allow_jetson_dispatch && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && ((inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',jetson-nvmap-gpu,') || contains(format(',{0},', inputs.targets), ',jetson-nvmap-gpu,')) }} + concurrency: + group: jetson-nvmap-gpu-colossus + queue: max + cancel-in-progress: false + runs-on: ubuntu-latest timeout-minutes: 60 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "jetson-nvmap-gpu" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/jetson-nvmap-gpu - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-jetson-nvmap" - NEMOCLAW_RECREATE_SANDBOX: "1" - NEMOCLAW_PROVIDER: "ollama" - OPENSHELL_GATEWAY: "nemoclaw" + permissions: + contents: read + id-token: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check out trusted Jetson controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} + repository: NVIDIA/NemoClaw + ref: ${{ github.workflow_sha }} persist-credentials: false - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - with: - build-cli: "false" - - - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3 + - name: Set up Node for Jetson controller + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - - - name: Verify Jetson GPU availability - run: | - set -euo pipefail - echo "=== Tegra release ===" - cat /etc/nv_tegra_release 2>/dev/null || echo "(no /etc/nv_tegra_release)" - echo "" - echo "=== /dev/nvmap ===" - ls -l /dev/nvmap 2>/dev/null || echo "(no /dev/nvmap)" - echo "" - echo "=== Docker ===" - docker info --format '{{.ServerVersion}}' - docker info --format '{{json .Runtimes}}' + node-version: 22 - - name: Run Jetson nvmap GPU live Vitest test - # Keeps the - # Jetson/Tegra host, NVIDIA Docker runtime, install.sh, OpenShell - # sandbox exec, /dev/nvmap, CUDA cuInit(0), and status-proof boundary. - run: | - set -euo pipefail - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/jetson-nvmap-gpu.test.ts + - name: Dispatch exact commit to Jetson through Colossus + env: + E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu + JETSON_DISPATCH_CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} + JETSON_DISPATCH_URL: ${{ vars.JETSON_DISPATCH_URL }} + run: node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-client.mts - name: Upload Jetson nvmap GPU artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh + with: + name: e2e-jetson-nvmap-gpu + path: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/ concurrent-gateway-ports: needs: generate-matrix diff --git a/scripts/checks/verify-openshell-e2e-qualification.mts b/scripts/checks/verify-openshell-e2e-qualification.mts index dacf1bf4150..742def53fa1 100644 --- a/scripts/checks/verify-openshell-e2e-qualification.mts +++ b/scripts/checks/verify-openshell-e2e-qualification.mts @@ -179,6 +179,7 @@ export type PullRequestIdentity = { export type E2EDispatchReceipt = { allowDgxSparkRunnerQueue: boolean; + allowJetsonDispatch: boolean; allowJetsonRunnerQueue: boolean; baseSha: string; candidateRepository: string; @@ -199,6 +200,7 @@ export type E2EDispatchReceipt = { type E2EDispatchReceiptEnvelope = Omit< E2EDispatchReceipt, | "allowDgxSparkRunnerQueue" + | "allowJetsonDispatch" | "allowJetsonRunnerQueue" | "emptySelectors" | "includeStagingBrevLaunchable" @@ -206,6 +208,7 @@ type E2EDispatchReceiptEnvelope = Omit< | "targets" > & { allowDgxSparkRunnerQueue: boolean; + allowJetsonDispatch: boolean; allowJetsonRunnerQueue: boolean; emptySelectors: boolean; includeStagingBrevLaunchable: boolean; @@ -728,6 +731,7 @@ export function validateDispatchReceipt( receipt.targets !== "" || receipt.emptySelectors !== true || receipt.allowDgxSparkRunnerQueue !== false || + receipt.allowJetsonDispatch !== false || receipt.allowJetsonRunnerQueue !== false || receipt.includeStagingBrevLaunchable !== false ) { @@ -745,6 +749,7 @@ function validateDispatchReceiptEnvelope( value, [ "allowDgxSparkRunnerQueue", + "allowJetsonDispatch", "allowJetsonRunnerQueue", "baseSha", "candidateRepository", @@ -780,6 +785,7 @@ function validateDispatchReceiptEnvelope( typeof value.emptySelectors !== "boolean" || value.emptySelectors !== (value.jobs === "" && value.targets === "") || typeof value.allowDgxSparkRunnerQueue !== "boolean" || + typeof value.allowJetsonDispatch !== "boolean" || typeof value.allowJetsonRunnerQueue !== "boolean" || typeof value.includeStagingBrevLaunchable !== "boolean" ) { @@ -987,6 +993,7 @@ async function selectQualifiedRun(options: { receipt.targets !== "" || !receipt.emptySelectors || receipt.allowDgxSparkRunnerQueue || + receipt.allowJetsonDispatch || receipt.allowJetsonRunnerQueue || receipt.includeStagingBrevLaunchable ) { diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts index bf313088e62..a61a5a23ce8 100644 --- a/test/e2e-private-file.test.ts +++ b/test/e2e-private-file.test.ts @@ -18,6 +18,7 @@ describe("private E2E controller files", () => { it("writes private regular files without following links or truncating hardlink targets", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-file-")); const regular = path.join(directory, "regular.json"); + const binary = path.join(directory, "artifact.tar.gz"); const target = path.join(directory, "target.json"); const symlink = path.join(directory, "symlink.json"); const hardlink = path.join(directory, "hardlink.json"); @@ -28,6 +29,9 @@ describe("private E2E controller files", () => { writePrivateRegularFile(regular, "updated\n"); appendPrivateRegularFile(regular, "appended\n", { maxBytes: 64 }); expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("updated\nappended\n"); + writePrivateRegularFile(binary, Uint8Array.from([0, 255, 31, 139])); + expect(fs.readFileSync(binary)).toEqual(Buffer.from([0, 255, 31, 139])); + expect(fs.statSync(binary).mode & 0o777).toBe(0o600); fs.writeFileSync(target, "protected\n"); fs.symlinkSync(target, symlink); diff --git a/test/e2e/README.md b/test/e2e/README.md index 8eddccba21c..1397870f5e1 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -455,17 +455,19 @@ environment approval. The job uses the non-cancelling runs remain queued instead of replacing one another. The Jetson nvmap and DGX Spark llama.cpp jobs remain excluded from ordinary and -full runs unless their independent runner-queue flags are `true`. -A user permitted to dispatch this workflow may set either flag, but only after -a repository administrator confirms the corresponding runner is online in the -authoritative repository runner inventory. -Set `allow_jetson_runner_queue=true` to select `jetson-nvmap-gpu`. +full runs unless their independent opt-in flags are `true`. +Set `allow_jetson_dispatch=true` to select `jetson-nvmap-gpu` only after every +deployment check in +[Jetson Dispatch Through Colossus](docs/jetson-colossus-dispatch.md) passes. +The Colossus lifecycle removes and verifies only the fixed job-owned cleanup +allowlist. Its cleanup evidence does not attest that every possible candidate +host change was reversed. Set `allow_dgx_spark_runner_queue=true` to select both `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification`. GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment before it reaches the DGX Spark runner. -Pre-tag evidence requires both runner-queue flags to remain `false`. +Pre-tag evidence requires both hardware opt-in flags to remain `false`. Results from opt-in hardware runs do not enter the required pre-tag E2E denominator. @@ -796,9 +798,9 @@ For a PR revision run, a repository maintainer or administrator leaves `jobs` an - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. -The run skips `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and -`llama-cpp-dgx-spark-qualification` unless their separate runner-queue flags -are `true`. +The run skips `jetson-nvmap-gpu` unless its Colossus dispatch flag is `true`. +It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` +unless their runner-queue flag is `true`. The trusted workflow definition remains on `main` and binds the candidate head to the current PR base SHA. It does not run GitHub's synthetic merge commit. @@ -827,7 +829,7 @@ For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVI For a manual PR run, provide the current PR number, lowercase 40-character head SHA, head repository, lowercase 40-character base SHA, trusted `main` workflow SHA, and a review reason containing 10 to 500 printable characters. Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. -Keep `allow_jetson_runner_queue=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. +Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. To select the protected managed-image runtime qualification, set `jobs=managed-image-protected-runtime`. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 3c573288b7f..8f17b23e4ca 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -306,7 +306,7 @@ test/e2e/ `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. Keep - `allow_jetson_runner_queue=false` and `allow_dgx_spark_runner_queue=false` for + `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for this default selection. If the DGX Spark flag is `true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. @@ -314,6 +314,11 @@ test/e2e/ `jobs` value is `managed-image-protected-runtime`; `targets` must remain empty. Refer to [NemoClaw E2E CI](../README.md). +- [Jetson Dispatch Through Colossus](jetson-colossus-dispatch.md) defines how + maintainers deploy and verify the temporary authenticated controller, + dispatcher, SSH, bounded cleanup, and stale-lock recovery path for + `jetson-nvmap-gpu`. + - `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist with JSON summaries plus action, log, and shell command-evidence directories under diff --git a/test/e2e/docs/jetson-colossus-dispatch.md b/test/e2e/docs/jetson-colossus-dispatch.md new file mode 100644 index 00000000000..87ad9e1d0b1 --- /dev/null +++ b/test/e2e/docs/jetson-colossus-dispatch.md @@ -0,0 +1,1236 @@ + + + +# Jetson Dispatch Through Colossus + +This temporary path runs the `jetson-nvmap-gpu` end-to-end (E2E) job on one Jetson behind a Colossus host. +GitHub Actions controls the job through an authenticated HTTPS endpoint. +Colossus retains the Cloudflare Tunnel credential, Jetson SSH key, SSH host key, device lock, and cleanup capability. +Candidate code runs only on the Jetson. + +Do not set `allow_jetson_dispatch=true` until every deployment check on this page passes. + +## Trust Boundary + +The deployment has these input and credential boundaries: + +| System | Receives | Must not receive | +| --- | --- | --- | +| GitHub-hosted controller | Candidate commit SHA, workflow run identity, public dispatch URL, and short-lived GitHub OpenID Connect (OIDC) token | Jetson SSH key, tunnel credential, and cleanup privilege | +| Colossus dispatcher | Validated workflow identity, fixed target, and candidate commit SHA | Request-controlled command, SSH host, repository, path, or cleanup command | +| Jetson | Candidate checkout and the fixed live E2E command | GitHub OIDC token, tunnel credential, and Colossus SSH private key | + +The dispatcher accepts only `NVIDIA/NemoClaw` and the trusted `main` E2E workflow. +It requires a GitHub-hosted controller, one repository ID, one workflow run identity, and the `jetson-nvmap-gpu` target. +It also requires a lowercase 40-character candidate commit SHA. +The temporary path rejects fork repositories. + +The credentials have these lifecycles: + +| Credential | Location and access | Lifetime and removal | +| --- | --- | --- | +| GitHub OIDC token | GitHub-hosted controller process memory and the authenticated request | The client reuses a token in process memory for at most four minutes and then requests another. It does not write the token to disk. | +| Cloudflare account certificate | The administrator account that runs `cloudflared tunnel login` | Remove `cert.pem` from Colossus after tunnel creation and DNS routing. Reauthenticate before later management changes. | +| Cloudflare Tunnel credential | `/etc/cloudflared/TUNNEL_UUID.json`, readable only by the tunnel service account | Keep it for this temporary deployment. Revoke the tunnel and remove the file when the deployment ends. | +| Jetson SSH private key | `/var/lib/nemoclaw-jetson-dispatch/id_ed25519`, readable only by the dispatcher service account | Keep it for this temporary deployment. Remove the matching Jetson public key and private key when the deployment ends. | + +The dispatcher rejects an OIDC token whose issued-to-expiry window exceeds 15 minutes. +The GitHub repository variable contains only the public dispatch URL. +Do not put a credential in `JETSON_DISPATCH_URL`. + +## Prepare the Dedicated Jetson + +Use a dedicated Jetson without production data or credentials. +The E2E job gives candidate code Docker access, which can control the dedicated host. +Cleanup removes only the fixed job-owned resources defined below. +It does not attest that cleanup reversed every possible host change made by candidate code. + +Run these checks as the `nvidia` account that the dispatcher uses: + +```bash +set -euo pipefail +uname -m +tr -d '\0' /dev/null 2>&1; then + echo 'OpenShell must be absent from the prepared Jetson' >&2 + exit 1 + fi + for host_bin in \ + "/usr/local/bin/$openshell_component" \ + "/usr/bin/$openshell_component" \ + "$HOME/.local/bin/$openshell_component"; do + test ! -e "$host_bin" && test ! -L "$host_bin" + done +done +ollama list +docker info --format '{{json .Runtimes}}' +test -c /dev/nvmap +if sudo -n true 2>/dev/null; then echo 'unexpected passwordless sudo'; exit 1; fi +``` + +The architecture must be `aarch64`. +Node.js must be version 22.19.0 or later, and npm must have major version 10 or later. +OpenShell must be absent from the host `PATH` and the three checked host binary directories. +Do not preinstall OpenShell on the Jetson. +The `ollama list` command must succeed. +Docker must expose the NVIDIA runtime required by the existing Jetson live E2E test. +Preinstall Ollama so candidate code does not invoke its host installer. +The `nvidia` account must not have passwordless `sudo` access. + +The worker creates `/var/tmp/nemoclaw-jetson-e2e/` and sets these job-local paths: + +- `HOME=/home` +- `TMPDIR=/tmp` +- `XDG_CACHE_HOME=/home/.cache` +- `XDG_CONFIG_HOME=/home/.config` +- `XDG_DATA_HOME=/home/.local/share` +- `XDG_STATE_HOME=/home/.local/state` +- `XDG_BIN_HOME=/home/.local/bin` +- `XDG_RUNTIME_DIR=/runtime` +- `npm_config_prefix=/npm-prefix` +- `PATH=$XDG_BIN_HOME:/npm-prefix/bin:$PATH` + +The worker unsets `DBUS_SESSION_BUS_ADDRESS` so onboarding uses OpenShell's existing standalone fallback when the job-local systemd user service cannot load. +The worker must not set `NEMOCLAW_DEFER_OPENSHELL_INSTALL`. +It must not invoke `scripts/install-openshell.sh`. +The existing live E2E runs `bash install.sh --non-interactive`. +NemoClaw onboarding owns the compatible pinned OpenShell installation in the job workspace. +After onboarding, `nemoclaw`, `openshell`, `openshell-gateway`, and `openshell-sandbox` must resolve canonically inside the job workspace. +The worker rejects a symbolic link or resolved path that leaves that workspace. + +Reserve these names and paths for this E2E target: + +- The current dispatcher-created `/var/tmp/nemoclaw-jetson-e2e/` workspace. +- The `/tmp/nemoclaw-services-e2e-jetson-nvmap` helper-service directory. +- The `e2e-jetson-nvmap` NemoClaw and OpenShell sandbox name. +- The `nemoclaw` OpenShell gateway name and forwards for the named sandbox. +- The recorded `ollama-auth-proxy`, OpenShell Docker gateway, and helper `cloudflared` processes. +- OpenShell-managed Docker containers labeled for `e2e-jetson-nvmap`. +- The volumes recorded from those labeled containers. +- The `openshell-cluster-nemoclaw` gateway container, volume, and recorded attached volumes. +- `nemoclaw-sandbox-local` images whose tag begins with `e2e-jetson-nvmap-`. + +Do not run unrelated work under these reserved names. +The cleanup program must act only on this allowlist. + +Before candidate execution, the worker records this protected tool and model baseline: + +- The resolved Node.js path and version. +- The resolved npm path and version. +- The resolved Ollama path and the sorted pre-existing Ollama model names and IDs. +- The required absence of host-level `openshell`, `openshell-gateway`, and `openshell-sandbox` binaries. + +The Ollama list accepts at most 64 sorted, unique model name and ID rows whose decoded content is at most 3 KiB. +The SSH probe output is limited to 4 KiB, while the persisted baseline record and its reader are limited to 8 KiB. +Before candidate execution, the worker serializes the baseline and rejects it if persistence would exceed that 8 KiB read bound. +The dispatcher stores the recorded values in a private `.baseline.json` state file. +After cleanup, the worker repeats the probes and requires every pre-existing Ollama model name and ID to remain. +Additional models installed by the job, such as `qwen3.5:9b`, are allowed only while the post-cleanup list stays within the same row and size bounds. +An oversized post-cleanup list fails cleanup and retains the device lock without removing any model. +Before and after candidate execution, the worker also requires `/dev/nvmap` and the Docker NVIDIA runtime to be available. +The worker keeps the record through cleanup and device-lock release so startup recovery can repeat the same comparison. +The host retention policy may remove it only after no device lock exists. +Before a replay of the same job ID, the worker removes the earlier record and writes a new record before it invokes candidate code. +If the initial baseline probe fails, the worker never invokes candidate code; cleanup verifies current prerequisites and allowlisted-resource absence without a before-and-after comparison. + +These other host resources also remain outside the cleanup allowlist: + +- Node.js, npm, and the Docker engine. +- The Ollama binary, service, models, configuration, and unrelated `ollama serve` processes. +- The NVIDIA container runtime and Docker images that the test does not own exclusively. +- The `/dev/nvmap` character device and its permissions. +- JetPack, Jetson Linux, CUDA, NVIDIA packages, other `apt` packages, SDK Manager, and downloaded flashing files. +- User accounts, SSH keys, and user files outside the current job workspace and named NemoClaw resources. +- Docker resources without the exact label, repository tag, or name association defined above. +- Processes other than the recorded job-home helper processes defined above. +- Colossus credentials, service configuration, and job evidence. + +The cleanup program must not change or remove any resource outside its allowlist. +It never removes Ollama models. +Review its command construction and target resolution before deployment. +Do not use broad process termination, Docker pruning, wildcard paths, or host-wide package removal. + +## Prepare the Colossus Dispatcher Account and Deployment Command + +Run these commands on Colossus. +The dispatcher requires `/usr/bin/node` 22.19.0 or later. + +```bash +set -euo pipefail +/usr/bin/node --version +/usr/bin/node -e ' + const [major, minor] = process.versions.node.split(".").map(Number); + if (major < 22 || (major === 22 && minor < 19)) process.exit(1); +' +sudo useradd --system --create-home \ + --home-dir /var/lib/nemoclaw-jetson-dispatch \ + --shell /usr/sbin/nologin nemoclaw-jetson-dispatch +sudo install -d -o nemoclaw-jetson-dispatch -g nemoclaw-jetson-dispatch -m 0700 \ + /var/lib/nemoclaw-jetson-dispatch +``` + +Bootstrap the deployment command from a separately reviewed checkout of `https://github.com/NVIDIA/NemoClaw.git`. +Replace the placeholder with the full lowercase 40-character SHA of that reviewed checkout: + +```bash +set -euo pipefail +REVIEWED_BOOTSTRAP_SHA=0000000000000000000000000000000000000000 +test "$(git remote get-url origin)" = https://github.com/NVIDIA/NemoClaw.git +test "$(git rev-parse HEAD)" = "$REVIEWED_BOOTSTRAP_SHA" +test -z "$(git status --short)" +sudo install -o root -g root -m 0755 \ + tools/e2e/colossus-jetson-dispatch-deploy.sh \ + /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy +test "$(sudo stat --format='%F:%u:%g:%a' -- \ + /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy)" = \ + 'regular file:0:0:755' +``` + +The deployment command requires root and accepts only `--commit `. +It rejects any additional argument. +The command does not update its installed copy. +Install a reviewed deployment-command revision separately when that command changes. + +The Colossus service runs the dispatcher files from the selected reviewed release. +The GitHub-hosted controller runs the client from the trusted workflow commit. +The Colossus service never checks out candidate code. + +Create a dedicated SSH key owned by the service account: + +```bash +sudo -u nemoclaw-jetson-dispatch ssh-keygen -t ed25519 -N '' \ + -f /var/lib/nemoclaw-jetson-dispatch/id_ed25519 +``` + +Add the public key to the Jetson `nvidia` account. +Restrict it to the Colossus source IP address and disable SSH session features that the worker does not use. +For the default USB link, the entry has this shape: + +```text +from="192.168.55.100",restrict ssh-ed25519 AAAA... +``` + +Capture the Jetson host key and verify its fingerprint through the serial console or another trusted channel: + +```bash +set -euo pipefail +umask 077 +known_hosts_tmp="$(mktemp /tmp/nemoclaw-jetson-known-hosts.XXXXXX)" +trap 'rm -f -- "$known_hosts_tmp"' EXIT +ssh-keyscan -T 10 -H 192.168.55.1 >"$known_hosts_tmp" +ssh-keygen -lf "$known_hosts_tmp" +sudo install -o nemoclaw-jetson-dispatch -g nemoclaw-jetson-dispatch -m 0600 \ + "$known_hosts_tmp" /var/lib/nemoclaw-jetson-dispatch/known_hosts +rm -f -- "$known_hosts_tmp" +trap - EXIT +``` + +The pinned `known_hosts` file validates the SSH host identity separately from the persisted cleanup baseline. +Do not accept a changed host key without reconciling it through the serial console or another trusted channel. + +## Define the Cleanup Program + +The deployment command installs `/usr/local/libexec/nemoclaw-jetson-cleanup` as a root-owned stable symbolic link to `/opt/nemoclaw-jetson-dispatch/current/tools/e2e/jetson-dispatch-cleanup.sh`. +The selected target file is owned by `root:root` with mode `0755`. +One atomic `current` switch selects both the dispatcher code and cleanup program, so an interruption cannot select them from different releases. +The deployed worker configuration must name that exact cleanup path. +The worker accepts the symbolic link only when root owns it, its target is the exact managed `current` cleanup program, and that target passes the regular executable ownership and write-permission checks. +It rejects arbitrary cleanup symbolic links and an unsafe managed target. +The deployment command accepts no cleanup path, resource selector, or cleanup-scope override. +Review a release that changes the cleanup allowlist separately before deployment. + +It must accept no arguments and must not be group- or world-writable. +It must derive the lowercase 64-character job ID from the private dispatcher `device.lock`. +It must reject a missing or malformed lock instead of selecting a broader target. +The bundled program fixes the state directory, SSH files, SSH destination, and cleanup names listed on this page. +Changing one of those values requires a corresponding reviewed source change. +The dispatcher invokes it after success, failure, cancellation, and timeout. +The dispatcher also invokes it during startup when `device.lock` remains from an interrupted service process. + +Before destructive cleanup, the program discovers and validates the job-owned Docker volume and process identities. +It accepts a helper or forward PID only when its environment has the exact job `HOME` and its command has one of these markers: + +- `ollama-auth-proxy.` +- `openshell-gateway` +- `openshell-forward` +- `openshell forward` +- `cloudflared` + +Discovery, worker absence verification, and the teardown probe fail closed when the owner, environment, or command of a live same-user process cannot be read from `/proc`. +A process that disappears during inspection or is identified as owned by a different user is ignored. +It merges them into a mode-`0600` private `.cleanup.json` record on Colossus. +The record survives retries so stale-lock recovery can clean and verify the same identities. +The cleanup program must not execute OpenShell or any other executable installed in the job workspace. + +The destructive phase must perform these bounded actions: + +1. Stop recorded helper and forward PIDs after verifying the process owner, exact job `HOME`, and command marker. +2. Remove the exact labeled sandbox containers and exact gateway container. +3. Remove the recorded volumes and reserved test image tags. +4. Remove the helper-service directory and `/var/tmp/nemoclaw-jetson-e2e/` workspace for the locked job. +5. Independently verify that the allowlisted resources are absent, host OpenShell remains absent, the protected tools match, and every pre-existing Ollama model name and ID remains. + +The cleanup program must be idempotent. +It must treat an already absent allowlisted resource as success. +It must exit nonzero when target ownership is ambiguous, cleanup fails, or absence verification is inconclusive. +It must not remove `/var/lib/nemoclaw-jetson-dispatch/state/device.lock`. +The dispatcher removes that lock only after cleanup and absence verification succeed. + +The private cleanup record uses this path: + +```text +/var/lib/nemoclaw-jetson-dispatch/state/.cleanup.json +``` + +The file has this exact schema: + +```json +{ + "schemaVersion": 1, + "volumes": ["example-volume"], + "processIds": [1234] +} +``` + +Either identity array can be empty. +The helper merges new identities with the existing record and cleans every recorded identity. +Any helper failure or interruption keeps the device lock for startup recovery. +The dispatcher retains the cleanup record after it removes the device lock. + +After the helper succeeds, the worker independently verifies these conditions over pinned-host-key SSH: + +- The validated job workspace is absent and is not a symbolic link. +- No OpenShell-managed container has the `e2e-jetson-nvmap` sandbox label. +- The `openshell-cluster-nemoclaw` gateway container and volume are absent. +- No `nemoclaw-sandbox-local` image has a tag that begins with `e2e-jetson-nvmap-`. +- Every Docker volume and process ID in the merged cleanup record is absent. +- Every recorded Node.js, npm, and Ollama tool value matches, and every pre-existing Ollama model name and ID remains. +- No host-level `openshell`, `openshell-gateway`, or `openshell-sandbox` binary resolves or exists in a checked host binary path. +- `/dev/nvmap` exists, and Docker still reports the NVIDIA runtime. + +The worker reports cleanup failure when any helper or independent verification step fails. + +The dispatcher runs the cleanup program as `nemoclaw-jetson-dispatch`. +The managed service unit prevents privilege elevation. +Do not remove that protection or grant the dispatcher account passwordless `sudo` access. + +## Configure the Dispatcher Service + +Review these fixed files in the selected commit before deployment: + +- `tools/e2e/colossus-jetson-dispatch.environment` +- `tools/e2e/nemoclaw-jetson-dispatch.service` + +The environment fixes the repository ID, state path, timeouts, Jetson SSH destination, pinned SSH files, and cleanup executable. +The unit fixes the service account, selected-release working directory, Node.js command, restart policy, and filesystem restrictions. +It sets `TimeoutStopSec=360` so systemd allows the bounded cleanup interval during service stop. + +For an initial deployment, the dispatcher service must have `LoadState=not-found`. +The device lock, destination environment file, and destination unit file must be absent. +The Cloudflare Tunnel service must be absent or both disabled and inactive. + +Replace the placeholder with the full lowercase 40-character SHA of the reviewed release, then run one command: + +```bash +sudo nemoclaw-colossus-jetson-dispatch-deploy --commit 0000000000000000000000000000000000000000 +``` + +The command reports five stages: + +1. Validate the request, service-owned home, dispatcher account, SSH key, pinned host key, and Node.js version. +2. Verify the absent service, device lock, destination files, and public ingress. +3. Fetch and verify the exact commit from `https://github.com/NVIDIA/NemoClaw.git`. +4. Select the dispatcher and cleanup code with one atomic `current` switch. +5. Install, enable, start, and verify the dispatcher service. + +The command accepts an absent cleanup link or verifies the exact managed link before release selection. +It stores each verified release at `/opt/nemoclaw-jetson-dispatch/releases/`. +Each release directory must be owned by `root:root` with mode `0755`. +The command verifies the fixed origin, exact commit, unmodified worktree, cleanup program, environment, and unit before selection. +It installs `/etc/nemoclaw-jetson-dispatch/environment` as `root:root` mode `0600`. +It installs `/etc/systemd/system/nemoclaw-jetson-dispatch.service` as `root:root` mode `0644`. +It runs `systemctl daemon-reload` and `systemctl enable --now nemoclaw-jetson-dispatch.service`. +It requires the Cloudflare Tunnel service to be absent or disabled and inactive immediately before the dispatcher starts, immediately after it starts, and after loopback verification. +Before success, it also requires the dispatcher to listen only on `127.0.0.1:8787` and an anonymous job request to return HTTP `401`. + +If initial startup or verification fails after enablement, the command attempts to disable and stop the service before rollback. +It restores the prior release and cleanup selection only after proving exact `ActiveState=inactive` and an absent device lock. +An earlier selection or configuration-installation failure restores the prior selection and removes only files or links created by the attempt. +Rollback removes any environment and unit that the attempt installed, reloads systemd, and requires `LoadState=not-found`. +If it cannot establish these conditions, it reports rollback failure. +If initial activation detects public ingress and rollback cannot disable and stop the dispatcher, its state is inconclusive. +Immediately disable or delete public ingress through the approved Cloudflare administrator path and prove that the public endpoint is unavailable before local recovery. +The command never enables or starts the Cloudflare Tunnel service. +It accepts no cleanup override and does not change the cleanup allowlist. + +After the command returns, test cleanup-program access and the SSH path as the service account: + +```bash +sudo -u nemoclaw-jetson-dispatch test -x \ + /usr/local/libexec/nemoclaw-jetson-cleanup +sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -o BatchMode=yes \ + -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ + -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ + nvidia@192.168.55.1 'uname -m' +``` + +Do not invoke the cleanup program manually without the dispatcher-created device lock and baseline record. +The proof-job procedure below exercises cleanup and verifies its result. + +For independent verification or recovery, repeat the command's loopback checks: + +```bash +set -euo pipefail +test "$(sudo systemctl show --property=ActiveState --value \ + nemoclaw-jetson-dispatch.service)" = active +listeners="$(sudo ss -H -ltn 'sport = :8787')" +test -n "$listeners" +printf '%s\n' "$listeners" | awk '$4 != "127.0.0.1:8787" { exit 1 }' +HTTP_CODE="$( + curl --disable --noproxy '*' --silent --show-error --output /dev/null \ + --write-out '%{http_code}' --max-time 5 --request POST \ + --header 'Content-Type: application/json' \ + --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ + http://127.0.0.1:8787/v1/jobs +)" +test "$HTTP_CODE" = 401 +``` + +A malformed request returns HTTP `400`, and an unknown route returns HTTP `404`. + +## Deploy a Later Reviewed Commit + +This procedure applies after the tunnel has been published once. +Use the installed deployment command for each later reviewed release: + +```bash +( + set -euo pipefail + tunnel_verified=0 + stop_unverified_tunnel() { + original_status=$? + trap - EXIT + if [ "$tunnel_verified" = 1 ]; then + exit "$original_status" + fi + + set +e + sudo systemctl stop nemoclaw-jetson-tunnel.service + stop_status=$? + observed_state="$(sudo systemctl show --property=ActiveState --value \ + nemoclaw-jetson-tunnel.service)" + inspection_status=$? + set -e + + exit_status=$original_status + if [ "$exit_status" -eq 0 ]; then + exit_status=1 + fi + if [ "$inspection_status" -ne 0 ]; then + state_report=unknown + elif [ -n "$observed_state" ]; then + state_report=$observed_state + else + state_report=empty + fi + if [ "$stop_status" -ne 0 ] || [ "$inspection_status" -ne 0 ] || \ + [ "$observed_state" != inactive ]; then + printf 'PUBLIC INGRESS CONTAINMENT FAILED: tunnel stop status=%s; ActiveState=%s; inspection status=%s\n' \ + "$stop_status" "$state_report" "$inspection_status" >&2 + fi + exit "$exit_status" + } + trap stop_unverified_tunnel EXIT + + REVIEWED_COMMIT_SHA=0000000000000000000000000000000000000000 + sudo systemctl stop nemoclaw-jetson-tunnel.service + test "$(sudo systemctl show --property=ActiveState --value \ + nemoclaw-jetson-tunnel.service)" = inactive + sudo /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy \ + --commit "$REVIEWED_COMMIT_SHA" + sudo systemctl start nemoclaw-jetson-tunnel.service + test "$(sudo systemctl show --property=ActiveState --value \ + nemoclaw-jetson-tunnel.service)" = active + PUBLIC_HTTP_CODE="$( + curl --disable --noproxy '*' --silent --show-error --output /dev/null \ + --write-out '%{http_code}' --max-time 10 --request POST \ + --header 'Content-Type: application/json' \ + --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ + https://jetson-e2e.example.com/v1/jobs + )" + test "$PUBLIC_HTTP_CODE" = 401 + tunnel_verified=1 +) +``` + +The tunnel stop and exact `ActiveState=inactive` check happen before the deployer can change the dispatcher release. +The subshell installs an `EXIT` trap before tunnel operations and leaves its `tunnel_verified` sentinel unset until every check succeeds. +On any earlier exit, the trap disables itself, records the tunnel stop status, and independently inspects `ActiveState`. +Only a successful stop, successful inspection, and exact `ActiveState=inactive` confirm local public-ingress containment. +The trap preserves the original nonzero status, or returns `1` for an otherwise successful unverified exit. +When it cannot confirm containment, it reports `PUBLIC INGRESS CONTAINMENT FAILED` with the stop status and observed or unknown state. +The deployer completes only after the dispatcher starts and its local loopback bind and anonymous HTTP `401` checks pass. +Only after that successful return does the block start the tunnel, require `ActiveState=active`, and repeat a config-free, direct anonymous HTTP `401` check through the public endpoint. +The tunnel remains running only when the block sets `tunnel_verified=1` after all three checks succeed. + +It accepts only systemd `LoadState` values `loaded` and `not-found`. +Both cases require `device.lock` to be absent before release preparation. +A loaded service is eligible only when `current` selects one managed release and the stable cleanup link follows `current`. +For a loaded service, the command performs these actions: + +1. Require the Cloudflare Tunnel service to be disabled and inactive, then stop the dispatcher. +2. Query dispatcher `ActiveState` and require the exact value `inactive`. +3. Refuse deployment when `/var/lib/nemoclaw-jetson-dispatch/state/device.lock` remains. +4. Validate the managed `current` release and stable cleanup link, then prepare and verify the requested release. +5. Require the tunnel to remain disabled and inactive, then atomically switch `current`, selecting the requested dispatcher code and cleanup program together. +6. Require the tunnel to remain disabled and inactive immediately before and after starting the dispatcher. +7. Verify that only `127.0.0.1:8787` listens and make a config-free, direct request with `curl --disable --noproxy '*'`; the anonymous job request must return HTTP `401`. +8. Require the tunnel to remain disabled and inactive after loopback verification. + +If the stop or post-stop inspection cannot prove `inactive`, dispatcher state is inconclusive and the command does not prepare or select a release. +If `device.lock` remains after a proven stop, the command exits with the service stopped and does not prepare or select a release. +Do not remove the lock. +Complete the recovery procedure on this page before you rerun deployment. +If the loaded service's pre-stop public-ingress check fails, the previously verified dispatcher remains running and unchanged. + +An interruption before the atomic `current` switch leaves the previous dispatcher and cleanup program selected; an interruption after it selects both from the requested release. +If activation, start, or loopback verification fails for a loaded service, rollback must first stop the service successfully, require `ActiveState=inactive`, and require `device.lock` to be absent. +Only then does it atomically restore the prior `current` selection, which restores both the dispatcher and cleanup program. +If rollback cannot prove those three conditions after the requested pair is selected, it leaves that new code and cleanup pair selected and reports rollback failure. +When an earlier selected release exists, it restarts that release and repeats the loopback verification. +That rollback restart passes the same tunnel checks before and after start and after loopback verification. +If public ingress activates after a dispatcher start, the command attempts to stop it and does not restart either release while ingress remains active. +If that stop, the `ActiveState` inspection, or the device-lock check fails, dispatcher state is inconclusive. +When no prior selection exists, rollback removes the new `current` and stable cleanup link. +If rollback cannot restore a verified service, the command exits with an explicit rollback failure. +When the trap confirms `ActiveState=inactive`, keep the tunnel stopped while you complete dispatcher recovery. +If it reports `PUBLIC INGRESS CONTAINMENT FAILED`, do not assume that the tunnel stopped. +Immediately disable or delete the public ingress through the approved Cloudflare administrator path and confirm that the public endpoint is unavailable. +Keep the dispatcher stopped until the tunnel is confirmed disabled and inactive and the public endpoint is unavailable. +Then rerun the complete later-deployment subshell instead of starting the dispatcher manually. +The deployer repeats the ingress gates and local bind and authentication verification before the subshell can publish the tunnel again. +Its exit trap keeps the tunnel stopped unless `ActiveState=active` and the config-free, direct public HTTP `401` proof both succeed. + +A later deployment does not modify the dispatcher environment, systemd unit, SSH credentials, or Cloudflare credentials. +It does not update itself. +Normal deployment requires the selected release to preserve the documented cleanup allowlist. +A cleanup-scope change requires separate review and a corresponding runbook update before deployment. +After every later deployment, repeat the authenticated successful, controlled-failure, and cancellation proof jobs under [Configure GitHub and Run a Proof Job](#configure-github-and-run-a-proof-job). + +## Publish the Dispatcher With Cloudflare Tunnel + +Install `cloudflared` from the approved package source for Colossus. +The next commands create a public DNS route to the authenticated dispatcher. +Do not create the route until the loopback authentication check returns HTTP `401` for an anonymous request. +Use an administrator account to create one named tunnel and route one DNS hostname: + +```bash +cloudflared tunnel login +cloudflared tunnel create nemoclaw-jetson-dispatch +cloudflared tunnel route dns nemoclaw-jetson-dispatch jetson-e2e.example.com +``` + +Record the tunnel UUID from the create command. +Substitute it for `TUNNEL_UUID` in each later command and file. +Create a dedicated `cloudflared` service account if the approved package did not create one: + +```bash +sudo useradd --system --create-home --home-dir /var/lib/cloudflared \ + --shell /usr/sbin/nologin cloudflared +``` + +Install only the tunnel credential for that account: + +```bash +sudo install -d -o cloudflared -g cloudflared -m 0700 /etc/cloudflared +sudo install -o cloudflared -g cloudflared -m 0600 \ + "$HOME/.cloudflared/TUNNEL_UUID.json" /etc/cloudflared/TUNNEL_UUID.json +sudo -u cloudflared test -r /etc/cloudflared/TUNNEL_UUID.json +rm "$HOME/.cloudflared/cert.pem" "$HOME/.cloudflared/TUNNEL_UUID.json" +``` + +The removed `cert.pem` authorizes Cloudflare account management. +Do not give it to the tunnel service account. + +Create `/etc/cloudflared/jetson-dispatch.yml` with mode `0600` and the tunnel service account as owner: + +```yaml +tunnel: TUNNEL_UUID +credentials-file: /etc/cloudflared/TUNNEL_UUID.json +ingress: + - hostname: jetson-e2e.example.com + service: http://127.0.0.1:8787 + - service: http_status:404 +``` + +Run `cloudflared` as its own restricted service account. +Install `/etc/systemd/system/nemoclaw-jetson-tunnel.service`: + +```ini +[Unit] +Description=NemoClaw Jetson Cloudflare Tunnel +After=network-online.target nemoclaw-jetson-dispatch.service +Wants=network-online.target + +[Service] +Type=simple +User=cloudflared +Group=cloudflared +ExecStart=/usr/bin/cloudflared --no-autoupdate --config /etc/cloudflared/jetson-dispatch.yml tunnel run +Restart=on-failure +RestartSec=5 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target +``` + +Start the tunnel service and inspect its status: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now nemoclaw-jetson-tunnel.service +sudo systemctl status nemoclaw-jetson-tunnel.service +``` + +The origin service validates the GitHub OIDC signature and claims on every job, status, cancel, and artifact request. +Cloudflare Tunnel does not replace origin authentication. +Apply a Cloudflare rate limit to `/v1/jobs*` if the managed zone supports one. +Without that rate limit, origin authentication still rejects unauthorized work, but public requests can consume dispatcher connections. + +Confirm the public endpoint also rejects an anonymous request with HTTP `401`. +Confirm Colossus has outbound HTTPS access to GitHub's OIDC key endpoint: + +```bash +PUBLIC_HTTP_CODE="$( + curl --disable --noproxy '*' --silent --show-error --output /dev/null \ + --write-out '%{http_code}' --max-time 10 --request POST \ + --header 'Content-Type: application/json' \ + --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ + https://jetson-e2e.example.com/v1/jobs +)" +test "$PUBLIC_HTTP_CODE" = 401 +curl --fail-with-body \ + https://token.actions.githubusercontent.com/.well-known/jwks +``` + +## Configure GitHub and Run a Proof Job + +The next command updates a repository Actions variable. +Set the public HTTPS origin as a repository Actions variable: + +```bash +gh variable set JETSON_DISPATCH_URL --repo NVIDIA/NemoClaw \ + --body 'https://jetson-e2e.example.com' +``` + +Resolve the non-fork pull request (PR) and trusted workflow identities. +Replace `1234` with the PR number: + +```bash +set -euo pipefail +PR_NUMBER=1234 +PR_JSON="$(gh api "repos/NVIDIA/NemoClaw/pulls/$PR_NUMBER")" +CANDIDATE_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" +CANDIDATE_SHA="$(jq -r .head.sha <<<"$PR_JSON")" +BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" +git fetch --prune origin main +WORKFLOW_SHA="$(git rev-parse origin/main)" +test "$CANDIDATE_REPOSITORY" = NVIDIA/NemoClaw +[[ "$CANDIDATE_SHA" =~ ^[a-f0-9]{40}$ ]] +[[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] +[[ "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] +``` + +Select only the Jetson target from the trusted `main` workflow: + +```bash +gh workflow run .github/workflows/e2e.yaml --repo NVIDIA/NemoClaw --ref main \ + -f targets=jetson-nvmap-gpu \ + -f jobs= \ + -f inference_mode=mock \ + -f include_staging_brev_launchable=false \ + -f allow_jetson_dispatch=true \ + -f allow_dgx_spark_runner_queue=false \ + -f "pr_number=$PR_NUMBER" \ + -f "checkout_sha=$CANDIDATE_SHA" \ + -f "checkout_repository=$CANDIDATE_REPOSITORY" \ + -f "base_sha=$BASE_SHA" \ + -f "workflow_sha=$WORKFLOW_SHA" \ + -f review_reason='Reviewed the commit under review for the isolated Jetson E2E.' +``` + +The `jetson-nvmap-gpu` job uses the fixed `jetson-nvmap-gpu-colossus` concurrency group. +`queue: max` queues every dispatch, and `cancel-in-progress: false` prevents automatic cancellation. + +The controller must run on `ubuntu-latest`. +The GitHub controller log must show one `Jetson dispatch accepted as ` line. +The Colossus journal must not contain a bearer token. +The uploaded `e2e-jetson-nvmap-gpu` artifact must contain `jetson-dispatch.json` with these results: + +- The requested candidate commit SHA and workflow run ID and attempt. +- The Jetson model, JetPack package version or `unavailable`, Jetson Linux release, and kernel. +- The test conclusion and bounded log. +- `status.cleanup: "succeeded"`. +- `status.conclusion: "success"` for the successful proof job. + +Download the trusted `e2e-dispatch--` receipt artifact and the `e2e-jetson-nvmap-gpu` artifact. +Set `RUN_ID` and `RUN_ATTEMPT` from the selected workflow run: + +```bash +EVIDENCE_DIR="$(mktemp -d)" +chmod 700 "$EVIDENCE_DIR" +RUN_ID=123456789 +RUN_ATTEMPT=1 +gh run download "$RUN_ID" --repo NVIDIA/NemoClaw \ + --name "e2e-dispatch-$RUN_ID-$RUN_ATTEMPT" \ + --dir "$EVIDENCE_DIR/trusted-dispatch" +gh run download "$RUN_ID" --repo NVIDIA/NemoClaw \ + --name e2e-jetson-nvmap-gpu \ + --dir "$EVIDENCE_DIR/jetson" +DISPATCH_JSON="$EVIDENCE_DIR/trusted-dispatch/dispatch.json" +JETSON_JSON="$EVIDENCE_DIR/jetson/jetson-dispatch.json" +``` + +Require `dispatch.json` to establish the repository, PR number, base SHA, candidate SHA, and trusted workflow SHA independently: + +```bash +jq -e \ + --arg repository NVIDIA/NemoClaw \ + --arg candidateRepository "$CANDIDATE_REPOSITORY" \ + --arg candidateSha "$CANDIDATE_SHA" \ + --arg baseSha "$BASE_SHA" \ + --arg workflowSha "$WORKFLOW_SHA" \ + --arg workflowRunId "$RUN_ID" \ + --argjson prNumber "$PR_NUMBER" \ + --argjson workflowRunAttempt "$RUN_ATTEMPT" ' + .kind == "nemoclaw-e2e-dispatch-v2" and + .repository == $repository and + .candidateRepository == $candidateRepository and + .candidateRepository == .repository and + .candidateSha == $candidateSha and + .prNumber == $prNumber and + .baseSha == $baseSha and + .workflowSha == $workflowSha and + .workflowRunId == $workflowRunId and + .workflowRunAttempt == $workflowRunAttempt and + .allowJetsonDispatch == true + ' "$DISPATCH_JSON" +``` + +Then bind `jetson-dispatch.json` to that trusted receipt by comparing its candidate SHA and workflow run ID and attempt: + +```bash +jq -e --slurpfile dispatch "$DISPATCH_JSON" ' + .status.request.candidateSha == $dispatch[0].candidateSha and + .status.request.workflowRunId == $dispatch[0].workflowRunId and + .status.request.workflowRunAttempt == $dispatch[0].workflowRunAttempt and + .status.cleanup == "succeeded" and + .status.conclusion == "success" +' "$JETSON_JSON" +``` + +It must also contain `jetson-e2e-artifacts.tar.gz`. +The dispatcher creates that archive from the remote E2E artifact directory before it removes the candidate workspace. +It rejects an artifact directory or compressed archive larger than 1 MiB. +During initialization, the dispatcher validates the schema, request, job ID, and state in every private file whose basename matches `^[a-f0-9]{64}\.json$`. +For a completed record, it also validates the terminal fields. +It ignores valid queued or running records and restores only the 128 newest completed statuses to memory. +An invalid matching file fails initialization before the dispatcher accepts work, and the error names its exact `.json` basename. +Status and artifact requests both restore an evicted completed status from its private file. +Only an artifact request returns the persisted log and archive. +A repeated deterministic dispatch returns the persisted completed status without rerunning candidate code or clearing its evidence. +Before worker execution, the dispatcher creates `device.lock`, persists the queued status, and synchronizes the Colossus state directory. +If queued-state persistence or that directory synchronization fails, candidate code does not run and the dispatcher does not unlink the exact live job lock. +Because a failed directory synchronization cannot prove reboot durability, inspect the live state and restart the dispatcher service to invoke startup cleanup without rebooting the Colossus host. +The dispatcher persists terminal status before it removes `device.lock`. +After removing the lock, it synchronizes the state directory. +If terminal-status persistence fails, it reports a completed in-memory failure and retains the lock. +If the lock-removal directory synchronization fails and restoration succeeds, it re-establishes the exact same job lock, synchronizes the directory again, and reports the lock-removal failure. +If restoration fails, lock state is inconclusive; the dispatcher sets an in-memory recovery barrier, blocks later dispatches, and reports that operator recovery is required during shutdown. + +After the workflow completes, independently verify every allowlisted resource is absent. +Require the private `.cleanup.json` file and verify every recorded volume and process ID is absent. +The private cleanup record is Colossus state and is not part of the uploaded artifact. +Then run one controlled failing candidate. +Confirm that its `jetson-dispatch.json` artifact shows `status.cleanup: "succeeded"` and `status.conclusion: "failure"`. +Require its private cleanup record and verify every recorded identity is absent. +For a cancellation proof, cancel a controller after it logs the job ID. +The controller can exit before it downloads an artifact. +Inspect the private Colossus `.json` status for `conclusion: "cancelled"` and `cleanup: "succeeded"`. +Independently verify the cleanup allowlist after that cancellation. + +These checks establish bounded cleanup of the allowlisted resources. +They do not attest that cleanup reversed every possible host change made by candidate code. + +## Recover or Disable the Deployment + +The dispatcher permits one active job. +A process interruption leaves `device.lock` in the state directory. +On startup, the dispatcher invokes the cleanup program before it removes that stale lock or accepts more work. +After successful cleanup, a valid queued or running record for the locked job becomes a durable completed failure with `cleanup: "succeeded"` and error `Jetson dispatcher restarted before terminal status was persisted`. +Only then does the dispatcher remove the lock; if this recovery persistence fails, the lock remains. +A repeated deterministic dispatch returns that recovered failure without rerunning candidate code. +If cleanup fails, startup fails or the completed job reports `conclusion: "cleanup-failed"` with `cleanup: "failed"`. +If cleanup succeeds but lock removal fails, the completed job reports `conclusion: "cleanup-failed"` with `cleanup: "succeeded"`. +The lock remains after a cleanup failure or successful lock restoration. +A lock-restoration failure does not prove whether the lock exists. +Bounded status error text retains the newest cleanup, persistence, or lock-removal error. + +If the service stops before durable persistence completes, destructive cleanup has not started. +If it stops after durable persistence, the cleanup record and device lock remain for startup recovery. +On startup, the helper discovers current identities and merges them with every retained identity before cleanup resumes. +For example, termination after container removal cannot erase the volume identities recorded before that removal. +The startup cleanup passes those retained volumes into deletion and independent verification. +After the helper succeeds, the worker revalidates its output and every retained identity before the dispatcher removes the lock. + +The unit uses `Restart=on-failure`, so a startup cleanup failure otherwise retries every five seconds. +Stop that retry loop before investigation or repair: + +```bash +sudo systemctl stop nemoclaw-jetson-dispatch.service +``` + +An invalid matching `.json` file also fails startup and enters the same retry loop. +Set `JOB_ID` from the exact basename in the startup error. +For this failure, preserve and inspect the named private file under the approved incident-retention policy; treat its JSON as private candidate output. +The following command validates the exact basename, stops the retry loop, requires `ActiveState=inactive`, and accepts either no `device.lock` or the service-owned, mode `0600`, single-link regular lock whose exact contents are `JOB_ID` plus its terminating newline. +A mismatched, nonregular, or symbolic-link lock blocks recovery. +It rejects a symbolic link or non-directory quarantine parent and verifies `root:root` mode `0700` without following links. +It then moves the exact object without overwriting retained evidence and restarts the service. +For a matching lock, startup reruns cleanup and removes the lock only after cleanup succeeds; never remove the lock manually. +After it returns, repeat the loopback-bind and anonymous HTTP `401` checks under [Configure the Dispatcher Service](#configure-the-dispatcher-service) before accepting work. + +```bash +set -euo pipefail +JOB_ID=0000000000000000000000000000000000000000000000000000000000000000 +[[ "$JOB_ID" =~ ^[a-f0-9]{64}$ ]] +state=/var/lib/nemoclaw-jetson-dispatch/state +status_file="$state/$JOB_ID.json" +quarantine=/var/lib/nemoclaw-jetson-status-quarantine +quarantine_file="$quarantine/$JOB_ID.json.invalid" +lock="$state/device.lock" +dispatcher_uid="$(id -u nemoclaw-jetson-dispatch)" +dispatcher_gid="$(id -g nemoclaw-jetson-dispatch)" +sudo systemctl stop nemoclaw-jetson-dispatch.service +test "$(sudo systemctl show --property=ActiveState --value \ + nemoclaw-jetson-dispatch.service)" = inactive +sudo stat -- "$status_file" +sudo test ! -L "$lock" +if sudo test -e "$lock"; then + test "$(sudo stat --format='%F:%u:%g:%a:%h' -- "$lock")" = \ + "regular file:$dispatcher_uid:$dispatcher_gid:600:1" + printf '%s\n' "$JOB_ID" | sudo cmp --silent -- "$lock" - +fi +sudo test ! -L "$quarantine" +if sudo test -e "$quarantine"; then + test "$(sudo stat --format='%F:%u:%g:%a' -- "$quarantine")" = \ + directory:0:0:700 +else + sudo install -d -o root -g root -m 0700 "$quarantine" +fi +test "$(sudo stat --format='%F:%u:%g:%a' -- "$quarantine")" = \ + directory:0:0:700 +sudo test ! -e "$quarantine_file" +sudo test ! -L "$quarantine_file" +sudo mv -- "$status_file" "$quarantine_file" +sudo test ! -e "$status_file" +sudo test ! -L "$status_file" +sudo systemctl start nemoclaw-jetson-dispatch.service +test "$(sudo systemctl show --property=ActiveState --value \ + nemoclaw-jetson-dispatch.service)" = active +``` + +Do not delete `device.lock` to bypass recovery. +For `cleanup: "failed"`, inspect the recorded error before choosing a recovery action. +Repair the cleanup program or allowlisted resource state only when that named operation failed. +If the protected tool or Ollama model baseline differs after cleanup, investigate candidate activity and external host drift without assigning the change to cleanup. +For `cleanup: "succeeded"` with a lock-removal error, repair the state-directory filesystem or permissions. +If the error also reports lock-restoration failure, stop the dispatcher and treat the exact `device.lock` state as inconclusive. +Use the ownership, file-type, mode, and exact job-ID checks above to inspect it without following links; do not accept more work until approved recovery and a service restart complete startup cleanup. +Start the dispatcher after the named condition is fixed: + +```bash +sudo systemctl start nemoclaw-jetson-dispatch.service +``` + +Startup runs cleanup and absence verification again before it removes the stale lock. + +Do not dispatch another job when cleanup verification is inconclusive. +Manual recovery must stay within the same cleanup allowlist. +Escalate any suspected protected baseline change for separate host investigation. +Cleanup evidence alone is not evidence that every candidate host change was reversed. + +Retain every `.cleanup.json` file until the temporary path teardown completes. +Preserve the other private job state required for diagnosis and stale-lock recovery. +These private state files can contain candidate output, and the dispatcher does not otherwise prune them. +Apply the host's approved retention policy to other private state after GitHub uploads the artifact and no device lock exists. + +To disable the temporary path, first prevent another controller from reaching the dispatcher. +Delete the repository variable and stop the public tunnel, but keep the dispatcher and its SSH credentials available while any accepted job finishes its bounded execution and cleanup: + +```bash +gh variable delete JETSON_DISPATCH_URL --repo NVIDIA/NemoClaw +sudo systemctl disable --now nemoclaw-jetson-tunnel.service +``` + +Use the last accepted job ID from the controller log. +Wait for the device lock to disappear and require that job's private status to report successful cleanup. +Require the last job's cleanup record. +Use Node.js `readdirSync` with the exact `^[a-f0-9]{64}\.cleanup\.json$` basename pattern. +Validate and aggregate every retained cleanup record that matches this pattern. +Independently verify every aggregated resource identity and the remaining fixed allowlist: + +```bash +set -euo pipefail +LAST_JOB_ID=0000000000000000000000000000000000000000000000000000000000000000 +[[ "$LAST_JOB_ID" =~ ^[a-f0-9]{64}$ ]] +state=/var/lib/nemoclaw-jetson-dispatch/state +sudo -u nemoclaw-jetson-dispatch \ + timeout 3600 bash -c 'while [ -e "$1" ]; do sleep 5; done' wait-lock \ + "$state/device.lock" +sudo -u nemoclaw-jetson-dispatch test ! -e "$state/device.lock" +sudo -u nemoclaw-jetson-dispatch /usr/bin/node -e ' + const fs = require("node:fs"); + const status = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (status.state !== "completed" || status.cleanup !== "succeeded") process.exit(1); +' "$state/$LAST_JOB_ID.json" +cleanup_identities="$(sudo -u nemoclaw-jetson-dispatch /usr/bin/node -e ' + const fs = require("node:fs"); + const path = require("node:path"); + const stateDirectory = process.argv[1]; + const lastJobId = process.argv[2]; + const cleanupName = /^[a-f0-9]{64}\.cleanup\.json$/; + const names = fs.readdirSync(stateDirectory).filter((name) => cleanupName.test(name)).sort(); + if (!names.includes(`${lastJobId}.cleanup.json`)) process.exit(1); + const expectedKeys = ["processIds", "schemaVersion", "volumes"]; + const validVolume = (value) => + typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(value); + const validProcessId = (value) => Number.isSafeInteger(value) && value > 0; + const volumes = new Set(); + const processIds = new Set(); + for (const name of names) { + const file = path.join(stateDirectory, name); + const metadata = fs.lstatSync(file); + if (!metadata.isFile() || (metadata.mode & 0o777) !== 0o600) process.exit(1); + const raw = fs.readFileSync(file, "utf8"); + if (Buffer.byteLength(raw) > 64 * 1024) process.exit(1); + const record = JSON.parse(raw); + const keys = Object.keys(record).sort(); + if ( + JSON.stringify(keys) !== JSON.stringify(expectedKeys) || + record.schemaVersion !== 1 || + !Array.isArray(record.volumes) || + !record.volumes.every(validVolume) || + !Array.isArray(record.processIds) || + !record.processIds.every(validProcessId) + ) process.exit(1); + for (const volume of record.volumes) volumes.add(volume); + for (const processId of record.processIds) processIds.add(processId); + } + for (const volume of [...volumes].sort()) console.log(`volume\t${volume}`); + for (const processId of [...processIds].sort((a, b) => a - b)) { + console.log(`processId\t${processId}`); + } +' "$state" "$LAST_JOB_ID")" +cleanup_volumes=() +cleanup_process_ids=() +while IFS=$'\t' read -r identity_kind identity; do + [ -n "$identity_kind" ] || continue + case "$identity_kind" in + volume) cleanup_volumes+=("$identity") ;; + processId) cleanup_process_ids+=("$identity") ;; + *) exit 1 ;; + esac +done <<<"$cleanup_identities" +for volume in "${cleanup_volumes[@]}"; do + timeout --kill-after=5 120 \ + sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ + -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ + -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ + -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ + nvidia@192.168.55.1 bash -s -- "$volume" <<'VERIFY_RECORDED_VOLUME' +set -euo pipefail +volume_names="$(docker volume ls --format '{{.Name}}')" || { + echo 'Unable to list Docker volumes' >&2 + exit 1 +} +if printf '%s\n' "$volume_names" | grep -Fqx -- "$1"; then + echo "A recorded job-owned Docker volume remains: $1" >&2 + exit 1 +fi +VERIFY_RECORDED_VOLUME +done +for process_id in "${cleanup_process_ids[@]}"; do + timeout --kill-after=5 120 \ + sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ + -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ + -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ + -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ + nvidia@192.168.55.1 bash -s -- "$process_id" <<'VERIFY_RECORDED_PROCESS' +set -euo pipefail +if [ -e "/proc/$1" ]; then + echo "A recorded job-owned process ID remains: $1" >&2 + exit 1 +fi +VERIFY_RECORDED_PROCESS +done +timeout --kill-after=5 120 \ + sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ + -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ + -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ + -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ + nvidia@192.168.55.1 bash -s -- "$LAST_JOB_ID" <<'VERIFY_JETSON_IDLE' +set -euo pipefail +job_id="$1" +[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] +job_home="/var/tmp/nemoclaw-jetson-e2e/$job_id/home" +test ! -e "/var/tmp/nemoclaw-jetson-e2e/$job_id" +test ! -e /tmp/nemoclaw-services-e2e-jetson-nvmap +if [ -e /var/tmp/nemoclaw-jetson-e2e ]; then + test -d /var/tmp/nemoclaw-jetson-e2e + test ! -L /var/tmp/nemoclaw-jetson-e2e + test -z "$(find /var/tmp/nemoclaw-jetson-e2e -mindepth 1 -maxdepth 1 -print -quit)" +fi +sandbox_container_output="$(docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter label=openshell.ai/sandbox-name=e2e-jetson-nvmap)" || { + echo 'Unable to list labeled sandbox containers' >&2 + exit 1 +} +test -z "$sandbox_container_output" +container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { + echo 'Unable to list Docker containers' >&2 + exit 1 +} +if printf '%s\n' "$container_rows" | + awk -F '\t' '$2 == "openshell-cluster-nemoclaw" { found = 1 } END { exit found ? 0 : 1 }'; then + echo 'The job-owned gateway container remains' >&2 + exit 1 +fi +volume_names="$(docker volume ls --format '{{.Name}}')" || { + echo 'Unable to list Docker volumes' >&2 + exit 1 +} +if printf '%s\n' "$volume_names" | grep -Fqx openshell-cluster-nemoclaw; then + echo 'The job-owned gateway volume remains' >&2 + exit 1 +fi +read_proc_uid() { + awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' \ + "$1/status" 2>/dev/null +} +read_proc_environment() { + dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' +} +read_proc_command() { + dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' +} +handle_proc_read_failure() { + local proc_dir="$1" field="$2" process_uid directory_uid + [ -d "$proc_dir" ] || return 0 + if process_uid="$(read_proc_uid "$proc_dir")"; then + [ "$process_uid" = "$(id -u)" ] || return 0 + else + [ -d "$proc_dir" ] || return 0 + if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then + [ -d "$proc_dir" ] || return 0 + echo "Unable to verify the owner of a live process after a failed $field read" >&2 + exit 1 + fi + [ "$directory_uid" = "$(id -u)" ] || return 0 + fi + echo "Unable to inspect $field for a live same-user process" >&2 + exit 1 +} +for proc_dir in /proc/[0-9]*; do + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" owner + continue + fi + [ "$process_uid" = "$(id -u)" ] || continue + if ! environment="$(read_proc_environment "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" environment + continue + fi + printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue + if ! cmdline="$(read_proc_command "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" command + continue + fi + case "$cmdline" in + *ollama-auth-proxy.*|*openshell-gateway*|*openshell-forward*|*openshell\ forward*|*cloudflared*) + echo "A job-owned helper process remains: ${proc_dir##*/}" >&2 + exit 1 + ;; + esac +done +image_rows="$(docker image ls nemoclaw-sandbox-local --format '{{.Repository}}\t{{.Tag}}')" || { + echo 'Unable to list job-owned Docker images' >&2 + exit 1 +} +test -z "$(printf '%s\n' "$image_rows" | + awk '$1 == "nemoclaw-sandbox-local" && index($2, "e2e-jetson-nvmap-") == 1 { print $1 ":" $2 }')" +command -v node npm ollama +timeout --kill-after=5 30 ollama list >/dev/null +for openshell_component in openshell openshell-gateway openshell-sandbox; do + if command -v "$openshell_component" >/dev/null 2>&1; then + echo "A host-level OpenShell binary remains after cleanup: $openshell_component" >&2 + exit 1 + fi + for host_bin in \ + "/usr/local/bin/$openshell_component" \ + "/usr/bin/$openshell_component" \ + "$HOME/.local/bin/$openshell_component"; do + test ! -e "$host_bin" + test ! -L "$host_bin" + done +done +test -c /dev/nvmap +case "$(docker info --format '{{json .Runtimes}}')" in + *nvidia*) ;; + *) exit 1 ;; +esac +VERIFY_JETSON_IDLE +``` + +If the last cleanup record is missing, keep the dispatcher and SSH credentials. +Keep them when any retained cleanup record is malformed or an absence check fails. +Use the recovery procedure above before teardown. +Only after every check passes should you stop the dispatcher: + +```bash +sudo systemctl disable --now nemoclaw-jetson-dispatch.service +``` + +Reauthenticate with `cloudflared tunnel login` as a Cloudflare administrator. +Delete the `jetson-e2e.example.com` DNS record from the Cloudflare zone. +Delete the named tunnel with `cloudflared tunnel delete TUNNEL_UUID`. +Confirm that `cloudflared tunnel list` no longer returns `TUNNEL_UUID`. +Remove the new local account certificate after the administrative deletion. +Remove the local tunnel credential, configuration, and service unit: + +```bash +rm "$HOME/.cloudflared/cert.pem" +sudo rm /etc/cloudflared/TUNNEL_UUID.json \ + /etc/cloudflared/jetson-dispatch.yml \ + /etc/systemd/system/nemoclaw-jetson-tunnel.service +sudo systemctl daemon-reload +test ! -e /etc/cloudflared/TUNNEL_UUID.json +test ! -e /etc/cloudflared/jetson-dispatch.yml +test ! -e /etc/systemd/system/nemoclaw-jetson-tunnel.service +``` + +Confirm that the public endpoint is unreachable and the repository variable is absent: + +```bash +if curl --silent --show-error --output /dev/null --connect-timeout 10 \ + https://jetson-e2e.example.com/v1/jobs; then + echo 'Jetson dispatch endpoint is still reachable' >&2 + exit 1 +fi +test -z "$(gh variable list --repo NVIDIA/NemoClaw --json name \ + --jq '.[] | select(.name == "JETSON_DISPATCH_URL") | .name')" +``` + +Remove the dedicated Jetson public key and the Colossus SSH private key. +Encode the exact authorized-key line so the space-containing value remains one remote-command argument, then verify that exact line is absent before deleting the Colossus key: + +```bash +set -euo pipefail +JETSON_PUBLIC_KEY="$(sudo cat /var/lib/nemoclaw-jetson-dispatch/id_ed25519.pub)" +case "$JETSON_PUBLIC_KEY" in + ssh-ed25519\ *) ;; + *) echo 'Unexpected Jetson public key format' >&2; exit 1 ;; +esac +JETSON_AUTHORIZED_KEY_LINE="from=\"192.168.55.100\",restrict $JETSON_PUBLIC_KEY" +JETSON_AUTHORIZED_KEY_B64="$(printf '%s' "$JETSON_AUTHORIZED_KEY_LINE" | base64 --wrap=0)" +timeout --kill-after=5 120 \ + sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ + -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ + -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ + -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ + nvidia@192.168.55.1 bash -s -- "$JETSON_AUTHORIZED_KEY_B64" <<'REMOVE_DISPATCH_KEY' +set -euo pipefail +authorized_key_line="$(printf '%s' "$1" | base64 --decode)" +authorized_keys="$HOME/.ssh/authorized_keys" +test -f "$authorized_keys" +temporary="$(mktemp "$HOME/.ssh/.authorized_keys.XXXXXX")" +trap 'rm -f "$temporary"' EXIT +chmod 600 "$temporary" +while IFS= read -r line || [ -n "$line" ]; do + if [ "$line" != "$authorized_key_line" ]; then + printf '%s\n' "$line" + fi +done <"$authorized_keys" >"$temporary" +mv "$temporary" "$authorized_keys" +trap - EXIT +if grep -Fqx -- "$authorized_key_line" "$authorized_keys"; then + echo 'Dedicated Jetson public key remains authorized' >&2 + exit 1 +fi +REMOVE_DISPATCH_KEY +sudo rm -- \ + /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ + /var/lib/nemoclaw-jetson-dispatch/id_ed25519.pub +sudo test ! -e /var/lib/nemoclaw-jetson-dispatch/id_ed25519 +sudo test ! -e /var/lib/nemoclaw-jetson-dispatch/id_ed25519.pub +``` + +An SSH transport failure keeps the Colossus SSH key and pinned host-key files. +The Jetson `authorized_keys` result is inconclusive in this case. +Use the serial console to confirm that the exact key line is absent before you continue credential removal. + +Remove the cleanup executable and pinned SSH host-key file, then verify their absence: + +```bash +sudo rm -- \ + /usr/local/libexec/nemoclaw-jetson-cleanup \ + /var/lib/nemoclaw-jetson-dispatch/known_hosts +sudo test ! -e /usr/local/libexec/nemoclaw-jetson-cleanup +sudo test ! -L /usr/local/libexec/nemoclaw-jetson-cleanup +sudo test ! -e /var/lib/nemoclaw-jetson-dispatch/known_hosts +``` + +After the required retention period, remove the private job state and logs. +When no investigation, recovery, or retention requirement remains, remove these remaining deployment files: + +```bash +sudo rm -- \ + /etc/nemoclaw-jetson-dispatch/environment \ + /etc/systemd/system/nemoclaw-jetson-dispatch.service \ + /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy +sudo rm -rf -- \ + /opt/nemoclaw-jetson-dispatch \ + /var/lib/nemoclaw-jetson-status-quarantine +sudo systemctl daemon-reload +sudo test ! -e /etc/nemoclaw-jetson-dispatch/environment +sudo test ! -e /etc/systemd/system/nemoclaw-jetson-dispatch.service +sudo test ! -e /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy +sudo test ! -e /opt/nemoclaw-jetson-dispatch +sudo test ! -L /opt/nemoclaw-jetson-dispatch +sudo test ! -e /var/lib/nemoclaw-jetson-status-quarantine +sudo test ! -L /var/lib/nemoclaw-jetson-status-quarantine +test "$(systemctl show --property=LoadState --value \ + nemoclaw-jetson-dispatch.service)" = not-found +``` diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index b69de1713a9..b632b0818c2 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -21,12 +21,22 @@ const TIMEOUT_MS = 50 * 60_000; function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { ...buildAvailabilityProbeEnv(), + HOME: process.env.HOME, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_JETSON_WORKSPACE: process.env.NEMOCLAW_JETSON_WORKSPACE, NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_PROVIDER: process.env.NEMOCLAW_PROVIDER ?? "ollama", NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + PATH: process.env.PATH, + TMPDIR: process.env.TMPDIR, + XDG_BIN_HOME: process.env.XDG_BIN_HOME, + XDG_CACHE_HOME: process.env.XDG_CACHE_HOME, + XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME, + XDG_DATA_HOME: process.env.XDG_DATA_HOME, + XDG_STATE_HOME: process.env.XDG_STATE_HOME, + npm_config_prefix: process.env.npm_config_prefix, ...extra, }; } @@ -55,9 +65,7 @@ fi if command -v openshell >/dev/null 2>&1; then openshell sandbox delete "$NEMOCLAW_SANDBOX_NAME" 2>/dev/null || true openshell gateway destroy -g nemoclaw 2>/dev/null || true -fi -pkill -f "ollama serve" 2>/dev/null || true -pkill -f "ollama-auth-proxy" 2>/dev/null || true`, +fi`, "cleanup-jetson-nvmap", 120_000, ).catch(() => undefined); @@ -112,25 +120,6 @@ fi`, "Not a Jetson/Tegra host (/dev/nvmap absent) — reporter workflow requires Jetson hardware; hermetic #4231 coverage remains in src/lib/onboard/docker-gpu-patch.test.ts.", ); - cleanup.trackDisposable("stop Jetson Ollama processes", async () => { - const stop = await hostShell( - host, - String.raw`set +e -status=0 -for pattern in '[o]llama serve' '[o]llama-auth-proxy'; do - pkill -f "$pattern" - rc=$? - case "$rc" in - 0|1) ;; - *) status="$rc" ;; - esac -done -exit "$status"`, - "cleanup-jetson-ollama-processes", - 120_000, - ); - expect(stop.exitCode, resultText(stop)).toBe(0); - }); const gatewayCleanupOptions = { artifactName: "cleanup-jetson-openshell-gateway", env: env(), @@ -224,17 +213,13 @@ exit "$status"`, // A3: preserve the reporter workflow by installing/running the real onboarding shell path. progress.phase("install NemoClaw on the Jetson host"); - const installOllama = await hostShell( + const ollamaBaseline = await hostShell( host, - 'if [ "${NEMOCLAW_PROVIDER:-ollama}" = "ollama" ] && ! command -v ollama >/dev/null 2>&1; then\n' + - " curl -fsSL https://ollama.com/install.sh | sh 2>&1 || true\n" + - " systemctl stop ollama 2>/dev/null || true\n" + - ' pkill -f "ollama serve" 2>/dev/null || true\n' + - "fi", - "phase-1-install-ollama-if-needed", - 10 * 60_000, + "command -v ollama && ollama list", + "phase-1-ollama-baseline", + 120_000, ); - expect(installOllama.exitCode, resultText(installOllama)).toBe(0); + expect(ollamaBaseline.exitCode, resultText(ollamaBaseline)).toBe(0); const install = await host.command("bash", ["install.sh", "--non-interactive"], { artifactName: "phase-2-install-jetson-nvmap", @@ -249,6 +234,23 @@ exit "$status"`, expect(installedCli.exitCode, resultText(installedCli)).toBe(0); expect(installedCli.stdout.trim()).not.toBe(""); + const installedBinaries = await hostShell( + host, + String.raw`set -euo pipefail +[ -n "$NEMOCLAW_JETSON_WORKSPACE" ] +for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox; do + installed_path="$(command -v "$installed_command")" + canonical_path="$(realpath -e "$installed_path")" + case "$canonical_path" in + "$NEMOCLAW_JETSON_WORKSPACE"/*) printf '%s\t%s\n' "$installed_command" "$canonical_path" ;; + *) echo "$installed_command resolved outside the Jetson job workspace" >&2; exit 1 ;; + esac +done`, + "phase-2-job-local-installation", + ); + expect(installedBinaries.exitCode, resultText(installedBinaries)).toBe(0); + expect(installedBinaries.stdout.trim().split("\n")).toHaveLength(4); + // A4: the Jetson recreate must grant Tegra device-node groups via --group-add. expect(resultText(install)).toContain( "Granting sandbox user the detected Jetson GPU device groups via --group-add", diff --git a/test/e2e/support/colossus-jetson-dispatch-deploy.test.ts b/test/e2e/support/colossus-jetson-dispatch-deploy.test.ts new file mode 100644 index 00000000000..365642f4fbd --- /dev/null +++ b/test/e2e/support/colossus-jetson-dispatch-deploy.test.ts @@ -0,0 +1,833 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +const DEPLOY_SCRIPT = path.join(process.cwd(), "tools/e2e/colossus-jetson-dispatch-deploy.sh"); +const ENVIRONMENT_SOURCE = path.join( + process.cwd(), + "tools/e2e/colossus-jetson-dispatch.environment", +); +const FIXED_ORIGIN = "https://github.com/NVIDIA/NemoClaw.git"; +const UNIT_SOURCE = path.join(process.cwd(), "tools/e2e/nemoclaw-jetson-dispatch.service"); +const temporaryDirectories: string[] = []; + +const HARNESS = String.raw` +set -euo pipefail +source "$DEPLOY_SCRIPT" + +deploy_root="$TEST_ROOT/opt/nemoclaw-jetson-dispatch" +releases_directory="$deploy_root/releases" +current_link="$deploy_root/current" +deploy_lock="$deploy_root/deploy.lock" +cleanup_executable="$TEST_ROOT/usr/local/libexec/nemoclaw-jetson-cleanup" +cleanup_link_target="$current_link/$cleanup_relative" +service_home="$TEST_ROOT/var/lib/nemoclaw-jetson-dispatch" +state_directory="$service_home/state" +device_lock="$state_directory/device.lock" +ssh_identity_file="$service_home/id_ed25519" +ssh_known_hosts_file="$service_home/known_hosts" +node_executable="$NODE_EXECUTABLE" +environment_file="$TEST_ROOT/etc/nemoclaw-jetson-dispatch/environment" +unit_file="$TEST_ROOT/etc/systemd/system/$service_name" + +mkdir -p "$state_directory" "${"$"}{environment_file%/*}" "${"$"}{unit_file%/*}" +printf 'test-private-key\n' >"$ssh_identity_file" +printf 'test-known-host\n' >"$ssh_known_hosts_file" +[ "$SERVICE_LOAD_STATE" != loaded ] || : >"$TEST_ROOT/service-active" +[ "$INITIAL_DEVICE_LOCK" != 1 ] || printf '%064d\n' 0 >"$device_lock" + +effective_uid() { + printf '%s\n' "$FAKE_UID" +} + +id_exec() { + [ "$MISSING_ACCOUNT" != 1 ] || return 1 + printf '1001\n' +} + +require_service_owned_file() { + [ -f "$1" ] && [ ! -L "$1" ] +} + +require_service_owned_directory() { + [ "$INVALID_SERVICE_HOME" != 1 ] || fail "$1 must be a directory, not a symbolic link" + [ -d "$1" ] && [ ! -L "$1" ] +} + +ssh_identity_is_valid() { + [ "$INVALID_SSH_KEY" != 1 ] +} + +known_hosts_has_jetson() { + [ "$INVALID_KNOWN_HOSTS" != 1 ] +} + +node_version_is_supported() { + [ "$UNSUPPORTED_NODE" != 1 ] +} + +install_directory() { + /usr/bin/install -d -m "$2" "$1" +} + +install_file() { + /usr/bin/install -m "$3" "$1" "$2" +} + +require_root_owned_directory() { + [ -d "$1" ] && [ ! -L "$1" ] +} + +require_root_owned_file() { + [ -f "$1" ] && [ ! -L "$1" ] +} + +root_owned_file_matches() { + [ -f "$1" ] && [ ! -L "$1" ] +} + +symlink_owner() { + printf '0:0\n' +} + +acquire_deploy_lock() { + : +} + +move_replace() { + if [ "$2" = "$current_link" ] && [ -n "$FAIL_SELECT_SHA" ]; then + candidate="$(/usr/bin/readlink "$1")" + [ "${"$"}{candidate##*/}" != "$FAIL_SELECT_SHA" ] || return 1 + fi + "$NODE_EXECUTABLE" -e 'require("node:fs").renameSync(process.argv[1], process.argv[2])' "$1" "$2" +} + +move_directory() { + /bin/mv "$1" "$2" +} + +copy_file_preserving_metadata() { + /bin/cp -p "$1" "$2" +} + +files_match() { + /usr/bin/cmp -s "$1" "$2" +} + +git_exec() { + { + printf 'git' + printf '\t%s' "$@" + printf '\n' + } >>"$TEST_ROOT/git.log" + case "${"$"}{3:-}" in + fetch) + /usr/bin/git -c protocol.file.allow=always -C "$2" fetch \ + --depth=1 --no-tags "$FIXTURE_REPOSITORY" "${"$"}{!#}" + ;; + *) /usr/bin/git "$@" ;; + esac +} + +service_load_state() { + if [ -f "$unit_file" ]; then + printf 'loaded\n' + else + printf '%s\n' "$SERVICE_LOAD_STATE" + fi +} + +service_active_state() { + [ "$ACTIVE_STATE_ERROR" != 1 ] || return 1 + if [ -f "$TEST_ROOT/service-active" ]; then + printf 'active\n' + else + printf 'inactive\n' + fi +} + +current_sha() { + /usr/bin/readlink "$current_link" 2>/dev/null | /usr/bin/awk -F/ '{print $NF}' +} + +tunnel_is_active() { + tunnel_check_count="$(cat "$TEST_ROOT/tunnel-check-count" 2>/dev/null || printf '0')" + [ "$TUNNEL_ACTIVE" = 1 ] || + { [ "$TUNNEL_ACTIVATE_AFTER_CHECK" -gt 0 ] && + [ "$tunnel_check_count" -gt "$TUNNEL_ACTIVATE_AFTER_CHECK" ]; } +} + +systemctl_exec() { + printf 'systemctl\t%s\n' "$*" >>"$TEST_ROOT/service.log" + case "$1" in + show) + if [ "${"$"}{!#}" = "$tunnel_service_name" ]; then + case "$2" in + --property=LoadState) + tunnel_check_count="$(cat "$TEST_ROOT/tunnel-check-count" 2>/dev/null || printf '0')" + tunnel_check_count="$((tunnel_check_count + 1))" + printf '%s\n' "$tunnel_check_count" >"$TEST_ROOT/tunnel-check-count" + if tunnel_is_active; then printf 'loaded\n'; else printf 'not-found\n'; fi + ;; + --property=ActiveState) + if tunnel_is_active; then printf 'active\n'; else printf 'inactive\n'; fi + ;; + --property=UnitFileState) + if tunnel_is_active; then printf 'enabled\n'; else printf 'disabled\n'; fi + ;; + *) return 1 ;; + esac + else + return 1 + fi + ;; + daemon-reload) : ;; + stop) + stop_count="$(cat "$TEST_ROOT/stop-count" 2>/dev/null || printf '0')" + stop_count="$((stop_count + 1))" + printf '%s\n' "$stop_count" >"$TEST_ROOT/stop-count" + if [ "$FAIL_ROLLBACK_STOP" = 1 ] && [ "$stop_count" -gt 1 ]; then + return 1 + fi + /bin/rm -f "$TEST_ROOT/service-active" + if [ "$LEAVE_DEVICE_LOCK" = 1 ]; then + printf '%064d\n' 0 >"$device_lock" + else + /bin/rm -f "$device_lock" + fi + ;; + enable) + selected="$(current_sha)" + printf 'enable-sha\t%s\n' "$selected" >>"$TEST_ROOT/service.log" + [ "$selected" != "$FAIL_START_SHA" ] || return 1 + : >"$TEST_ROOT/service-active" + ;; + disable) + [ "$FAIL_INITIAL_ROLLBACK_DISABLE" != 1 ] || return 1 + /bin/rm -f "$TEST_ROOT/service-active" + ;; + start) + selected="$(current_sha)" + printf 'start-sha\t%s\n' "$selected" >>"$TEST_ROOT/service.log" + [ "$selected" != "$FAIL_START_SHA" ] || return 1 + : >"$TEST_ROOT/service-active" + ;; + is-active) [ -f "$TEST_ROOT/service-active" ] ;; + *) return 1 ;; + esac +} + +ss_exec() { + if [ "$(current_sha)" = "$FAIL_VERIFY_SHA" ]; then + printf 'LISTEN 0 511 0.0.0.0:8787 0.0.0.0:*\n' + else + printf 'LISTEN 0 511 127.0.0.1:8787 0.0.0.0:*\n' + fi +} + +curl_exec() { + { + printf 'curl' + printf '\t%s' "$@" + printf '\n' + } >>"$TEST_ROOT/curl.log" + if [ "$(current_sha)" = "$FAIL_VERIFY_SHA" ]; then + printf '503' + else + printf '401' + fi +} + +sleep_exec() { + : +} + +main "$@" +`; + +interface Fixture { + directory: string; + repository: string; + firstSha: string; + secondSha: string; +} + +interface DeployOptions { + activeStateError?: boolean; + fakeUid?: string; + failInitialRollbackDisable?: boolean; + failRollbackStop?: boolean; + failSelectSha?: string; + serviceLoadState?: "loaded" | "not-found"; + failStartSha?: string; + failVerifySha?: string; + initialDeviceLock?: boolean; + invalidKnownHosts?: boolean; + invalidServiceHome?: boolean; + invalidSshKey?: boolean; + leaveDeviceLock?: boolean; + missingAccount?: boolean; + tunnelActive?: boolean; + tunnelActivatesAfterCheck?: number; + unsupportedNode?: boolean; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +function git(repository: string, ...args: string[]): string { + return execFileSync("/usr/bin/git", ["-C", repository, ...args], { + encoding: "utf8", + }).trim(); +} + +function createFixture(): Fixture { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-colossus-deploy-")); + temporaryDirectories.push(directory); + const repository = path.join(directory, "origin"); + fs.mkdirSync(path.join(repository, "tools/e2e"), { recursive: true }); + execFileSync("/usr/bin/git", ["init", "--quiet", repository]); + git(repository, "config", "user.name", "NemoClaw Test"); + git(repository, "config", "user.email", "test@example.com"); + const cleanup = path.join(repository, "tools/e2e/jetson-dispatch-cleanup.sh"); + const environment = path.join(repository, "tools/e2e/colossus-jetson-dispatch.environment"); + const unit = path.join(repository, "tools/e2e/nemoclaw-jetson-dispatch.service"); + fs.writeFileSync(cleanup, "#!/usr/bin/env bash\necho first-release\n", { mode: 0o755 }); + fs.writeFileSync(environment, "JETSON_DISPATCH_PORT=8787\n"); + fs.writeFileSync(unit, "[Service]\nEnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment\n"); + git(repository, "add", "tools/e2e"); + git(repository, "commit", "--quiet", "-m", "first release"); + const firstSha = git(repository, "rev-parse", "HEAD"); + fs.writeFileSync(cleanup, "#!/usr/bin/env bash\necho second-release\n", { mode: 0o755 }); + git(repository, "add", cleanup); + git(repository, "commit", "--quiet", "-m", "second release"); + return { directory, repository, firstSha, secondSha: git(repository, "rev-parse", "HEAD") }; +} + +function runDeploy( + fixture: Fixture, + args: string[], + options: DeployOptions = {}, +): ReturnType { + return spawnSync("/bin/bash", ["-c", HARNESS, "deploy-harness", ...args], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + ACTIVE_STATE_ERROR: options.activeStateError ? "1" : "0", + DEPLOY_SCRIPT, + FAIL_INITIAL_ROLLBACK_DISABLE: options.failInitialRollbackDisable ? "1" : "0", + FAIL_ROLLBACK_STOP: options.failRollbackStop ? "1" : "0", + FAIL_SELECT_SHA: options.failSelectSha ?? "", + FAIL_START_SHA: options.failStartSha ?? "", + FAIL_VERIFY_SHA: options.failVerifySha ?? "", + FAKE_UID: options.fakeUid ?? "0", + FIXTURE_REPOSITORY: fixture.repository, + INITIAL_DEVICE_LOCK: options.initialDeviceLock ? "1" : "0", + INVALID_KNOWN_HOSTS: options.invalidKnownHosts ? "1" : "0", + INVALID_SERVICE_HOME: options.invalidServiceHome ? "1" : "0", + INVALID_SSH_KEY: options.invalidSshKey ? "1" : "0", + LEAVE_DEVICE_LOCK: options.leaveDeviceLock ? "1" : "0", + MISSING_ACCOUNT: options.missingAccount ? "1" : "0", + NODE_EXECUTABLE: process.execPath, + SERVICE_LOAD_STATE: options.serviceLoadState ?? "not-found", + TEST_ROOT: fixture.directory, + TUNNEL_ACTIVE: options.tunnelActive ? "1" : "0", + TUNNEL_ACTIVATE_AFTER_CHECK: String(options.tunnelActivatesAfterCheck ?? 0), + UNSUPPORTED_NODE: options.unsupportedNode ? "1" : "0", + }, + }); +} + +function deploymentPaths(fixture: Fixture) { + const root = path.join(fixture.directory, "opt/nemoclaw-jetson-dispatch"); + return { + cleanup: path.join(fixture.directory, "usr/local/libexec/nemoclaw-jetson-cleanup"), + current: path.join(root, "current"), + environment: path.join(fixture.directory, "etc/nemoclaw-jetson-dispatch/environment"), + releases: path.join(root, "releases"), + serviceLog: path.join(fixture.directory, "service.log"), + unit: path.join(fixture.directory, "etc/systemd/system/nemoclaw-jetson-dispatch.service"), + }; +} + +function readIfPresent(file: string): string { + return fs.existsSync(file) ? fs.readFileSync(file, "utf8") : ""; +} + +describe("Colossus Jetson dispatcher deployment", () => { + it("ships the fixed dispatcher environment and restricted systemd unit (#8142)", () => { + const environment = fs.readFileSync(ENVIRONMENT_SOURCE, "utf8"); + const unit = fs.readFileSync(UNIT_SOURCE, "utf8"); + + expect(environment).toContain("JETSON_DISPATCH_GITHUB_REPOSITORY_ID=1182547092"); + expect(environment).toContain("JETSON_DISPATCH_SSH_DESTINATION=nvidia@192.168.55.1"); + expect(environment).toContain( + "JETSON_DISPATCH_CLEANUP_EXECUTABLE=/usr/local/libexec/nemoclaw-jetson-cleanup", + ); + expect(environment).not.toContain("=REPOSITORY_ID"); + expect(unit).toContain("EnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment"); + expect(unit).toContain("WorkingDirectory=/opt/nemoclaw-jetson-dispatch/current"); + expect(unit).toContain("NoNewPrivileges=true"); + expect(unit).not.toContain("nemoclaw-jetson-tunnel.service"); + }); + + it("installs and verifies an exact release when the service is absent (#8142)", () => { + const fixture = createFixture(); + const result = runDeploy(fixture, ["--commit", fixture.firstSha]); + const paths = deploymentPaths(fixture); + + expect(result.status, String(result.stderr)).toBe(0); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readlinkSync(paths.cleanup)).toBe( + path.join(paths.current, "tools/e2e/jetson-dispatch-cleanup.sh"), + ); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + expect(fs.readdirSync(paths.releases)).toEqual([fixture.firstSha]); + expect(fs.readFileSync(paths.environment, "utf8")).toBe("JETSON_DISPATCH_PORT=8787\n"); + expect(fs.readFileSync(paths.unit, "utf8")).toContain( + "EnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment", + ); + expect(readIfPresent(paths.serviceLog)).toMatch( + /systemctl\tdaemon-reload[\s\S]*systemctl\tenable --now nemoclaw-jetson-dispatch\.service/u, + ); + expect(result.stdout).toMatch( + /\[1\/5\].*[\s\S]*\[2\/5\].*[\s\S]*\[3\/5\].*[\s\S]*\[4\/5\].*[\s\S]*\[5\/5\]/u, + ); + expect(result.stdout).toContain("public ingress remains disabled"); + expect(readIfPresent(paths.serviceLog)).not.toContain( + "enable --now nemoclaw-jetson-tunnel.service", + ); + expect(fs.readFileSync(path.join(fixture.directory, "curl.log"), "utf8")).toContain( + "curl\t--disable\t--noproxy\t*\t--silent", + ); + expect(fs.readFileSync(path.join(fixture.directory, "git.log"), "utf8")).toContain( + `remote\tadd\torigin\t${FIXED_ORIGIN}`, + ); + }); + + it.each([ + ["dispatcher account", { missingAccount: true }, "account is required"], + ["service home", { invalidServiceHome: true }, "must be a directory"], + ["SSH identity", { invalidSshKey: true }, "is not a readable OpenSSH private key"], + ["pinned host key", { invalidKnownHosts: true }, "does not contain the pinned Jetson"], + ["Node.js version", { unsupportedNode: true }, "must be Node.js 22.19.0 or later"], + ] satisfies Array< + [string, DeployOptions, string] + >)("rejects initial deployment when the prepared %s is invalid (#8142)", (_name, options, error) => { + const fixture = createFixture(); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], options); + const paths = deploymentPaths(fixture); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(error); + expect(fs.existsSync(paths.current)).toBe(false); + expect(fs.existsSync(paths.environment)).toBe(false); + expect(fs.existsSync(paths.unit)).toBe(false); + }); + + it("rejects an unmanaged cleanup path before initial deployment (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + fs.mkdirSync(path.dirname(paths.cleanup), { recursive: true }); + fs.writeFileSync(paths.cleanup, "unmanaged\n"); + + const result = runDeploy(fixture, ["--commit", fixture.firstSha]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("exists without one managed current release"); + expect(fs.readFileSync(paths.cleanup, "utf8")).toBe("unmanaged\n"); + expect(fs.existsSync(paths.environment)).toBe(false); + expect(fs.existsSync(paths.unit)).toBe(false); + }); + + it("rejects an unmanaged current link before initial deployment (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + fs.mkdirSync(path.dirname(paths.current), { recursive: true }); + fs.symlinkSync(fixture.repository, paths.current); + + const result = runDeploy(fixture, ["--commit", fixture.firstSha]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("does not select one managed release"); + expect(fs.readlinkSync(paths.current)).toBe(fixture.repository); + expect(fs.existsSync(paths.cleanup)).toBe(false); + expect(fs.existsSync(paths.environment)).toBe(false); + expect(fs.existsSync(paths.unit)).toBe(false); + }); + + it("rejects initial deployment while public ingress is active (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { + tunnelActive: true, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("must be disabled and inactive"); + expect(fs.existsSync(paths.current)).toBe(false); + expect(fs.existsSync(paths.environment)).toBe(false); + expect(fs.existsSync(paths.unit)).toBe(false); + }); + + it("rolls back initial deployment when public ingress becomes active (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { + tunnelActivatesAfterCheck: 1, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("release, unit, and environment were rolled back"); + expect(fs.existsSync(paths.current)).toBe(false); + expect(fs.existsSync(paths.cleanup)).toBe(false); + expect(fs.existsSync(paths.environment)).toBe(false); + expect(fs.existsSync(paths.unit)).toBe(false); + }); + + it("rolls back the initial release, unit, and environment when verification fails (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { + failVerifySha: fixture.firstSha, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("release, unit, and environment were rolled back"); + expect(fs.existsSync(paths.current)).toBe(false); + expect(fs.existsSync(paths.cleanup)).toBe(false); + expect(fs.existsSync(paths.environment)).toBe(false); + expect(fs.existsSync(paths.unit)).toBe(false); + expect(readIfPresent(paths.serviceLog)).toMatch( + /systemctl\tenable --now nemoclaw-jetson-dispatch\.service[\s\S]*systemctl\tdisable --now nemoclaw-jetson-dispatch\.service[\s\S]*systemctl\tdaemon-reload/u, + ); + }); + + it("reports when initial rollback cannot disable the service (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { + failInitialRollbackDisable: true, + failVerifySha: fixture.firstSha, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("rollback did not restore the prepared host"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + expect(fs.existsSync(paths.environment)).toBe(true); + expect(fs.existsSync(paths.unit)).toBe(true); + }); + + it("reports inconclusive containment when ingress activates and initial disable fails (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { + failInitialRollbackDisable: true, + tunnelActivatesAfterCheck: 2, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("rollback did not restore the prepared host"); + expect(fs.existsSync(path.join(fixture.directory, "service-active"))).toBe(true); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.existsSync(paths.environment)).toBe(true); + expect(fs.existsSync(paths.unit)).toBe(true); + }); + + it("stops, switches, starts, and verifies an installed loopback service (#8142)", () => { + const fixture = createFixture(); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + serviceLoadState: "loaded", + }); + const serviceLog = fs.readFileSync(deploymentPaths(fixture).serviceLog, "utf8"); + const curlLog = fs.readFileSync(path.join(fixture.directory, "curl.log"), "utf8"); + + expect(result.status).toBe(0); + expect(serviceLog).toMatch( + /systemctl\tstop nemoclaw-jetson-dispatch\.service[\s\S]*start-sha\t[0-9a-f]{40}/u, + ); + expect(result.stdout).toContain("verified its loopback service"); + expect(curlLog).toContain("curl\t--disable\t--noproxy\t*\t--silent"); + }); + + it("rejects a later dispatcher deployment while public ingress is active (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + const serviceLogBeforeDeployment = fs.readFileSync(paths.serviceLog, "utf8"); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + serviceLoadState: "loaded", + tunnelActive: true, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("must be disabled and inactive"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + const deploymentServiceLog = fs + .readFileSync(paths.serviceLog, "utf8") + .slice(serviceLogBeforeDeployment.length); + expect(deploymentServiceLog).not.toContain("systemctl\tstop"); + expect(deploymentServiceLog).not.toContain("start-sha"); + }); + + it.each([ + ["after the dispatcher starts", 3], + ["after loopback verification", 4], + ])("leaves the dispatcher stopped when public ingress activates %s (#8142)", (_phase, check) => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + fs.rmSync(path.join(fixture.directory, "tunnel-check-count"), { force: true }); + const serviceLogBeforeDeployment = fs.readFileSync(paths.serviceLog, "utf8"); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + serviceLoadState: "loaded", + tunnelActivatesAfterCheck: check, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("rollback did not restore a verified service"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.existsSync(path.join(fixture.directory, "service-active"))).toBe(false); + const deploymentServiceLog = fs + .readFileSync(paths.serviceLog, "utf8") + .slice(serviceLogBeforeDeployment.length); + expect(deploymentServiceLog).toContain(`start-sha\t${fixture.secondSha}`); + expect(deploymentServiceLog).not.toContain(`start-sha\t${fixture.firstSha}`); + }); + + it("reports inconclusive dispatcher state when ingress containment cannot stop it (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + fs.rmSync(path.join(fixture.directory, "tunnel-check-count"), { force: true }); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + failRollbackStop: true, + serviceLoadState: "loaded", + tunnelActivatesAfterCheck: 3, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("rollback did not restore a verified service"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.secondSha)); + expect(fs.existsSync(path.join(fixture.directory, "service-active"))).toBe(true); + }); + + it.each([ + ["a branch", ["--commit", "main"]], + ["a tag", ["--commit", "v0.1.0"]], + ["a URL", ["--commit", FIXED_ORIGIN]], + ["an abbreviated SHA", ["--commit", "a".repeat(12)]], + ["an uppercase SHA", ["--commit", "A".repeat(40)]], + ["an extra argument", ["--commit", "a".repeat(40), "extra"]], + ["a missing flag", ["a".repeat(40)]], + ])("rejects %s before deployment (#8142)", (_name, args) => { + const result = runDeploy(createFixture(), args); + + expect(result.status).toBe(2); + expect(result.stderr).toContain("--commit "); + }); + + it("rejects deployment by a non-root caller (#8142)", () => { + const fixture = createFixture(); + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { fakeUid: "1000" }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("this command requires root"); + }); + + it("rejects a release whose configured origin is not the fixed repository (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + git( + path.join(paths.releases, fixture.firstSha), + "remote", + "set-url", + "origin", + "https://example.test/other.git", + ); + + const result = runDeploy(fixture, ["--commit", fixture.firstSha]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(`release origin is not ${FIXED_ORIGIN}`); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + }); + + it("rejects a modified release checkout (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + fs.appendFileSync( + path.join(paths.releases, fixture.firstSha, "tools/e2e/jetson-dispatch-cleanup.sh"), + "modified\n", + ); + + const result = runDeploy(fixture, ["--commit", fixture.firstSha]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("release checkout is modified"); + }); + + it("rejects a current link that does not select one managed release (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + fs.unlinkSync(paths.current); + fs.symlinkSync(fixture.repository, paths.current); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("does not select one managed release"); + expect(fs.readlinkSync(paths.current)).toBe(fixture.repository); + }); + + it("rolls back code and cleanup when service startup fails (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + failStartSha: fixture.secondSha, + serviceLoadState: "loaded", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("previous deployment state was restored"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + }); + + it("rejects an inconclusive service state after stopping (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + activeStateError: true, + serviceLoadState: "loaded", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "could not inspect nemoclaw-jetson-dispatch.service after stop", + ); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + }); + + it("does not switch releases when service cleanup leaves the device lock (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + leaveDeviceLock: true, + serviceLoadState: "loaded", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("recover cleanup before deployment"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + }); + + it("rejects bootstrap when a stale device lock remains (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + + const result = runDeploy(fixture, ["--commit", fixture.firstSha], { + initialDeviceLock: true, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("recover cleanup before deployment"); + expect(fs.existsSync(paths.current)).toBe(false); + expect(fs.existsSync(paths.cleanup)).toBe(false); + expect(fs.existsSync(paths.releases)).toBe(true); + expect(fs.readdirSync(paths.releases)).toEqual([]); + }); + + it("redeploys the selected commit without creating another release (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + expect(fs.readdirSync(paths.releases)).toEqual([fixture.firstSha]); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + }); + + it("rolls back code and cleanup when loopback verification fails (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + failVerifySha: fixture.secondSha, + serviceLoadState: "loaded", + }); + const serviceLog = fs.readFileSync(paths.serviceLog, "utf8"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("previous deployment state was restored"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + expect(serviceLog).toContain(`start-sha\t${fixture.secondSha}`); + expect(serviceLog).toContain(`start-sha\t${fixture.firstSha}`); + }); + + it("keeps code and cleanup on the prior release when selection fails (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + failSelectSha: fixture.secondSha, + serviceLoadState: "loaded", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("previous deployment state was restored"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); + expect(fs.readlinkSync(paths.cleanup)).toBe( + path.join(paths.current, "tools/e2e/jetson-dispatch-cleanup.sh"), + ); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); + }); + + it("keeps the new code and cleanup paired when rollback cannot stop the service (#8142)", () => { + const fixture = createFixture(); + const paths = deploymentPaths(fixture); + expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); + + const result = runDeploy(fixture, ["--commit", fixture.secondSha], { + failRollbackStop: true, + failVerifySha: fixture.secondSha, + serviceLoadState: "loaded", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("rollback did not restore a verified service"); + expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.secondSha)); + expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("second-release"); + }); +}); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 94811be309a..9a789dd6a82 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -14,6 +14,7 @@ import { validateE2eOperationsWorkflowBoundary, } from "../../../tools/e2e/operations-workflow-boundary.mts"; import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts"; +import { testTimeoutOptions } from "../../helpers/timeouts.ts"; const AsyncFunction = Object.getPrototypeOf(async () => undefined).constructor as new ( ...parameters: string[] @@ -26,7 +27,7 @@ function workflowScript(jobName: string, stepName: string): string { return step?.with?.script as string; } -describe("E2E operations workflow boundary", () => { +describe("E2E operations workflow boundary", testTimeoutOptions(15_000), () => { it("accepts the checked-in workflow and rejects aggregation, permission, and secret-scope drift", () => { expect(validateE2eOperationsWorkflowBoundary()).toEqual([]); @@ -228,18 +229,18 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); - it("limits manual PR runs to credential-free selectors for the commit under review", () => { + it("limits manual PR runs to trusted controller selectors or opted-in Jetson dispatch", () => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", )!; authentication.run = authentication.run!.replace( - "Manual PR E2E accepts only empty selectors, inference-routing, or managed-image-protected-runtime", + "Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, or jetson-nvmap-gpu with its dispatch flag", "Manual PR E2E accepts arbitrary selectors", ); expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR authentication must retain Manual PR E2E accepts only empty selectors, inference-routing, or managed-image-protected-runtime", + "Manual PR authentication must retain Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, or jetson-nvmap-gpu with its dispatch flag", ); }); @@ -248,31 +249,23 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", )!; - authentication.run = authentication.run!.replace("inference-routing::false | ", ""); + authentication.run = authentication.run!.replace("inference-routing::false:false | ", ""); expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR authentication must retain ::false | inference-routing::false | managed-image-protected-runtime::false) ;;", + "Manual PR authentication must retain ::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | :jetson-nvmap-gpu:false:true) ;;", ); }); it.each([ - ["maintain", "", 0, ""], - ["maintain", "inference-routing", 0, ""], - ["maintain", "managed-image-protected-runtime", 0, ""], - [ - "maintain", - "network-policy", - 1, - "accepts only empty selectors, inference-routing, or managed-image-protected-runtime", - ], - [ - "maintain", - "gpu-e2e", - 1, - "accepts only empty selectors, inference-routing, or managed-image-protected-runtime", - ], - ["write", "", 1, "requires a repository maintainer or administrator"], - ])("requires a maintainer role and bounded selector before manual PR E2E for %s with %s", (role, jobs, expectedStatus, expectedStderr) => { + ["maintain", "", "", "false", 0, ""], + ["maintain", "inference-routing", "", "false", 0, ""], + ["maintain", "managed-image-protected-runtime", "", "false", 0, ""], + ["maintain", "", "jetson-nvmap-gpu", "true", 0, ""], + ["maintain", "", "jetson-nvmap-gpu", "false", 1, "accepts only empty selectors"], + ["maintain", "network-policy", "", "false", 1, "accepts only empty selectors"], + ["maintain", "gpu-e2e", "", "false", 1, "accepts only empty selectors"], + ["write", "", "", "false", 1, "requires a repository maintainer or administrator"], + ])("requires a maintainer role and bounded selector before manual PR E2E for %s with jobs %s and targets %s", (role, jobs, targets, allowJetsonDispatch, expectedStatus, expectedStderr) => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", @@ -297,6 +290,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; env: { ...process.env, ACTOR: "maintainer", + ALLOW_JETSON_DISPATCH: allowJetsonDispatch, BASE_SHA: baseSha, CHECKOUT_REPOSITORY: "contributor/NemoClaw", CHECKOUT_SHA: headSha, @@ -308,7 +302,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; PR_NUMBER: "42", REVIEW_REASON: "Reviewed PR head revision", RUN_ATTEMPT: "1", - TARGETS: "", + TARGETS: targets, TRIGGERING_ACTOR: "maintainer", WORKFLOW_EVENT: "workflow_dispatch", WORKFLOW_REF: "refs/heads/main", @@ -420,9 +414,10 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }); it.each([ - "inference-routing", - "managed-image-protected-runtime", - ])("selects no shared targets for the %s job selector", (jobSelector) => { + ["inference-routing job", "inference-routing", ""], + ["managed-image-protected-runtime job", "managed-image-protected-runtime", ""], + ["jetson-nvmap-gpu target", "", "jetson-nvmap-gpu"], + ])("selects no shared targets for the %s selector", (_name, jobSelector, targetSelector) => { const workflow = readE2eOperationsWorkflow(); const controller = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Build trusted controller target matrix", @@ -446,7 +441,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ...process.env, GITHUB_OUTPUT: output, JOBS: jobSelector, - TARGETS: "", + TARGETS: targetSelector, }, }, ); @@ -469,7 +464,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: jobSelector, - TARGETS: "", + TARGETS: targetSelector, }, }, ); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 9775e6a96be..1b3b06790d3 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -197,11 +197,13 @@ describe("e2e workflow boundary", () => { }; workflow.concurrency.group = "e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'pr-gate' || inputs.jobs || 'all-jobs' }}"; + workflow.concurrency["cancel-in-progress"] = "${{ inputs.checkout_sha != '' }}"; delete workflow.jobs["staging-brev-launchable"]!.concurrency!.queue; expect(validateE2eWorkflow(workflow)).toEqual( expect.arrayContaining([ "workflow concurrency must isolate each full dispatch with github.run_id", + "workflow concurrency must not cancel an active Jetson dispatch", "staging-brev-launchable concurrency must queue all pending Launchable E2E runs without cancellation", ]), ); diff --git a/test/e2e/support/jetson-dispatch-contract.test.ts b/test/e2e/support/jetson-dispatch-contract.test.ts new file mode 100644 index 00000000000..287452a0e44 --- /dev/null +++ b/test/e2e/support/jetson-dispatch-contract.test.ts @@ -0,0 +1,1022 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createSign, generateKeyPairSync, type KeyObject } from "node:crypto"; +import fs from "node:fs"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createGitHubOidcTokenProvider, + pollJetsonDispatch, +} from "../../../tools/e2e/jetson-dispatch-client.mts"; +import { + createGitHubJwksResolver, + JETSON_DISPATCH_AUDIENCE, + JETSON_DISPATCH_REPOSITORY, + JETSON_DISPATCH_TARGET, + JETSON_DISPATCH_WORKFLOW_REF, + parseJetsonDispatchRequest, + verifyGitHubOidcToken, +} from "../../../tools/e2e/jetson-dispatch-contract.mts"; +import { + JetsonDispatchBusyError, + JetsonDispatchCoordinator, + type JetsonDispatchStatus, + type JetsonDispatchWorker, + MAX_JETSON_ARTIFACT_ARCHIVE_BYTES, + MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, + MAX_JETSON_DISPATCH_LOG_BYTES, +} from "../../../tools/e2e/jetson-dispatch-lifecycle.mts"; +import { createJetsonDispatchServer } from "../../../tools/e2e/jetson-dispatch-service.mts"; +import { testTimeoutOptions } from "../../helpers/timeouts.ts"; + +const NOW_SECONDS = 1_800_000_000; +const REPOSITORY_ID = "987654321"; +const CANDIDATE_SHA = "a".repeat(40); +const WORKFLOW_SHA = "b".repeat(40); +const ARTIFACT_ARCHIVE_BASE64 = Buffer.from("bounded archive").toString("base64"); +const REQUEST = parseJetsonDispatchRequest({ + schemaVersion: 1, + target: JETSON_DISPATCH_TARGET, + candidateSha: CANDIDATE_SHA, + workflowRunId: "123456789", + workflowRunAttempt: 1, +}); + +const temporaryDirectories: string[] = []; + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +function temporaryDirectory(): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-jetson-dispatch-")); + temporaryDirectories.push(directory); + return directory; +} + +function failTest(message: string): never { + throw new Error(message); +} + +function claims(overrides: Record = {}): Record { + return { + iss: "https://token.actions.githubusercontent.com", + aud: JETSON_DISPATCH_AUDIENCE, + sub: "repo:NVIDIA/NemoClaw:ref:refs/heads/main", + repository: JETSON_DISPATCH_REPOSITORY, + repository_id: REPOSITORY_ID, + workflow_ref: JETSON_DISPATCH_WORKFLOW_REF, + workflow_sha: WORKFLOW_SHA, + sha: WORKFLOW_SHA, + ref: "refs/heads/main", + event_name: "workflow_dispatch", + runner_environment: "github-hosted", + run_id: REQUEST.workflowRunId, + run_attempt: String(REQUEST.workflowRunAttempt), + jti: "token-id", + iat: NOW_SECONDS - 60, + nbf: NOW_SECONDS - 60, + exp: NOW_SECONDS + 9 * 60, + ...overrides, + }; +} + +function signToken(privateKey: KeyObject, payload: Record): string { + const header = Buffer.from( + JSON.stringify({ alg: "RS256", kid: "test-key", typ: "JWT" }), + ).toString("base64url"); + const body = Buffer.from(JSON.stringify(payload)).toString("base64url"); + const signer = createSign("RSA-SHA256"); + signer.update(`${header}.${body}`, "ascii"); + signer.end(); + return `${header}.${body}.${signer.sign(privateKey).toString("base64url")}`; +} + +function signingKeys(): { privateKey: KeyObject; publicKey: KeyObject } { + return generateKeyPairSync("rsa", { modulusLength: 2048 }); +} + +function identity() { + return { + repository: JETSON_DISPATCH_REPOSITORY, + repositoryId: REPOSITORY_ID, + runId: REQUEST.workflowRunId, + runAttempt: REQUEST.workflowRunAttempt, + workflowSha: WORKFLOW_SHA, + tokenId: "token-id", + } as const; +} + +async function waitForCompletion( + coordinator: JetsonDispatchCoordinator, + jobId: string, +): Promise { + await vi.waitFor(() => expect(coordinator.status(jobId).state).toBe("completed"), { + interval: 1, + timeout: 1_000, + }); + return coordinator.status(jobId); +} + +function worker( + options: { cleanup?: JetsonDispatchWorker["cleanup"]; run?: JetsonDispatchWorker["run"] } = {}, +): JetsonDispatchWorker { + return { + run: + options.run ?? + (async () => ({ + artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, + device: { + model: "NVIDIA Jetson AGX Thor Developer Kit", + jetpackVersion: "7.2.2", + jetsonLinuxRelease: "R38", + kernel: "6.8.12-tegra", + }, + log: "passed\n", + })), + cleanup: options.cleanup ?? (async () => {}), + }; +} + +function coordinator(stateDirectory: string, dispatchWorker: JetsonDispatchWorker) { + return new JetsonDispatchCoordinator({ + stateDirectory, + worker: dispatchWorker, + executionTimeoutMs: 60_000, + cleanupTimeoutMs: 10_000, + }); +} + +describe("Jetson dispatch request and OIDC boundary", () => { + it("accepts only the fixed target and exact workflow run identity (#8142)", () => { + expect(REQUEST).toEqual({ + schemaVersion: 1, + target: "jetson-nvmap-gpu", + candidateSha: CANDIDATE_SHA, + workflowRunId: "123456789", + workflowRunAttempt: 1, + }); + }); + + it.each([ + "command", + "host", + "identityFile", + "repository", + "url", + ])("rejects request-controlled %s fields (#8142)", (field) => { + expect(() => parseJetsonDispatchRequest({ ...REQUEST, [field]: "unsafe" })).toThrow( + "dispatch request fields must match the fixed Jetson contract", + ); + }); + + it("verifies the signature and all trusted GitHub controller claims (#8142)", async () => { + const keys = signingKeys(); + const verified = await verifyGitHubOidcToken({ + token: signToken(keys.privateKey, claims()), + request: REQUEST, + policy: { repositoryId: REPOSITORY_ID }, + resolveSigningKey: async () => keys.publicKey, + nowMs: NOW_SECONDS * 1_000, + }); + + expect(verified).toEqual(identity()); + }); + + it("reuses each GitHub OIDC token for a bounded polling interval (#8142)", async () => { + let nowMs = 1_000; + const fetchImpl = vi.fn(async () => Response.json({ value: `token-${nowMs}` })); + const token = createGitHubOidcTokenProvider({ fetchImpl, now: () => nowMs }); + const env = { + ACTIONS_ID_TOKEN_REQUEST_TOKEN: "request-token", + ACTIONS_ID_TOKEN_REQUEST_URL: "https://token.actions.test/id-token", + }; + + await expect(token(env)).resolves.toBe("token-1000"); + nowMs += 60_000; + await expect(token(env)).resolves.toBe("token-1000"); + nowMs += 4 * 60_000; + await expect(token(env)).resolves.toBe("token-301000"); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it.each([ + ["issuer", { iss: "https://example.test" }], + ["audience", { aud: "another-service" }], + ["repository", { repository: "someone/NemoClaw" }], + ["repository ID", { repository_id: "123" }], + ["workflow", { workflow_ref: "NVIDIA/NemoClaw/.github/workflows/other.yaml@refs/heads/main" }], + ["ref", { ref: "refs/heads/feature" }], + ["event", { event_name: "pull_request" }], + ["runner", { runner_environment: "self-hosted" }], + ["run ID", { run_id: "999" }], + ["run attempt", { run_attempt: "2" }], + ["workflow SHA", { sha: "c".repeat(40) }], + ])("rejects a mismatched %s claim (#8142)", async (_name, override) => { + const keys = signingKeys(); + await expect( + verifyGitHubOidcToken({ + token: signToken(keys.privateKey, claims(override)), + request: REQUEST, + policy: { repositoryId: REPOSITORY_ID }, + resolveSigningKey: async () => keys.publicKey, + nowMs: NOW_SECONDS * 1_000, + }), + ).rejects.toThrow("OIDC claims do not match the trusted Jetson controller"); + }); + + it("rejects expired and incorrectly signed controller tokens (#8142)", async () => { + const trusted = signingKeys(); + const untrusted = signingKeys(); + await expect( + verifyGitHubOidcToken({ + token: signToken(trusted.privateKey, claims({ exp: NOW_SECONDS - 60 })), + request: REQUEST, + policy: { repositoryId: REPOSITORY_ID }, + resolveSigningKey: async () => trusted.publicKey, + nowMs: NOW_SECONDS * 1_000, + }), + ).rejects.toThrow("OIDC token is outside its allowed validity window"); + await expect( + verifyGitHubOidcToken({ + token: signToken(untrusted.privateKey, claims()), + request: REQUEST, + policy: { repositoryId: REPOSITORY_ID }, + resolveSigningKey: async () => trusted.publicKey, + nowMs: NOW_SECONDS * 1_000, + }), + ).rejects.toThrow("OIDC signature verification failed"); + }); + + it("coalesces key-set loads and bounds unknown-key refreshes (#8142)", async () => { + const keys = signingKeys(); + const jwk = keys.publicKey.export({ format: "jwk" }); + const fetchImpl = vi.fn(async () => + Response.json({ keys: [{ ...jwk, alg: "RS256", kid: "known", use: "sig" }] }), + ); + const resolve = createGitHubJwksResolver(fetchImpl as typeof fetch); + + await Promise.all([ + expect(resolve("missing-1")).rejects.toThrow("OIDC signing key is unknown"), + expect(resolve("missing-2")).rejects.toThrow("OIDC signing key is unknown"), + ]); + await expect(resolve("missing-3")).rejects.toThrow("OIDC signing key is unknown"); + await expect(resolve("known")).resolves.toMatchObject({ type: "public" }); + + expect(fetchImpl).toHaveBeenCalledOnce(); + }); +}); + +describe("Jetson single-device lifecycle", () => { + it("keeps the maximum log and archive response within the controller bound (#8142)", () => { + const artifact = { + artifactArchiveBase64: Buffer.alloc(MAX_JETSON_ARTIFACT_ARCHIVE_BYTES).toString("base64"), + log: "\0".repeat(MAX_JETSON_DISPATCH_LOG_BYTES), + status: { + schemaVersion: 1, + jobId: "d".repeat(64), + request: REQUEST, + state: "completed", + conclusion: "success", + createdAt: new Date(0).toISOString(), + completedAt: new Date(0).toISOString(), + cleanup: "succeeded", + }, + }; + + expect(Buffer.byteLength(`${JSON.stringify(artifact)}\n`)).toBeLessThanOrEqual( + MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, + ); + }); + + it("records success only after cleanup succeeds (#8142)", async () => { + const cleanup = vi.fn(async () => {}); + const dispatch = coordinator(temporaryDirectory(), worker({ cleanup })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ + state: "completed", + conclusion: "success", + cleanup: "succeeded", + device: { model: "NVIDIA Jetson AGX Thor Developer Kit" }, + }); + expect(cleanup).toHaveBeenCalledOnce(); + expect(dispatch.artifact(accepted.jobId)).toMatchObject({ + artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, + log: "passed\n", + }); + }); + + it("records test failure after cleanup succeeds (#8142)", async () => { + const cleanup = vi.fn(async () => {}); + const dispatch = coordinator( + temporaryDirectory(), + worker({ cleanup, run: async () => Promise.reject(new Error("test failed")) }), + ); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ + cleanup: "succeeded", + conclusion: "failure", + error: "test failed", + }); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("returns completed evidence when its result log is missing (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const dispatch = coordinator(stateDirectory, worker()); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + await waitForCompletion(dispatch, accepted.jobId); + fs.rmSync(path.join(stateDirectory, `${accepted.jobId}.log`)); + + expect(dispatch.artifact(accepted.jobId)).toMatchObject({ + artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, + log: "", + status: { conclusion: "success", state: "completed" }, + }); + }); + + it("rejects a second job while the Jetson lock is held (#8142)", async () => { + let finish: (() => void) | undefined; + const run = vi.fn( + () => + new Promise((_resolve, reject) => { + finish = () => reject(new Error("finished by test")); + }), + ); + const dispatch = coordinator(temporaryDirectory(), worker({ run })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const other = { ...REQUEST, candidateSha: "c".repeat(40) }; + expect(() => dispatch.dispatch(other, identity())).toThrow(JetsonDispatchBusyError); + finish!(); + await waitForCompletion(dispatch, accepted.jobId); + }); + + it("persists the lock and queued status before worker execution (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const events: string[] = []; + const fsyncSync = fs.fsyncSync.bind(fs); + vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { + events.push(fs.fstatSync(descriptor).isDirectory() ? "directory-fsync" : "file-fsync"); + fsyncSync(descriptor); + }); + const unlinkSync = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((file) => { + events.push(file === path.join(stateDirectory, "device.lock") ? "lock-unlink" : "unlink"); + unlinkSync(file); + }); + const run = vi.fn(async (_request, options) => { + events.push("worker-run"); + expect(fs.readFileSync(path.join(stateDirectory, "device.lock"), "utf8")).toBe( + `${options.jobId}\n`, + ); + expect( + JSON.parse(fs.readFileSync(path.join(stateDirectory, `${options.jobId}.json`), "utf8")), + ).toMatchObject({ jobId: options.jobId, state: "queued" }); + return worker().run(_request, options); + }); + const dispatch = coordinator(stateDirectory, worker({ run })); + await dispatch.initialize(); + + const accepted = dispatch.dispatch(REQUEST, identity()); + await waitForCompletion(dispatch, accepted.jobId); + + expect(events.indexOf("directory-fsync")).toBeLessThan(events.indexOf("worker-run")); + expect(events.indexOf("worker-run")).toBeLessThan(events.indexOf("lock-unlink")); + expect(events.indexOf("lock-unlink")).toBeLessThan(events.lastIndexOf("directory-fsync")); + }); + + it("does not start the worker when state directory persistence fails (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const fsyncSync = fs.fsyncSync.bind(fs); + let rejectDirectoryFsync = true; + vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { + const isDirectory = fs.fstatSync(descriptor).isDirectory(); + const shouldReject = rejectDirectoryFsync && isDirectory; + rejectDirectoryFsync = rejectDirectoryFsync && !isDirectory; + return shouldReject ? failTest("state directory persistence failed") : fsyncSync(descriptor); + }); + const run = vi.fn(worker().run); + const cleanup = vi.fn(async () => {}); + const dispatch = coordinator(stateDirectory, worker({ cleanup, run })); + await dispatch.initialize(); + + expect(() => dispatch.dispatch(REQUEST, identity())).toThrow( + "state directory persistence failed", + ); + + expect(run).not.toHaveBeenCalled(); + expect(cleanup).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + + const restarted = coordinator(stateDirectory, worker({ cleanup, run })); + await restarted.initialize(); + + expect(cleanup).toHaveBeenCalledOnce(); + expect(run).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); + }); + + it("cancels the worker and still cleans the device with a fresh signal (#8142)", async () => { + let runSignal: AbortSignal | undefined; + let cleanupSignal: AbortSignal | undefined; + const cleanup = vi.fn(async (options: { jobId: string; signal: AbortSignal }) => { + cleanupSignal = options.signal; + }); + const run = vi.fn( + (_request, options) => + new Promise((_resolve, reject) => { + runSignal = options.signal; + options.signal.addEventListener("abort", () => reject(new Error("cancelled")), { + once: true, + }); + }), + ); + const dispatch = coordinator(temporaryDirectory(), worker({ cleanup, run })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + dispatch.cancel(accepted.jobId); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ cleanup: "succeeded", conclusion: "cancelled" }); + expect(cleanup).toHaveBeenCalledOnce(); + expect(cleanupSignal).not.toBe(runSignal); + expect(cleanupSignal?.aborted).toBe(false); + }); + + it("times out the worker and still cleans the device (#8142)", async () => { + vi.useFakeTimers(); + const cleanup = vi.fn(async () => {}); + const run = vi.fn( + (_request, options) => + new Promise((_resolve, reject) => { + options.signal.addEventListener("abort", () => reject(new Error("timed out")), { + once: true, + }); + }), + ); + const dispatch = coordinator(temporaryDirectory(), worker({ cleanup, run })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + await vi.advanceTimersByTimeAsync(60_000); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ cleanup: "succeeded", conclusion: "timed-out" }); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("keeps cleanup evidence accurate when lock removal fails (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const cleanup = vi.fn(async () => {}); + const unlinkSync = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync") + .mockImplementationOnce(() => { + throw new Error("lock filesystem unavailable"); + }) + .mockImplementation(unlinkSync); + const dispatch = coordinator(stateDirectory, worker({ cleanup })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ + conclusion: "cleanup-failed", + error: "Jetson lock removal failed: lock filesystem unavailable", + cleanup: "succeeded", + }); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + + const restarted = coordinator(stateDirectory, worker({ cleanup })); + await restarted.initialize(); + + expect(cleanup).toHaveBeenCalledTimes(2); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); + }); + + it("restores the device lock when directory fsync fails after removal (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const cleanup = vi.fn(async () => {}); + const fsyncSync = fs.fsyncSync.bind(fs); + let directorySyncCount = 0; + vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { + const isDirectory = fs.fstatSync(descriptor).isDirectory(); + directorySyncCount += Number(isDirectory); + return isDirectory && directorySyncCount === 2 + ? failTest("lock directory persistence failed") + : fsyncSync(descriptor); + }); + const dispatch = coordinator(stateDirectory, worker({ cleanup })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ + conclusion: "cleanup-failed", + error: "Jetson lock removal failed: lock directory persistence failed", + cleanup: "succeeded", + }); + expect(directorySyncCount).toBe(3); + expect(fs.readFileSync(path.join(stateDirectory, "device.lock"), "utf8")).toBe( + `${accepted.jobId}\n`, + ); + + const restarted = coordinator(stateDirectory, worker({ cleanup })); + await restarted.initialize(); + + expect(cleanup).toHaveBeenCalledTimes(2); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); + }); + + it("blocks later dispatch when lock restoration cannot recreate the lock (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const lockPath = path.join(stateDirectory, "device.lock"); + const fsyncSync = fs.fsyncSync.bind(fs); + let directorySyncCount = 0; + vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { + const isDirectory = fs.fstatSync(descriptor).isDirectory(); + directorySyncCount += Number(isDirectory); + return isDirectory && directorySyncCount === 2 + ? failTest("lock directory persistence failed") + : fsyncSync(descriptor); + }); + const run = vi.fn(worker().run); + const dispatch = coordinator(stateDirectory, worker({ run })); + await dispatch.initialize(); + const openSync = fs.openSync.bind(fs); + let lockOpenCount = 0; + vi.spyOn(fs, "openSync").mockImplementation((file, flags, mode) => { + const isLock = file === lockPath; + lockOpenCount += Number(isLock); + return isLock && lockOpenCount === 2 + ? failTest("lock restoration unavailable") + : openSync(file, flags, mode); + }); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed.error).toContain("lock restoration failed: lock restoration unavailable"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(() => + dispatch.dispatch({ ...REQUEST, candidateSha: "c".repeat(40) }, identity()), + ).toThrow("Jetson device lock requires recovery"); + expect(run).toHaveBeenCalledOnce(); + await expect(dispatch.shutdown()).rejects.toThrow( + "Jetson device lock state requires operator recovery", + ); + }); + + it("keeps the lock until restart persists a terminal result (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const expectedResult = await worker().run(REQUEST, { + jobId: "e".repeat(64), + signal: new AbortController().signal, + }); + let finishRun!: () => void; + const run = vi.fn(async () => { + await new Promise((resolve) => { + finishRun = resolve; + }); + return expectedResult; + }); + const first = coordinator(stateDirectory, worker({ run })); + await first.initialize(); + const accepted = first.dispatch(REQUEST, identity()); + await vi.waitFor(() => expect(run).toHaveBeenCalledOnce()); + const statusPath = path.join(stateDirectory, `${accepted.jobId}.json`); + const runningStatus = fs.readFileSync(statusPath, "utf8"); + fs.rmSync(statusPath); + fs.mkdirSync(statusPath); + + finishRun(); + const failedPersistence = await waitForCompletion(first, accepted.jobId); + + expect(failedPersistence).toMatchObject({ + cleanup: "succeeded", + conclusion: "failure", + error: expect.stringContaining("Final status persistence failed"), + }); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + + const recoveryCleanup = vi.fn(async () => {}); + await expect( + coordinator(stateDirectory, worker({ cleanup: recoveryCleanup })).initialize(), + ).rejects.toThrow(`Jetson dispatch status ${accepted.jobId}.json is invalid`); + expect(recoveryCleanup).toHaveBeenCalledOnce(); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + + fs.rmSync(statusPath, { recursive: true }); + fs.writeFileSync(statusPath, runningStatus, { mode: 0o600 }); + const replayRun = vi.fn(worker().run); + const restarted = coordinator( + stateDirectory, + worker({ cleanup: recoveryCleanup, run: replayRun }), + ); + await restarted.initialize(); + + expect(restarted.status(accepted.jobId)).toMatchObject({ + cleanup: "succeeded", + conclusion: "failure", + error: "Jetson dispatcher restarted before terminal status was persisted", + state: "completed", + }); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); + expect(recoveryCleanup).toHaveBeenCalledTimes(2); + expect(restarted.dispatch(REQUEST, identity()).state).toBe("completed"); + expect(replayRun).not.toHaveBeenCalled(); + }); + + it("restores completed evidence without replaying the job after restart (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const first = coordinator(stateDirectory, worker()); + await first.initialize(); + const firstAccepted = first.dispatch(REQUEST, identity()); + const firstCompleted = await waitForCompletion(first, firstAccepted.jobId); + expect(first.artifact(firstAccepted.jobId).artifactArchiveBase64).toBe(ARTIFACT_ARCHIVE_BASE64); + + const replayRun = vi.fn(worker().run); + const restarted = coordinator(stateDirectory, worker({ run: replayRun })); + await restarted.initialize(); + const restoredStatus = restarted.status(firstAccepted.jobId); + const restoredArtifact = restarted.artifact(firstAccepted.jobId); + const replayAccepted = restarted.dispatch(REQUEST, identity()); + + expect(replayAccepted.jobId).toBe(firstAccepted.jobId); + expect(replayAccepted).toEqual(firstCompleted); + expect(restoredStatus).toEqual(firstCompleted); + expect(restoredArtifact).toMatchObject({ + artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, + log: "passed\n", + status: firstCompleted, + }); + expect(replayRun).not.toHaveBeenCalled(); + }); + + it("rejects initialization when a completed status record is invalid (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const first = coordinator(stateDirectory, worker()); + await first.initialize(); + const accepted = first.dispatch(REQUEST, identity()); + await waitForCompletion(first, accepted.jobId); + const statusPath = path.join(stateDirectory, `${accepted.jobId}.json`); + const persisted = JSON.parse(fs.readFileSync(statusPath, "utf8")); + const invalidRecords = [ + { + contents: `${JSON.stringify({ ...persisted, jobId: "f".repeat(64) })}\n`, + error: "persisted Jetson dispatch status does not match its job ID", + }, + { contents: "{", error: "persisted Jetson dispatch status is not valid JSON" }, + { contents: " ".repeat(16 * 1024 + 1), error: "exceeds 16384 bytes" }, + ]; + + for (const record of invalidRecords) { + fs.writeFileSync(statusPath, record.contents); + await expect(coordinator(stateDirectory, worker()).initialize()).rejects.toThrow( + new RegExp(`Jetson dispatch status ${accepted.jobId}\\.json is invalid: .*${record.error}`), + ); + } + }); + + it( + "restores private completed status after in-memory eviction (#8142)", + testTimeoutOptions(10_000), + async () => { + const stateDirectory = temporaryDirectory(); + const dispatch = coordinator(stateDirectory, worker()); + await dispatch.initialize(); + let firstJobId = ""; + for (let index = 0; index < 129; index += 1) { + const accepted = dispatch.dispatch( + { ...REQUEST, candidateSha: index.toString(16).padStart(40, "0") }, + identity(), + ); + firstJobId ||= accepted.jobId; + await waitForCompletion(dispatch, accepted.jobId); + } + + expect(dispatch.status(firstJobId)).toMatchObject({ state: "completed" }); + expect(fs.existsSync(path.join(stateDirectory, `${firstJobId}.json`))).toBe(true); + fs.writeFileSync(path.join(stateDirectory, `${firstJobId}.json`), "{"); + await expect(coordinator(stateDirectory, worker()).initialize()).rejects.toThrow( + `Jetson dispatch status ${firstJobId}.json is invalid`, + ); + }, + ); + + it("keeps the device locked when cleanup loses a pre-existing Ollama model (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const dispatch = coordinator( + stateDirectory, + worker({ + cleanup: async () => + Promise.reject( + new Error("Jetson cleanup did not preserve every pre-existing Ollama model"), + ), + }), + ); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ cleanup: "failed", conclusion: "cleanup-failed" }); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + expect(() => + dispatch.dispatch({ ...REQUEST, candidateSha: "c".repeat(40) }, identity()), + ).toThrow("Jetson device lock requires recovery"); + await expect(dispatch.shutdown()).rejects.toThrow( + "Jetson device lock still requires cleanup recovery", + ); + }); + + it("reports both candidate and cleanup failures while retaining the lock (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const dispatch = coordinator( + stateDirectory, + worker({ + cleanup: async () => Promise.reject(new Error("cleanup verification failed")), + run: async () => Promise.reject(new Error("artifact collection failed")), + }), + ); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed).toMatchObject({ + cleanup: "failed", + conclusion: "cleanup-failed", + error: "artifact collection failed; Jetson cleanup failed: cleanup verification failed", + }); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + }); + + it("keeps a cleanup failure visible after a 500-character execution error (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const dispatch = coordinator( + stateDirectory, + worker({ + cleanup: async () => Promise.reject(new Error("cleanup verification failed")), + run: async () => Promise.reject(new Error("x".repeat(500))), + }), + ); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed.error).toContain("Jetson cleanup failed: cleanup verification failed"); + expect(completed.error).toHaveLength(500); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + }); + + it("keeps a final persistence failure visible after a 500-character execution error (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + let rejectRun!: (error: Error) => void; + const run = vi.fn( + () => + new Promise((_resolve, reject) => { + rejectRun = reject; + }), + ); + const dispatch = coordinator(stateDirectory, worker({ run })); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + await vi.waitFor(() => expect(run).toHaveBeenCalledOnce()); + const statusPath = path.join(stateDirectory, `${accepted.jobId}.json`); + fs.rmSync(statusPath); + fs.mkdirSync(statusPath); + rejectRun(new Error("x".repeat(500))); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed.error).toContain("Final status persistence failed:"); + expect(completed.error).toHaveLength(500); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + }); + + it("keeps a lock-removal failure visible after a 500-character execution error (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { + throw new Error("lock filesystem unavailable"); + }); + const dispatch = coordinator( + stateDirectory, + worker({ run: async () => Promise.reject(new Error("x".repeat(500))) }), + ); + await dispatch.initialize(); + const accepted = dispatch.dispatch(REQUEST, identity()); + const completed = await waitForCompletion(dispatch, accepted.jobId); + + expect(completed.error).toContain("Jetson lock removal failed: lock filesystem unavailable"); + expect(completed.error).toHaveLength(500); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + }); + + it("cleans a stale lock before accepting work after restart (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + const staleJobId = "d".repeat(64); + fs.writeFileSync(path.join(stateDirectory, "device.lock"), `${staleJobId}\n`, { mode: 0o600 }); + const cleanup = vi.fn(async () => {}); + const dispatch = coordinator(stateDirectory, worker({ cleanup })); + await dispatch.initialize(); + + expect(cleanup).toHaveBeenCalledWith({ jobId: staleJobId, signal: expect.any(AbortSignal) }); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); + }); + + it("rejects a stale lock that cannot identify one dispatcher job (#8142)", async () => { + const stateDirectory = temporaryDirectory(); + fs.writeFileSync(path.join(stateDirectory, "device.lock"), "../../arbitrary-path\n", { + mode: 0o600, + }); + const cleanup = vi.fn(async () => {}); + const dispatch = coordinator(stateDirectory, worker({ cleanup })); + + await expect(dispatch.initialize()).rejects.toThrow( + "Jetson device lock contains an invalid job ID", + ); + expect(cleanup).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + }); + + it.each([ + `${"d".repeat(64)}\nextra\n`, + `${"d".repeat(64)}\n\n`, + ])("rejects trailing data in a stale device lock (#8142)", async (lockContents) => { + const stateDirectory = temporaryDirectory(); + fs.writeFileSync(path.join(stateDirectory, "device.lock"), lockContents, { mode: 0o600 }); + const cleanup = vi.fn(async () => {}); + const dispatch = coordinator(stateDirectory, worker({ cleanup })); + + await expect(dispatch.initialize()).rejects.toThrow( + "Jetson device lock contains an invalid job ID", + ); + expect(cleanup).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); + }); +}); + +describe("Jetson dispatch HTTP boundary", () => { + it("reports the controller deadline when cancellation also fails (#8142)", async () => { + const jobId = "d".repeat(64); + const request = vi.fn().mockRejectedValue(new Error("tunnel unavailable")); + + await expect( + pollJetsonDispatch({ + baseUrl: new URL("https://dispatch.test/"), + deadlineMs: 10_000, + initialStatus: { + schemaVersion: 1, + jobId, + request: REQUEST, + state: "queued", + createdAt: new Date(0).toISOString(), + cleanup: "pending", + }, + jobId, + now: () => 10_000, + request, + wait: async () => {}, + }), + ).rejects.toThrow("Jetson dispatcher did not complete before the controller deadline"); + expect(request).toHaveBeenCalledWith( + expect.objectContaining({ method: "DELETE", path: `v1/jobs/${jobId}` }), + ); + }); + + it("retries transient status failures and cancels after the bounded limit (#8142)", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const jobId = "d".repeat(64); + const initialStatus: JetsonDispatchStatus = { + schemaVersion: 1, + jobId, + request: REQUEST, + state: "queued", + createdAt: new Date(0).toISOString(), + cleanup: "pending", + }; + const completed = { + job: { + ...initialStatus, + state: "completed", + conclusion: "success", + cleanup: "succeeded", + }, + }; + const recoveredRequest = vi + .fn() + .mockRejectedValueOnce(new Error("tunnel reset")) + .mockRejectedValueOnce(new Error("tunnel reset")) + .mockResolvedValueOnce(completed); + + await expect( + pollJetsonDispatch({ + baseUrl: new URL("https://dispatch.test/"), + deadlineMs: 10_000, + initialStatus, + jobId, + now: () => 0, + request: recoveredRequest, + wait: async () => {}, + }), + ).resolves.toMatchObject({ state: "completed" }); + + const failedRequest = vi.fn().mockRejectedValue(new Error("tunnel unavailable")); + await expect( + pollJetsonDispatch({ + baseUrl: new URL("https://dispatch.test/"), + deadlineMs: 10_000, + initialStatus, + jobId, + now: () => 0, + request: failedRequest, + wait: async () => {}, + }), + ).rejects.toThrow("tunnel unavailable"); + expect(failedRequest).toHaveBeenLastCalledWith( + expect.objectContaining({ method: "DELETE", path: `v1/jobs/${jobId}` }), + ); + }); + + it("denies anonymous requests and returns authenticated result evidence (#8142)", async () => { + const keys = signingKeys(); + const now = Math.floor(Date.now() / 1_000); + const token = signToken( + keys.privateKey, + claims({ exp: now + 9 * 60, iat: now - 60, nbf: now - 60 }), + ); + const dispatch = coordinator(temporaryDirectory(), worker()); + await dispatch.initialize(); + const server = createJetsonDispatchServer({ + coordinator: dispatch, + repositoryId: REPOSITORY_ID, + resolveSigningKey: async () => keys.publicKey, + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const { port } = server.address() as AddressInfo; + const url = `http://127.0.0.1:${port}/v1/jobs`; + try { + const anonymous = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(REQUEST), + }); + expect(anonymous.status).toBe(401); + + const injected = await fetch(url, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify({ ...REQUEST, command: "uname -a" }), + }); + expect(injected.status).toBe(400); + + const accepted = await fetch(url, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify(REQUEST), + }); + expect(accepted.status).toBe(202); + const acceptedBody = (await accepted.json()) as { job: { jobId: string } }; + await waitForCompletion(dispatch, acceptedBody.job.jobId); + + const anonymousExisting = await fetch(`${url}/${acceptedBody.job.jobId}`); + const anonymousUnknown = await fetch(`${url}/${"f".repeat(64)}`); + const authenticatedUnknown = await fetch(`${url}/${"f".repeat(64)}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect([ + anonymousExisting.status, + anonymousUnknown.status, + authenticatedUnknown.status, + ]).toEqual([401, 401, 401]); + + const artifact = await fetch(`${url}/${acceptedBody.job.jobId}/artifact`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(artifact.status).toBe(200); + await expect(artifact.json()).resolves.toMatchObject({ + log: "passed\n", + status: { cleanup: "succeeded", conclusion: "success" }, + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); +}); diff --git a/test/e2e/support/jetson-dispatch-worker.test.ts b/test/e2e/support/jetson-dispatch-worker.test.ts new file mode 100644 index 00000000000..b352a9818ab --- /dev/null +++ b/test/e2e/support/jetson-dispatch-worker.test.ts @@ -0,0 +1,1411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + loadSshJetsonWorkerConfig, + ProcessFailure, + SshJetsonDispatchWorker, +} from "../../../tools/e2e/jetson-dispatch-worker.mts"; + +const dispatcherRunbook = fs.readFileSync( + path.join(process.cwd(), "test/e2e/docs/jetson-colossus-dispatch.md"), + "utf8", +); +const cleanupProgram = fs.readFileSync( + path.join(process.cwd(), "tools/e2e/jetson-dispatch-cleanup.sh"), + "utf8", +); +const dispatcherServiceUnit = fs.readFileSync( + path.join(process.cwd(), "tools/e2e/nemoclaw-jetson-dispatch.service"), + "utf8", +); +const workerProgram = fs.readFileSync( + path.join(process.cwd(), "tools/e2e/jetson-dispatch-worker.mts"), + "utf8", +); +const jetsonLiveTest = fs.readFileSync( + path.join(process.cwd(), "test/e2e/live/jetson-nvmap-gpu.test.ts"), + "utf8", +); +const temporaryDirectories: string[] = []; +const BASELINE = { + nodePath: "/usr/bin/node", + nodeVersion: "v22.19.0", + npmPath: "/usr/bin/npm", + npmVersion: "10.9.3", + ollamaPath: "/usr/local/bin/ollama", + ollamaModels: [{ name: "existing:latest", id: "a".repeat(12) }], + openshellState: "absent", +}; + +function baselineOutput(overrides: Partial = {}): string { + const baseline = { ...BASELINE, ...overrides }; + const ollamaModelsBase64 = Buffer.from( + baseline.ollamaModels.map((model) => `${model.name}\t${model.id}`).join("\n"), + ).toString("base64"); + return `${Object.entries({ + nodePath: baseline.nodePath, + nodeVersion: baseline.nodeVersion, + npmPath: baseline.npmPath, + npmVersion: baseline.npmVersion, + ollamaPath: baseline.ollamaPath, + ollamaModelsBase64, + openshellState: baseline.openshellState, + }) + .map(([key, value]) => `${key}\t${value}`) + .join("\n")}\n`; +} + +const BASELINE_OUTPUT = baselineOutput(); + +function cleanupOutput({ + volumes = [], + processIds = [], +}: { + volumes?: string[]; + processIds?: number[]; +} = {}): string { + return [ + "nemoclaw-cleanup-evidence-v1-begin", + ...volumes.map((volume) => `volume\t${volume}`), + ...processIds.map((pid) => `processId\t${pid}`), + "nemoclaw-cleanup-evidence-v1-end", + "", + ].join("\n"); +} + +function persistCleanupEvidence( + files: ReturnType, + jobId: string, + evidence: { schemaVersion: 1; volumes: string[]; processIds: number[] }, +): string { + const file = path.join(files.stateDirectory, `${jobId}.cleanup.json`); + fs.writeFileSync(file, `${JSON.stringify(evidence)}\n`, { mode: 0o600 }); + return file; +} + +function cleanupProcessFixture( + options: { + command?: string; + deny?: "cmdline" | "environ"; + disappear?: "cmdline" | "environ"; + home?: string; + uid?: number; + } = {}, +) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-jetson-proc-")); + temporaryDirectories.push(directory); + const procRoot = path.join(directory, "proc"); + const procDirectory = path.join(procRoot, "1234"); + const binDirectory = path.join(directory, "bin"); + fs.mkdirSync(procDirectory, { recursive: true }); + fs.mkdirSync(binDirectory); + fs.writeFileSync( + path.join(procDirectory, "status"), + `Name:\ttest\nUid:\t${options.uid ?? process.getuid!()}\t0\t0\t0\n`, + ); + fs.writeFileSync( + path.join(procDirectory, "environ"), + Buffer.from(`HOME=${options.home ?? "/job/home"}\0PATH=/usr/bin\0`), + ); + fs.writeFileSync( + path.join(procDirectory, "cmdline"), + Buffer.from(`${options.command ?? "node ollama-auth-proxy.mjs"}\0`), + ); + fs.writeFileSync( + path.join(binDirectory, "dd"), + `#!/usr/bin/env bash +set -eu +input= +for argument in "$@"; do + case "$argument" in if=*) input="${"${argument#if=}"}" ;; esac +done +if [ "${"${DENY_PROC_FILE:-}"}" = "$input" ]; then exit 1; fi +if [ "${"${DISAPPEAR_PROC_FILE:-}"}" = "$input" ]; then rm -rf -- "${"${input%/*}"}"; exit 1; fi +/bin/cat "$input" +`, + { mode: 0o700 }, + ); + const functionsStart = cleanupProgram.indexOf("read_proc_uid() {"); + const functionsEnd = cleanupProgram.indexOf("\nauth_proxy_pid=", functionsStart); + assert.ok(functionsStart >= 0 && functionsEnd >= 0, "cleanup process functions are absent"); + const functions = cleanupProgram + .slice(functionsStart, functionsEnd) + .replace('proc_dir="/proc/$pid"', 'proc_dir="$PROC_ROOT/$pid"'); + return { + binDirectory, + environment: { + ...process.env, + DENY_PROC_FILE: options.deny ? path.join(procDirectory, options.deny) : "", + DISAPPEAR_PROC_FILE: options.disappear ? path.join(procDirectory, options.disappear) : "", + PATH: `${binDirectory}:${process.env.PATH ?? ""}`, + PROC_ROOT: procRoot, + }, + functions, + procRoot, + }; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +function deploymentFiles() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-jetson-worker-")); + temporaryDirectories.push(directory); + const identityFile = path.join(directory, "id_ed25519"); + const knownHostsFile = path.join(directory, "known_hosts"); + const cleanupExecutable = path.join(directory, "cleanup-jetson"); + fs.writeFileSync(identityFile, "test-key", { mode: 0o600 }); + fs.writeFileSync(knownHostsFile, "test-host-key", { mode: 0o600 }); + fs.writeFileSync(cleanupExecutable, "#!/bin/sh\nexit 0\n", { mode: 0o700 }); + return { cleanupExecutable, identityFile, knownHostsFile, stateDirectory: directory }; +} + +function environment(files = deploymentFiles()): NodeJS.ProcessEnv { + return { + JETSON_DISPATCH_SSH_DESTINATION: "nvidia@192.168.55.1", + JETSON_DISPATCH_SSH_IDENTITY_FILE: files.identityFile, + JETSON_DISPATCH_SSH_KNOWN_HOSTS_FILE: files.knownHostsFile, + JETSON_DISPATCH_CLEANUP_EXECUTABLE: files.cleanupExecutable, + JETSON_DISPATCH_TEST_TIMEOUT_SECONDS: "2700", + }; +} + +function loadConfig(files = deploymentFiles()) { + return loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, environment(files)); +} + +describe("Colossus SSH worker deployment boundary", () => { + it("loads the dispatcher service with direct Node.js type stripping (#8142)", () => { + const service = pathToFileURL( + path.join(process.cwd(), "tools/e2e/jetson-dispatch-service.mts"), + ).href; + const result = spawnSync( + "node", + [ + "--experimental-strip-types", + "--no-warnings", + "--input-type=module", + "--eval", + `await import(${JSON.stringify(service)})`, + ], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).not.toContain("does not provide an export named JetsonDeviceIdentity"); + }); + + it("uses one fixed command for initial service deployment (#8142)", () => { + const deployment = + "sudo nemoclaw-colossus-jetson-dispatch-deploy --commit 0000000000000000000000000000000000000000"; + const configure = dispatcherRunbook.indexOf("## Configure the Dispatcher Service"); + expect(dispatcherRunbook).toContain( + "tools/e2e/colossus-jetson-dispatch-deploy.sh \\\n /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy", + ); + expect(dispatcherRunbook).toContain(deployment); + expect(dispatcherRunbook).toContain("tools/e2e/colossus-jetson-dispatch.environment"); + expect(dispatcherRunbook).toContain("tools/e2e/nemoclaw-jetson-dispatch.service"); + expect(dispatcherServiceUnit).toContain( + "WorkingDirectory=/opt/nemoclaw-jetson-dispatch/current", + ); + expect(dispatcherRunbook.indexOf(deployment)).toBeGreaterThan(configure); + expect(dispatcherRunbook.indexOf(deployment)).toBeLessThan( + dispatcherRunbook.indexOf("## Deploy a Later Reviewed Commit"), + ); + expect(dispatcherRunbook).not.toContain("sudo git -C /opt/nemoclaw-jetson-dispatch init"); + expect(dispatcherRunbook).not.toContain("git clone --branch main"); + }); + + it("distinguishes pinned SSH identity from the persisted cleanup baseline (#8142)", () => { + expect(dispatcherRunbook).toContain( + "The pinned `known_hosts` file validates the SSH host identity separately from the persisted cleanup baseline.", + ); + expect(dispatcherRunbook).not.toContain( + "Treat the verified SSH host identity as part of the protected baseline.", + ); + }); + + it("keeps public ingress stopped until a later release passes local verification (#8142)", () => { + const section = dispatcherRunbook.slice( + dispatcherRunbook.indexOf("## Deploy a Later Reviewed Commit"), + dispatcherRunbook.indexOf("## Publish the Dispatcher With Cloudflare Tunnel"), + ); + const stopTunnel = "sudo systemctl stop nemoclaw-jetson-tunnel.service"; + const deploy = "sudo /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy"; + const startTunnel = "sudo systemctl start nemoclaw-jetson-tunnel.service"; + const publicProof = 'test "$PUBLIC_HTTP_CODE" = 401'; + + expect(section).toContain("set -euo pipefail"); + expect(section.indexOf(stopTunnel)).toBeLessThan(section.indexOf(deploy)); + expect(section.indexOf(deploy)).toBeLessThan(section.indexOf(startTunnel)); + expect(section.indexOf(startTunnel)).toBeLessThan(section.indexOf(publicProof)); + expect(section.indexOf(publicProof)).toBeLessThan(section.indexOf("tunnel_verified=1")); + expect(section).toContain("trap stop_unverified_tunnel EXIT"); + expect(section).toContain("curl --disable --noproxy '*'"); + }); + + it.each([ + { + expectedContainmentDiagnostics: [], + expectedLog: ["stop", "deploy", "start", "stop"], + expectedState: "inactive\n", + failContainmentStop: false, + name: "stops public ingress when the public proof fails", + }, + { + expectedContainmentDiagnostics: [ + "PUBLIC INGRESS CONTAINMENT FAILED: tunnel stop status=5; ActiveState=active; inspection status=0", + ], + expectedLog: ["stop", "deploy", "start", "stop-failed"], + expectedState: "active\n", + failContainmentStop: true, + name: "reports when public-ingress containment fails", + }, + ])("$name (#8142)", ({ + expectedContainmentDiagnostics, + expectedLog, + expectedState, + failContainmentStop, + }) => { + const section = dispatcherRunbook.slice( + dispatcherRunbook.indexOf("## Deploy a Later Reviewed Commit"), + dispatcherRunbook.indexOf("## Publish the Dispatcher With Cloudflare Tunnel"), + ); + const block = section.match(/```bash\n([\s\S]*?)\n```/u)?.[1]; + expect(block).toBeDefined(); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-tunnel-deploy-test-")); + temporaryDirectories.push(directory); + const log = path.join(directory, "service.log"); + const state = path.join(directory, "tunnel.state"); + const stopCount = path.join(directory, "stop.count"); + fs.writeFileSync(state, "active\n"); + fs.writeFileSync( + path.join(directory, "sudo"), + `#!/bin/sh +set -eu +case "$1:$2" in + systemctl:stop) + count="$(cat "$STOP_COUNT" 2>/dev/null || printf '0')" + count="$((count + 1))" + printf '%s\\n' "$count" >"$STOP_COUNT" + if [ "$FAIL_CONTAINMENT_STOP" = 1 ] && [ "$count" -gt 1 ]; then + printf 'stop-failed\\n' >>"$SERVICE_LOG" + exit 5 + fi + printf 'stop\\n' >>"$SERVICE_LOG" + printf 'inactive\\n' >"$TUNNEL_STATE" + ;; + systemctl:start) printf 'start\\n' >>"$SERVICE_LOG"; printf 'active\\n' >"$TUNNEL_STATE" ;; + systemctl:show) cat "$TUNNEL_STATE" ;; + /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy:--commit) printf 'deploy\\n' >>"$SERVICE_LOG" ;; + *) exit 1 ;; +esac +`, + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(directory, "curl"), "#!/bin/sh\nprintf '503'\n", { mode: 0o755 }); + + const result = spawnSync("/bin/bash", ["-c", block!], { + encoding: "utf8", + env: { + ...process.env, + FAIL_CONTAINMENT_STOP: failContainmentStop ? "1" : "0", + PATH: `${directory}:${process.env.PATH ?? ""}`, + SERVICE_LOG: log, + STOP_COUNT: stopCount, + TUNNEL_STATE: state, + }, + }); + + expect(result.status).not.toBe(0); + expect(fs.readFileSync(log, "utf8").trim().split("\n")).toEqual(expectedLog); + expect(fs.readFileSync(state, "utf8")).toBe(expectedState); + expect( + result.stderr + .split("\n") + .filter((line) => line.startsWith("PUBLIC INGRESS CONTAINMENT FAILED:")), + ).toEqual(expectedContainmentDiagnostics); + }); + + it("loads fixed SSH, host-key, timeout, and cleanup configuration (#8142)", () => { + const files = deploymentFiles(); + expect(loadConfig(files)).toEqual({ + cleanupExecutable: files.cleanupExecutable, + destination: "nvidia@192.168.55.1", + identityFile: files.identityFile, + knownHostsFile: files.knownHostsFile, + stateDirectory: files.stateDirectory, + testTimeoutSeconds: 2700, + }); + }); + + it("collects a bounded artifact archive before the remote workspace is removed (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); + fs.writeFileSync(baselinePath, `${JSON.stringify({ ...BASELINE, nodeVersion: "v22.0.0" })}\n`, { + mode: 0o600, + }); + const artifactArchiveBase64 = Buffer.from("remote archive").toString("base64"); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) + .mockResolvedValueOnce({ + stdout: + "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", + stderr: "", + }) + .mockResolvedValueOnce({ stdout: "test passed\n", stderr: "" }) + .mockResolvedValueOnce({ stdout: artifactArchiveBase64, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect( + worker.run( + { + schemaVersion: 1, + target: "jetson-nvmap-gpu", + candidateSha: "a".repeat(40), + workflowRunId: "123", + workflowRunAttempt: 1, + }, + { jobId, signal: new AbortController().signal }, + ), + ).resolves.toMatchObject({ + artifactArchiveBase64, + log: expect.stringContaining("test passed"), + }); + expect(processRunner.mock.calls[0]![0].input).toContain( + "major < 22 || (major === 22 && minor < 19)", + ); + expect(processRunner.mock.calls[0]![0].input).toContain("npm_major >= 10"); + expect(processRunner.mock.calls[2]![0].input).toContain( + "major < 22 || (major === 22 && minor < 19)", + ); + expect(processRunner.mock.calls[2]![0].input).toContain("npm_major >= 10"); + expect(processRunner.mock.calls[2]![0].input).toContain('"$workspace/runtime"'); + expect(processRunner.mock.calls[2]![0].input).toContain( + 'export XDG_RUNTIME_DIR="$workspace/runtime"', + ); + expect(processRunner.mock.calls[2]![0].input).toContain("unset DBUS_SESSION_BUS_ADDRESS"); + expect(JSON.parse(fs.readFileSync(baselinePath, "utf8"))).toEqual(BASELINE); + expect(processRunner.mock.calls[2]![0].input).not.toContain("trap cleanup EXIT"); + expect(processRunner.mock.calls[3]![0].input).not.toContain("trap cleanup EXIT"); + expect(processRunner.mock.calls[3]![0].input).toContain("base64 --wrap=0"); + }); + + it("preserves test-failure logs when artifact collection fails (#8142)", async () => { + const files = deploymentFiles(); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) + .mockResolvedValueOnce({ + stdout: + "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", + stderr: "", + }) + .mockRejectedValueOnce( + new ProcessFailure("Jetson test failed", { + stdout: "failed test output\n", + stderr: "failed test error\n", + }), + ) + .mockRejectedValueOnce(new Error("artifact collection unavailable")); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect( + worker.run( + { + schemaVersion: 1, + target: "jetson-nvmap-gpu", + candidateSha: "a".repeat(40), + workflowRunId: "123", + workflowRunAttempt: 1, + }, + { jobId: "b".repeat(64), signal: new AbortController().signal }, + ), + ).rejects.toMatchObject({ + message: "Jetson test failed; artifact collection failed: artifact collection unavailable", + log: expect.stringContaining("failed test output"), + }); + }); + + it("preserves successful test logs when artifact collection fails (#8142)", async () => { + const files = deploymentFiles(); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) + .mockResolvedValueOnce({ + stdout: + "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", + stderr: "", + }) + .mockResolvedValueOnce({ stdout: "successful test output\n", stderr: "" }) + .mockRejectedValueOnce(new Error("artifact collection unavailable")); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect( + worker.run( + { + schemaVersion: 1, + target: "jetson-nvmap-gpu", + candidateSha: "a".repeat(40), + workflowRunId: "123", + workflowRunAttempt: 1, + }, + { jobId: "b".repeat(64), signal: new AbortController().signal }, + ), + ).rejects.toMatchObject({ + message: "artifact collection unavailable", + log: expect.stringContaining("successful test output"), + }); + }); + + it.each([ + "nvidia@192.168.55.2", + "nvidia@192.168.55.1 -o StrictHostKeyChecking=no", + "nvidia@host;uname", + "root@", + "-oProxyCommand=unsafe", + ])("rejects an SSH destination that is not the fixed Jetson user and host: %s (#8142)", (destination) => { + const files = deploymentFiles(); + expect(() => + loadSshJetsonWorkerConfig( + { stateDirectory: files.stateDirectory }, + { + ...environment(files), + JETSON_DISPATCH_SSH_DESTINATION: destination, + }, + ), + ).toThrow("JETSON_DISPATCH_SSH_DESTINATION must be nvidia@192.168.55.1"); + }); + + it("rejects a writable SSH identity file (#8142)", () => { + const files = deploymentFiles(); + fs.chmodSync(files.identityFile, 0o666); + expect(() => loadConfig(files)).toThrow("must not be group- or world-writable"); + }); + + it("rejects a group-readable SSH identity file (#8142)", () => { + const files = deploymentFiles(); + fs.chmodSync(files.identityFile, 0o640); + expect(() => loadConfig(files)).toThrow("must be readable only by its owner"); + }); + + it("rejects a symbolic-link cleanup executable (#8142)", () => { + const files = deploymentFiles(); + const link = path.join(path.dirname(files.cleanupExecutable), "cleanup-link"); + fs.symlinkSync(files.cleanupExecutable, link); + expect(() => + loadSshJetsonWorkerConfig( + { stateDirectory: files.stateDirectory }, + { + ...environment(files), + JETSON_DISPATCH_CLEANUP_EXECUTABLE: link, + }, + ), + ).toThrow("must be a regular file"); + }); + + it("accepts only the exact root-owned current-release cleanup link (#8142)", () => { + const files = deploymentFiles(); + const cleanupExecutable = "/usr/local/libexec/nemoclaw-jetson-cleanup"; + const cleanupTarget = + "/opt/nemoclaw-jetson-dispatch/current/tools/e2e/jetson-dispatch-cleanup.sh"; + const localLink = path.join(files.stateDirectory, "managed-cleanup-link"); + fs.symlinkSync(files.cleanupExecutable, localLink); + const realLstatSync = fs.lstatSync.bind(fs); + const realReadlinkSync = fs.readlinkSync.bind(fs); + const localLinkStat = realLstatSync(localLink); + const rootLinkStat = Object.assign( + Object.create(Object.getPrototypeOf(localLinkStat)), + localLinkStat, + { uid: 0 }, + ) as fs.Stats; + const nonRootLinkStat = Object.assign( + Object.create(Object.getPrototypeOf(localLinkStat)), + localLinkStat, + { uid: 1000 }, + ) as fs.Stats; + const secureTargetStat = realLstatSync(files.cleanupExecutable); + const unsafeTargetStat = Object.assign( + Object.create(Object.getPrototypeOf(secureTargetStat)), + secureTargetStat, + { mode: secureTargetStat.mode | 0o022 }, + ) as fs.Stats; + let selectedTarget = "/tmp/unmanaged-cleanup"; + let managedLinkStat = rootLinkStat; + let managedTargetStat = secureTargetStat; + const lstatOverrides = new Map([ + [cleanupExecutable, () => managedLinkStat], + [cleanupTarget, () => managedTargetStat], + ]); + const lstat = vi.spyOn(fs, "lstatSync").mockImplementation((candidate, options) => { + return ( + lstatOverrides.get(String(candidate))?.() ?? realLstatSync(candidate, options as never) + ); + }); + const readlink = vi + .spyOn(fs, "readlinkSync") + .mockImplementation(((candidate, options) => + String(candidate) === cleanupExecutable + ? selectedTarget + : realReadlinkSync(candidate, options as never)) as typeof fs.readlinkSync); + + try { + const env = { + ...environment(files), + JETSON_DISPATCH_CLEANUP_EXECUTABLE: cleanupExecutable, + }; + expect(() => + loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), + ).toThrow("must select the managed current-release cleanup program"); + + selectedTarget = cleanupTarget; + managedLinkStat = nonRootLinkStat; + expect(() => + loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), + ).toThrow("symbolic link must be owned by root"); + + managedLinkStat = rootLinkStat; + managedTargetStat = unsafeTargetStat; + expect(() => + loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), + ).toThrow("must not be group- or world-writable"); + + managedTargetStat = secureTargetStat; + expect( + loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), + ).toMatchObject({ cleanupExecutable }); + } finally { + readlink.mockRestore(); + lstat.mockRestore(); + } + }); + + it("invokes the fixed cleanup executable without request-controlled arguments (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); + fs.writeFileSync(baselinePath, `${JSON.stringify(BASELINE)}\n`, { mode: 0o600 }); + persistCleanupEvidence(files, jobId, { + schemaVersion: 1, + volumes: ["sandbox-volume"], + processIds: [1234], + }); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ + stdout: cleanupOutput({ volumes: ["sandbox-volume"], processIds: [1234] }), + stderr: "", + }) + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect( + worker.cleanup({ jobId, signal: new AbortController().signal }), + ).resolves.toBeUndefined(); + + expect(processRunner.mock.calls[0]![0]).toMatchObject({ + args: [], + executable: files.cleanupExecutable, + }); + expect(processRunner.mock.calls[0]![0].input).toBeUndefined(); + expect(processRunner.mock.calls[1]![0]).toMatchObject({ + executable: "ssh", + input: expect.stringContaining('workspace="/var/tmp/nemoclaw-jetson-e2e/$job_id"'), + }); + expect(processRunner.mock.calls[1]![0].input).toContain("docker info --format"); + expect(processRunner.mock.calls[1]![0].input).toContain("test -c /dev/nvmap"); + expect(processRunner.mock.calls[1]![0].input).toContain("ollama list"); + expect(processRunner.mock.calls[1]![0].input).toContain( + 'for volume in "${recorded_volumes[@]}"', + ); + expect(processRunner.mock.calls[1]![0].input).toContain( + 'for pid in "${recorded_process_ids[@]}"', + ); + expect(processRunner.mock.calls[1]![0].input).toContain("docker container ls --all --no-trunc"); + expect(processRunner.mock.calls[1]![0].input).toContain( + "docker volume ls --format '{{.Name}}'", + ); + expect(processRunner.mock.calls[1]![0].input).not.toContain("docker container inspect"); + expect(processRunner.mock.calls[1]![0].input).not.toContain("docker volume inspect"); + expect(processRunner.mock.calls[1]![0].args).toEqual( + expect.arrayContaining(["nvidia@192.168.55.1", "--", jobId]), + ); + const encodedEvidence = processRunner.mock.calls[1]![0].args.at(-1); + expect(JSON.parse(Buffer.from(encodedEvidence!, "base64").toString("utf8"))).toEqual({ + schemaVersion: 1, + volumes: ["sandbox-volume"], + processIds: [1234], + }); + expect(fs.existsSync(baselinePath)).toBe(true); + expect( + JSON.parse(fs.readFileSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`), "utf8")), + ).toEqual({ schemaVersion: 1, volumes: ["sandbox-volume"], processIds: [1234] }); + expect(fs.statSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`)).mode & 0o777).toBe( + 0o600, + ); + }); + + it("verifies cleanup after a pre-candidate failure without a baseline record (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect( + worker.cleanup({ jobId, signal: new AbortController().signal }), + ).resolves.toBeUndefined(); + + expect(processRunner).toHaveBeenCalledTimes(2); + expect(fs.existsSync(path.join(files.stateDirectory, `${jobId}.baseline.json`))).toBe(false); + }); + + it.each([ + ["Node.js", { nodeVersion: "v22.20.0" }], + ["npm", { npmVersion: "11.0.0" }], + ["Ollama executable", { ollamaPath: "/tmp/ollama" }], + ])("rejects cleanup when the protected %s baseline changes (#8142)", async (_name, change) => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); + fs.writeFileSync(baselinePath, `${JSON.stringify(BASELINE)}\n`, { mode: 0o600 }); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: baselineOutput(change), stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + "Jetson protected tool baseline differs after cleanup", + ); + expect(fs.existsSync(baselinePath)).toBe(true); + }); + + it.each([ + { + name: "an initially empty model set followed by the onboarding model", + initial: [], + final: [{ name: "qwen3.5:9b", id: "b".repeat(12) }], + }, + { + name: "a pre-existing model plus a newly installed model", + initial: [{ name: "existing:latest", id: "a".repeat(12) }], + final: [ + { name: "existing:latest", id: "a".repeat(12) }, + { name: "qwen3.5:9b", id: "b".repeat(12) }, + ], + }, + ])("allows $name after cleanup (#8142)", async ({ initial, final }) => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + fs.writeFileSync( + path.join(files.stateDirectory, `${jobId}.baseline.json`), + `${JSON.stringify({ ...BASELINE, ollamaModels: initial })}\n`, + { mode: 0o600 }, + ); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: final }), stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect( + worker.cleanup({ jobId, signal: new AbortController().signal }), + ).resolves.toBeUndefined(); + }); + + it("keeps a maximum accepted Ollama baseline readable after persistence (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const models = Array.from({ length: 64 }, (_value, index) => ({ + name: `model-${index.toString().padStart(2, "0")}-${"x".repeat(22)}`, + id: index.toString(16).padStart(12, "0"), + })); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: models }), stderr: "" }) + .mockResolvedValueOnce({ + stdout: + "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", + stderr: "", + }) + .mockResolvedValueOnce({ stdout: "test passed\n", stderr: "" }) + .mockResolvedValueOnce({ stdout: Buffer.from("archive").toString("base64"), stderr: "" }) + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: models }), stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await worker.run( + { + schemaVersion: 1, + target: "jetson-nvmap-gpu", + candidateSha: "a".repeat(40), + workflowRunId: "123", + workflowRunAttempt: 1, + }, + { jobId, signal: new AbortController().signal }, + ); + const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); + expect(fs.statSync(baselinePath).size).toBeGreaterThan(4 * 1024); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + + await expect( + worker.cleanup({ jobId, signal: new AbortController().signal }), + ).resolves.toBeUndefined(); + }); + + it.each([ + { + name: "a pre-existing model is removed", + final: [], + }, + { + name: "a pre-existing model ID changes", + final: [{ name: "existing:latest", id: "c".repeat(12) }], + }, + ])("fails cleanup when $name (#8142)", async ({ final }) => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); + fs.writeFileSync(baselinePath, `${JSON.stringify(BASELINE)}\n`, { mode: 0o600 }); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: final }), stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + "Jetson cleanup did not preserve every pre-existing Ollama model", + ); + expect(fs.existsSync(baselinePath)).toBe(true); + }); + + it.each([ + { + name: "malformed model data", + encoded: Buffer.from("qwen3.5:9b\tnot-an-id").toString("base64"), + }, + { + name: "duplicate model names", + encoded: Buffer.from(`qwen3.5:9b\t${"a".repeat(12)}\nqwen3.5:9b\t${"b".repeat(12)}`).toString( + "base64", + ), + }, + { + name: "oversized model data", + encoded: Buffer.from( + Array.from( + { length: 65 }, + (_value, index) => `model-${index}:latest\t${index.toString(16).padStart(12, "0")}`, + ).join("\n"), + ).toString("base64"), + }, + ])("rejects $name in the protected baseline (#8142)", async ({ encoded }) => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const malformedOutput = baselineOutput().replace( + /^ollamaModelsBase64\t.*$/mu, + `ollamaModelsBase64\t${encoded}`, + ); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: malformedOutput, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + /Jetson protected-baseline Ollama model data/u, + ); + }); + + it("rejects cleanup unless OpenShell returns to the absent baseline (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + fs.writeFileSync( + path.join(files.stateDirectory, `${jobId}.baseline.json`), + `${JSON.stringify(BASELINE)}\n`, + { mode: 0o600 }, + ); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const processRunner = vi + .fn() + .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) + .mockResolvedValueOnce({ stdout: baselineOutput({ openshellState: "present" }), stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + "Jetson protected-baseline OpenShell state is malformed", + ); + }); + + it("retains every validated cleanup identity across stale-lock recovery (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + fs.writeFileSync( + path.join(files.stateDirectory, `${jobId}.baseline.json`), + `${JSON.stringify(BASELINE)}\n`, + { mode: 0o600 }, + ); + const processRunner = vi + .fn() + .mockImplementationOnce(async () => { + persistCleanupEvidence(files, jobId, { + schemaVersion: 1, + volumes: ["sandbox-volume"], + processIds: [1234], + }); + return { + stdout: cleanupOutput({ volumes: ["sandbox-volume"], processIds: [1234] }), + stderr: "", + }; + }) + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) + .mockImplementationOnce(async () => { + persistCleanupEvidence(files, jobId, { + schemaVersion: 1, + volumes: ["gateway-volume", "sandbox-volume"], + processIds: [1234, 5678], + }); + return { + stdout: cleanupOutput({ + volumes: ["gateway-volume", "sandbox-volume"], + processIds: [1234, 5678], + }), + stderr: "", + }; + }) + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await worker.cleanup({ jobId, signal: new AbortController().signal }); + await worker.cleanup({ jobId, signal: new AbortController().signal }); + + expect( + JSON.parse(fs.readFileSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`), "utf8")), + ).toEqual({ + schemaVersion: 1, + volumes: ["gateway-volume", "sandbox-volume"], + processIds: [1234, 5678], + }); + }); + + it("retains a failed volume identity for startup cleanup recovery (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const cleanupEvidencePath = path.join(files.stateDirectory, `${jobId}.cleanup.json`); + fs.writeFileSync( + path.join(files.stateDirectory, `${jobId}.baseline.json`), + `${JSON.stringify(BASELINE)}\n`, + { mode: 0o600 }, + ); + const processRunner = vi + .fn() + .mockImplementationOnce(async () => { + persistCleanupEvidence(files, jobId, { + schemaVersion: 1, + volumes: ["sandbox-volume"], + processIds: [], + }); + throw new ProcessFailure("cleanup interrupted after container removal", { + stdout: "container removed\n", + stderr: "service interrupted\n", + }); + }) + .mockResolvedValueOnce({ + stdout: cleanupOutput({ volumes: ["sandbox-volume"] }), + stderr: "", + }) + .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + "cleanup interrupted after container removal", + ); + expect(JSON.parse(fs.readFileSync(cleanupEvidencePath, "utf8"))).toEqual({ + schemaVersion: 1, + volumes: ["sandbox-volume"], + processIds: [], + }); + + const restartedWorker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + await expect( + restartedWorker.cleanup({ jobId, signal: new AbortController().signal }), + ).resolves.toBeUndefined(); + const encodedEvidence = processRunner.mock.calls[2]![0].args.at(-1); + expect(JSON.parse(Buffer.from(encodedEvidence!, "base64").toString("utf8"))).toEqual({ + schemaVersion: 1, + volumes: ["sandbox-volume"], + processIds: [], + }); + }); + + it.each([ + "", + "nemoclaw-cleanup-evidence-v1-begin\nvolume\t../unsafe\nnemoclaw-cleanup-evidence-v1-end\n", + "nemoclaw-cleanup-evidence-v1-begin\nprocessId\t0\nnemoclaw-cleanup-evidence-v1-end\n", + ])("rejects missing or unsafe cleanup identity evidence (#8142)", async (stdout) => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + const processRunner = vi.fn().mockResolvedValueOnce({ stdout, stderr: "" }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + /Jetson cleanup/u, + ); + expect(processRunner).toHaveBeenCalledTimes(1); + expect(fs.existsSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`))).toBe(false); + }); + + it("rejects cleanup output that differs from the durable identity record (#8142)", async () => { + const files = deploymentFiles(); + const jobId = "b".repeat(64); + persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); + const processRunner = vi.fn().mockResolvedValueOnce({ + stdout: cleanupOutput({ volumes: ["unpersisted-volume"] }), + stderr: "", + }); + const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); + + await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( + "Jetson cleanup output differs from its durable resource evidence", + ); + expect(processRunner).toHaveBeenCalledTimes(1); + }); + + it("reports a missing durable cleanup record after valid helper output (#8142)", async () => { + const files = deploymentFiles(); + const worker = new SshJetsonDispatchWorker( + loadConfig(files), + vi.fn().mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }), + ); + + await expect( + worker.cleanup({ jobId: "b".repeat(64), signal: new AbortController().signal }), + ).rejects.toThrow("Jetson cleanup evidence is missing"); + }); + + it("ignores permission denial while inspecting an unrelated process (#8142)", () => { + const fixture = cleanupProcessFixture({ + deny: "environ", + uid: process.getuid!() + 1, + }); + const result = spawnSync( + "/bin/bash", + [ + "-c", + `set -euo pipefail +${fixture.functions} +job_home=/job/home +for proc_dir in "$PROC_ROOT"/*; do + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" owner + continue + fi + [ "$process_uid" = "$(id -u)" ] || continue + if ! environment="$(read_proc_environment "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" environment + continue + fi + printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue + if ! cmdline="$(read_proc_command "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" command + continue + fi +done +printf 'inspection-complete\n' +`, + ], + { encoding: "utf8", env: fixture.environment }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("inspection-complete\n"); + }); + + it("rejects an unreadable live same-user process during discovery (#8142)", () => { + const fixture = cleanupProcessFixture({ deny: "environ" }); + const result = spawnSync( + "/bin/bash", + [ + "-c", + `set -euo pipefail +${fixture.functions} +job_home=/job/home +for proc_dir in "$PROC_ROOT"/*; do + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" owner + continue + fi + [ "$process_uid" = "$(id -u)" ] || continue + if ! environment="$(read_proc_environment "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" environment + continue + fi +done +`, + ], + { encoding: "utf8", env: fixture.environment }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Unable to inspect environment for a live same-user process"); + }); + + it("ignores a recorded process that disappears during inspection (#8142)", () => { + const fixture = cleanupProcessFixture({ disappear: "cmdline" }); + const result = spawnSync( + "/bin/bash", + [ + "-c", + `set -euo pipefail +${fixture.functions} +kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } +job_home=/job/home +validate_recorded_pid 1234 ollama-auth-proxy. +`, + ], + { encoding: "utf8", env: fixture.environment }, + ); + + expect(result.status, result.stderr).toBe(0); + }); + + it("completes the destructive cleanup phase when a process disappears during inspection (#8142)", () => { + const fixture = cleanupProcessFixture({ disappear: "environ" }); + fs.writeFileSync(path.join(fixture.binDirectory, "docker"), "#!/bin/sh\nexit 0\n", { + mode: 0o700, + }); + const marker = "<<'JETSON_CLEANUP'\n"; + const phaseStart = cleanupProgram.indexOf(marker); + const phaseEnd = cleanupProgram.indexOf("\nJETSON_CLEANUP", phaseStart + marker.length); + assert.ok(phaseStart >= 0 && phaseEnd >= 0, "destructive cleanup phase is absent"); + const script = cleanupProgram + .slice(phaseStart + marker.length, phaseEnd) + .replace("set -euo pipefail", "set -eo pipefail") + .replace( + "workspace_root=/var/tmp/nemoclaw-jetson-e2e", + 'workspace_root="$TEST_WORKSPACE_ROOT"', + ) + .replace( + 'service_directory="/tmp/nemoclaw-services-$sandbox_name"', + 'service_directory="$TEST_SERVICE_DIRECTORY"', + ) + .replace("for proc_dir in /proc/[0-9]*; do", 'for proc_dir in "$PROC_ROOT"/*; do') + .replace( + 'remaining_job_images="$(list_test_owned_images)"', + "printf 'destructive-phase-complete\\n'\nexit 0\nremaining_job_images=\"$(list_test_owned_images)\"", + ); + const evidence = Buffer.from( + JSON.stringify({ schemaVersion: 1, volumes: [], processIds: [] }), + ).toString("base64"); + const result = spawnSync("/bin/bash", ["-s", "--", "b".repeat(64), evidence], { + encoding: "utf8", + env: { + ...fixture.environment, + TEST_SERVICE_DIRECTORY: path.join(fixture.procRoot, "service"), + TEST_WORKSPACE_ROOT: path.join(fixture.procRoot, "workspace"), + }, + input: script, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("destructive-phase-complete\n"); + }); + + it("rejects a live recorded PID whose owner cannot be verified (#8142)", () => { + const fixture = cleanupProcessFixture(); + fs.rmSync(path.join(fixture.procRoot, "1234", "status")); + const result = spawnSync( + "/bin/bash", + [ + "-c", + `set -euo pipefail +${fixture.functions} +kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } +job_home=/job/home +validate_recorded_pid 1234 ollama-auth-proxy. +`, + ], + { encoding: "utf8", env: fixture.environment }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("owner cannot be verified"); + }); + + it.each([ + { + name: "owner", + options: { uid: process.getuid!() + 1 }, + error: "owned by another user", + }, + { + name: "HOME", + options: { home: "/unrelated/home" }, + error: "outside the job home", + }, + { + name: "command", + options: { command: "node unrelated-service.mjs" }, + error: "unexpected command", + }, + ])("rejects a recorded PID with an unexpected $name (#8142)", ({ error, options }) => { + const fixture = cleanupProcessFixture(options); + const result = spawnSync( + "/bin/bash", + [ + "-c", + `set -euo pipefail +${fixture.functions} +kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } +job_home=/job/home +validate_recorded_pid 1234 ollama-auth-proxy. +`, + ], + { encoding: "utf8", env: fixture.environment }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(error); + }); + + it("accepts a recorded PID with the expected owner, HOME, and command (#8142)", () => { + const fixture = cleanupProcessFixture(); + const result = spawnSync( + "/bin/bash", + [ + "-c", + `set -euo pipefail +${fixture.functions} +kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } +job_home=/job/home +validate_recorded_pid 1234 ollama-auth-proxy. +`, + ], + { encoding: "utf8", env: fixture.environment }, + ); + + expect(result.status, result.stderr).toBe(0); + }); + + it("keeps cleanup targets fixed and preserves the Jetson baseline (#8142)", () => { + const fileFsync = cleanupProgram.indexOf("fs.fsyncSync(descriptor)"); + const atomicRename = cleanupProgram.indexOf("fs.renameSync(temporaryFile, cleanupFile)"); + const directoryFsync = cleanupProgram.indexOf("fs.fsyncSync(directoryDescriptor)"); + const destructivePhase = cleanupProgram.indexOf("<<'JETSON_CLEANUP'"); + const stopFunctionStart = cleanupProgram.lastIndexOf("stop_recorded_pid()"); + const stopFunction = cleanupProgram.slice( + stopFunctionStart, + cleanupProgram.indexOf('for pid in "${recorded_process_ids[@]}"', stopFunctionStart), + ); + expect(cleanupProgram).toContain("sandbox_name=e2e-jetson-nvmap"); + expect(cleanupProgram).toContain("gateway_container=openshell-cluster-nemoclaw"); + expect(cleanupProgram).toContain("destination=nvidia@192.168.55.1"); + expect(cleanupProgram).toContain('if [ "$#" -ne 0 ]'); + expect(cleanupProgram).toContain('mapfile -t lock_lines <"$lock_file"'); + expect(cleanupProgram).toContain('[ "${#lock_lines[@]}" -ne 1 ]'); + expect(cleanupProgram).toContain('require_plain_directory_if_present "$service_directory"'); + expect(cleanupProgram).toContain('rm -rf -- "$workspace"'); + expect(cleanupProgram).toContain("ollama-auth-proxy.pid"); + expect(cleanupProgram).toContain("read_proc_uid()"); + expect(cleanupProgram).toContain('dd if="$1/environ"'); + expect(cleanupProgram).toContain('dd if="$1/cmdline"'); + expect( + cleanupProgram.match(/handle_proc_read_failure "\$proc_dir" environment/gu), + ).toHaveLength(2); + expect(cleanupProgram.match(/handle_proc_read_failure "\$proc_dir" command/gu)).toHaveLength(2); + expect(workerProgram).toContain('handle_proc_read_failure "$proc_dir" environment'); + expect(workerProgram).toContain('handle_proc_read_failure "$proc_dir" command'); + expect(cleanupProgram).toContain('grep -Fqx "HOME=$job_home"'); + expect(cleanupProgram).toContain("*openshell-forward*"); + expect(cleanupProgram).toContain("*openshell\\ forward*"); + expect(cleanupProgram).toContain('kill "$pid"'); + expect(stopFunction.indexOf('process_uid="$(read_proc_uid')).toBeLessThan( + stopFunction.indexOf('kill "$pid"'), + ); + expect(stopFunction.indexOf('cmdline="$(read_proc_command')).toBeLessThan( + stopFunction.indexOf('kill "$pid"'), + ); + expect(stopFunction.indexOf('grep -Fqx "HOME=$job_home"')).toBeLessThan( + stopFunction.indexOf('kill "$pid"'), + ); + expect(cleanupProgram).toContain("label=openshell.ai/sandbox-name=$sandbox_name"); + expect(cleanupProgram).toContain("image_repository=nemoclaw-sandbox-local"); + expect(cleanupProgram).toContain('docker image rm "$image"'); + expect(cleanupProgram.indexOf('docker image rm "$image"')).toBeLessThan( + cleanupProgram.indexOf('rm -rf -- "$workspace"'), + ); + expect(cleanupProgram).not.toContain("job_openshell"); + expect(cleanupProgram).not.toContain('"$job_home/.local/bin/openshell"'); + expect(cleanupProgram).toContain("docker container ls --all --no-trunc"); + expect(cleanupProgram).toContain("docker volume ls --format '{{.Name}}'"); + expect(cleanupProgram).not.toMatch(/if docker (?:container|volume) inspect/u); + expect(cleanupProgram).not.toContain("mapfile -t sandbox_containers < <("); + expect(cleanupProgram).toContain("A host-level OpenShell binary remains after cleanup"); + expect(cleanupProgram).toContain("nemoclaw-cleanup-evidence-v1-begin"); + expect(cleanupProgram).toContain("nemoclaw-cleanup-evidence-v1-end"); + expect(cleanupProgram.indexOf("nemoclaw-cleanup-evidence-v1-begin")).toBeLessThan( + cleanupProgram.indexOf('docker container rm --force "$container"'), + ); + expect(cleanupProgram.indexOf("nemoclaw-cleanup-evidence-v1-end")).toBeLessThan( + cleanupProgram.indexOf('docker volume rm "$volume"'), + ); + expect(cleanupProgram).toContain("fs.constants.O_EXCL"); + expect(fileFsync).toBeGreaterThan(cleanupProgram.indexOf("<<'JETSON_DISCOVERY'")); + expect(atomicRename).toBeGreaterThan(fileFsync); + expect(directoryFsync).toBeGreaterThan(atomicRename); + expect(destructivePhase).toBeGreaterThan(directoryFsync); + expect(cleanupProgram).not.toMatch(/pkill|pgrep|docker (?:system|container|volume) prune/u); + expect(cleanupProgram).not.toContain('rm -rf -- "$workspace_root"'); + expect(cleanupProgram).not.toContain("ollama serve"); + expect(cleanupProgram).not.toMatch(/ollama (?:rm|delete)/u); + expect(cleanupProgram).not.toMatch(/apt(?:-get)?|boardctl|reboot|shutdown|nvidia-l4t/u); + expect(cleanupProgram).not.toMatch(/npm uninstall|uninstall\.sh|rm .*ollama/u); + }); + + it("keeps OpenShell installation owned by NemoClaw onboarding inside the job workspace (#8142)", () => { + expect(workerProgram).not.toContain("NEMOCLAW_DEFER_OPENSHELL_INSTALL"); + expect(jetsonLiveTest).not.toContain("NEMOCLAW_DEFER_OPENSHELL_INSTALL"); + expect(workerProgram).not.toContain("scripts/install-openshell.sh"); + expect(cleanupProgram).not.toContain("scripts/install-openshell.sh"); + expect(jetsonLiveTest).not.toContain("scripts/install-openshell.sh"); + expect(jetsonLiveTest).toContain('host.command("bash", ["install.sh", "--non-interactive"]'); + expect(workerProgram).toContain('export XDG_BIN_HOME="$workspace/home/.local/bin"'); + expect(workerProgram).toContain('export PATH="$XDG_BIN_HOME:$workspace/npm-prefix/bin:$PATH"'); + expect(workerProgram).toContain( + "for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox", + ); + expect(jetsonLiveTest).toContain( + "for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox", + ); + expect(cleanupProgram.indexOf('stop_recorded_pid "$pid"')).toBeLessThan( + cleanupProgram.indexOf('docker container rm --force "$container"'), + ); + expect(cleanupProgram.indexOf('docker container rm --force "$container"')).toBeLessThan( + cleanupProgram.indexOf('rm -rf -- "$workspace"'), + ); + }); + + it("requires every protected Jetson prerequisite before dispatch (#8142)", () => { + expect(dispatcherRunbook).toContain("OpenShell must be absent"); + expect(dispatcherRunbook).toContain( + "if (major < 22 || (major === 22 && minor < 19)) process.exit(1)", + ); + expect(dispatcherRunbook).toContain('test "${npm_version%%.*}" -ge 10'); + expect(dispatcherRunbook).toContain("ollama list"); + }); + + it("uses a private temporary file for the pinned Jetson host key (#8142)", () => { + const hostKeySection = dispatcherRunbook.slice( + dispatcherRunbook.indexOf("Capture the Jetson host key"), + dispatcherRunbook.indexOf("## Define the Cleanup Program"), + ); + const temporary = hostKeySection.indexOf( + 'known_hosts_tmp="$(mktemp /tmp/nemoclaw-jetson-known-hosts.XXXXXX)"', + ); + const cleanupTrap = hostKeySection.indexOf("trap 'rm -f -- \"$known_hosts_tmp\"' EXIT"); + const keyScan = hostKeySection.indexOf("ssh-keyscan -T 10"); + const install = hostKeySection.indexOf("sudo install"); + + expect(hostKeySection).toContain("umask 077"); + expect(temporary).toBeGreaterThan(-1); + expect(cleanupTrap).toBeGreaterThan(temporary); + expect(keyScan).toBeGreaterThan(cleanupTrap); + expect(install).toBeGreaterThan(keyScan); + expect(hostKeySection).not.toContain("/tmp/jetson_known_hosts"); + }); + + it("bounds every Jetson teardown SSH probe and the Ollama model query (#8142)", () => { + const teardown = dispatcherRunbook.slice( + dispatcherRunbook.indexOf("## Recover or Disable the Deployment"), + ); + const sshProbes = teardown.match(/sudo -u nemoclaw-jetson-dispatch ssh/gmu) ?? []; + const boundedSshProbes = + teardown.match( + /timeout --kill-after=5 120 \\\n\s+sudo -u nemoclaw-jetson-dispatch ssh -F \/dev\/null -T \\\n\s+-o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \\\n\s+-o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3/gmu, + ) ?? []; + + expect(sshProbes).toHaveLength(4); + expect(boundedSshProbes).toHaveLength(sshProbes.length); + expect(teardown).toContain("timeout --kill-after=5 30 ollama list >/dev/null"); + }); + + it("stops credential teardown when the Jetson key-removal probe fails (#8142)", () => { + const credentialTeardown = dispatcherRunbook.slice( + dispatcherRunbook.indexOf("Remove the dedicated Jetson public key"), + dispatcherRunbook.indexOf("Remove the cleanup executable and pinned SSH host-key file"), + ); + const strictMode = credentialTeardown.indexOf("set -euo pipefail"); + const keyRemovalProbe = credentialTeardown.indexOf("timeout --kill-after=5 120"); + const privateKeyRemoval = credentialTeardown.indexOf( + "sudo rm -- \\\n /var/lib/nemoclaw-jetson-dispatch/id_ed25519", + ); + + expect(strictMode).toBeGreaterThan(-1); + expect(keyRemovalProbe).toBeGreaterThan(strictMode); + expect(privateKeyRemoval).toBeGreaterThan(keyRemovalProbe); + expect(credentialTeardown.slice(keyRemovalProbe, privateKeyRemoval)).not.toContain("|| true"); + expect(dispatcherRunbook).toContain( + "Use the serial console to confirm that the exact key line is absent", + ); + }); + + it("keeps recovery credentials until idle teardown verification succeeds (#8142)", () => { + const stopTunnel = dispatcherRunbook.indexOf( + "sudo systemctl disable --now nemoclaw-jetson-tunnel.service", + ); + const waitForLock = dispatcherRunbook.indexOf( + "timeout 3600 bash -c 'while [ -e \"$1\" ]; do sleep 5; done'", + ); + const verifyCleanup = dispatcherRunbook.indexOf( + 'if (status.state !== "completed" || status.cleanup !== "succeeded") process.exit(1)', + ); + const aggregateCleanupRecords = dispatcherRunbook.indexOf( + "const cleanupName = /^[a-f0-9]{64}\\.cleanup\\.json$/", + ); + const verifyRecordedVolume = dispatcherRunbook.indexOf("docker volume ls --format '{{.Name}}'"); + const verifyRecordedProcess = dispatcherRunbook.indexOf('if [ -e "/proc/$1" ]'); + const stopDispatcher = dispatcherRunbook.indexOf( + "sudo systemctl disable --now nemoclaw-jetson-dispatch.service", + ); + const removeSshKey = dispatcherRunbook.indexOf( + "Remove the dedicated Jetson public key and the Colossus SSH private key.", + ); + + expect(dispatcherServiceUnit).toContain("TimeoutStopSec=360"); + expect(stopTunnel).toBeGreaterThan(-1); + expect(waitForLock).toBeGreaterThan(stopTunnel); + expect(verifyCleanup).toBeGreaterThan(waitForLock); + expect(dispatcherRunbook).toContain( + 'const expectedKeys = ["processIds", "schemaVersion", "volumes"]', + ); + expect(dispatcherRunbook).toContain("record.schemaVersion !== 1"); + expect(aggregateCleanupRecords).toBeGreaterThan(verifyCleanup); + expect(verifyRecordedVolume).toBeGreaterThan(aggregateCleanupRecords); + expect(verifyRecordedProcess).toBeGreaterThan(verifyRecordedVolume); + expect(stopDispatcher).toBeGreaterThan(verifyRecordedProcess); + expect(removeSshKey).toBeGreaterThan(stopDispatcher); + }); +}); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index b68ccc73986..2e3a8a9476c 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import { validateE2eWorkflowBoundary, - validateJetsonRunnerDispatchBoundary, + validateJetsonDispatchBoundary, } from "../../../tools/e2e/workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract.ts"; @@ -13,65 +13,126 @@ function validateWorkflowMutation( ): string[] { const workflow = readWorkflow(); mutate(workflow); - return validateJetsonRunnerDispatchBoundary(workflow); + return validateJetsonDispatchBoundary(workflow); } describe("Jetson nvmap GPU E2E workflow boundary", () => { - it("rejects a permissive Jetson runner opt-in", () => { + it("rejects a permissive Colossus dispatch opt-in (#8142)", () => { const inputErrors = validateWorkflowMutation((workflow) => { const triggers = (workflow.on ?? workflow[true as unknown as string]) as { workflow_dispatch?: { inputs?: Record; }; }; - const input = triggers.workflow_dispatch!.inputs!.allow_jetson_runner_queue; + const input = triggers.workflow_dispatch!.inputs!.allow_jetson_dispatch; input.type = "string"; input.default = true; - input.description = "Queue the runner"; + input.description = "Dispatch the Jetson"; }); expect(inputErrors).toEqual( expect.arrayContaining([ - "workflow_dispatch allow_jetson_runner_queue input must be boolean", - "workflow_dispatch allow_jetson_runner_queue input must default to false", - "workflow_dispatch allow_jetson_runner_queue input must require repository administrator confirmation from the authoritative NVIDIA/NemoClaw Settings -> Actions -> Runners inventory and document queued timeout behavior", + "workflow_dispatch allow_jetson_dispatch input must be boolean", + "workflow_dispatch allow_jetson_dispatch input must default to false", + "workflow_dispatch allow_jetson_dispatch input must require the Colossus dispatcher, tunnel, cleanup helper, repository URL variable, and deployment checks", ]), ); + }); - const guardErrors = validateWorkflowMutation((workflow) => { + it("keeps opt-in and trusted selectors before controller assignment (#8142)", () => { + const errors = validateWorkflowMutation((workflow) => { const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { - "runs-on"?: string; if?: string; - steps?: Array<{ name?: string }>; }; - job["runs-on"] = "self-hosted"; job.if = "${{ true }}"; - job.steps!.push({ name: "Guard Jetson runner dispatch" }); }); - expect(guardErrors).toEqual( + + expect(errors).toContain( + "jetson-nvmap-gpu job must require the dispatch opt-in, trusted main workflow dispatch, same-repository candidate, and target selectors", + ); + }); + + it("queues every Jetson dispatch on one fixed device without cancellation (#8142)", () => { + const workflow = readWorkflow(); + const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { + concurrency?: Record; + }; + job.concurrency = { + group: "jetson-${{ github.ref }}", + queue: 1, + "cancel-in-progress": true, + }; + expect(validateJetsonDispatchBoundary(workflow)).toContain( + "jetson-nvmap-gpu concurrency must queue every dispatch on one fixed device without cancellation", + ); + }); + + it("rejects candidate execution or credential-bearing controller steps (#8142)", () => { + const errors = validateWorkflowMutation((workflow) => { + const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { + permissions?: Record; + steps?: Array<{ name?: string; run?: string; with?: Record }>; + }; + job.permissions!.contents = "write"; + const checkout = job.steps!.find( + (step) => step.name === "Check out trusted Jetson controller", + )!; + checkout.with!.ref = "${{ inputs.checkout_sha }}"; + const dispatch = job.steps!.find( + (step) => step.name === "Dispatch exact commit to Jetson through Colossus", + )!; + dispatch.run = "bash candidate-script.sh"; + }); + + expect(errors).toEqual( expect.arrayContaining([ - "jetson-nvmap-gpu job must require allow_jetson_runner_queue before runner assignment and retain trusted-main selectors", - "jetson-nvmap-gpu job must use the configured runner only after job-level opt-in", - "jetson-nvmap-gpu must enforce opt-in before runner assignment, not in a step", + "jetson-nvmap-gpu controller must grant only contents:read and id-token:write", + "jetson-nvmap-gpu checkout must use the trusted workflow SHA without credentials", + "jetson-nvmap-gpu controller must dispatch only the exact candidate and configured URL", ]), ); }); - it("requires the Jetson flag at the job boundary", () => { + it("rejects a direct OpenShell installer step in the Jetson controller (#8142)", () => { const errors = validateWorkflowMutation((workflow) => { const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { - if?: string; + steps?: Array<{ name?: string; run?: string }>; }; - job.if = job.if?.replace("inputs.allow_jetson_runner_queue && ", ""); + job.steps!.push({ + name: "Install OpenShell directly", + run: "bash scripts/install-openshell.sh", + }); }); expect(errors).toContain( - "jetson-nvmap-gpu job must require allow_jetson_runner_queue before runner assignment and retain trusted-main selectors", + "jetson-nvmap-gpu controller must contain only checkout, Node setup, dispatch, and upload", + ); + }); + + it("keeps the runner temporary artifact path on the dispatch step (#8142)", () => { + const errors = validateWorkflowMutation((workflow) => { + const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { + env?: Record; + steps?: Array<{ env?: Record; name?: string }>; + }; + job.env = { + E2E_ARTIFACT_DIR: "${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu", + }; + const dispatch = job.steps!.find( + (step) => step.name === "Dispatch exact commit to Jetson through Colossus", + )!; + dispatch.env!.E2E_ARTIFACT_DIR = "${{ github.workspace }}/e2e-artifacts"; + }); + + expect(errors).toEqual( + expect.arrayContaining([ + "jetson-nvmap-gpu controller must not define a job-level environment", + "jetson-nvmap-gpu controller must dispatch only the exact candidate and configured URL", + ]), ); }); - it("accepts the real workflow without Jetson queue contract errors", () => { - const errors = validateE2eWorkflowBoundary(); - expect(errors.filter((error) => /jetson|allow_jetson_runner_queue/iu.test(error))).toEqual([]); - expect(errors).toEqual([]); + it("accepts the real workflow without Jetson dispatch errors (#8142)", () => { + expect(validateJetsonDispatchBoundary(readWorkflow())).toEqual([]); + expect(validateE2eWorkflowBoundary()).toEqual([]); }); }); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 9378e7b6de4..06172fbf2ef 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -243,6 +243,18 @@ describe("E2E workflow plan", () => { } }); + it.each([ + "jobs", + "targets", + ] as const)("emits an empty shared plan for the Jetson dispatch %s selector (#8142)", (selector) => { + expect(buildE2eWorkflowPlan({ [selector]: "jetson-nvmap-gpu" })).toEqual({ + matrix: [], + testMatrix: [], + hermesSelected: false, + explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, + }); + }); + it("emits an empty matrix for retired free-standing rebuild selectors (#7615)", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); const output = path.join(directory, "github-output"); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 757ba125ccc..592ab5f4d4c 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -92,6 +92,22 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)scripts\/setup-jetson\.sh$/, testsToRun: runTests("test/setup-jetson.test.ts"), }, + { + pattern: /(?:^|\/)tools\/e2e\/jetson-dispatch-cleanup\.sh$/, + testsToRun: runTests("test/e2e/support/jetson-dispatch-worker.test.ts"), + }, + { + pattern: /(?:^|\/)tools\/e2e\/colossus-jetson-dispatch-deploy\.sh$/, + testsToRun: runTests("test/e2e/support/colossus-jetson-dispatch-deploy.test.ts"), + }, + { + pattern: + /(?:^|\/)tools\/e2e\/(?:colossus-jetson-dispatch\.environment|nemoclaw-jetson-dispatch\.service)$/, + testsToRun: runTests( + "test/e2e/support/colossus-jetson-dispatch-deploy.test.ts", + "test/e2e/support/jetson-dispatch-worker.test.ts", + ), + }, { pattern: /(?:^|\/)(?:\.github\/workflows\/base-image\.yaml|scripts\/export-managed-base-image-contract\.sh)$/, diff --git a/test/maintainer-e2e-skill.test.ts b/test/maintainer-e2e-skill.test.ts index b425c2f5e80..a4ae9584015 100644 --- a/test/maintainer-e2e-skill.test.ts +++ b/test/maintainer-e2e-skill.test.ts @@ -20,6 +20,7 @@ function validEvidence() { }, dispatch: { allowDgxSparkRunnerQueue: false, + allowJetsonDispatch: false, allowJetsonRunnerQueue: false, candidateSha, emptySelectors: true, @@ -119,6 +120,7 @@ describe("nemoclaw-maintainer-e2e evidence validation", () => { }, dispatch: { allowDgxSparkRunnerQueue: false, + allowJetsonDispatch: false, allowJetsonRunnerQueue: false, emptySelectors: true, includeStagingBrevLaunchable: true, @@ -220,7 +222,14 @@ describe("nemoclaw-maintainer-e2e evidence validation", () => { "dispatch.jobs", ], [ - "a full dispatch that opts into the Jetson runner queue", + "a full dispatch that opts into Colossus Jetson execution", + (evidence: ReturnType) => { + evidence.dispatch.allowJetsonDispatch = true; + }, + "dispatch.allowJetsonDispatch", + ], + [ + "a full dispatch that enables the retired Jetson runner receipt field", (evidence: ReturnType) => { evidence.dispatch.allowJetsonRunnerQueue = true; }, diff --git a/test/openshell-e2e-qualification.test.ts b/test/openshell-e2e-qualification.test.ts index 2a19801d4c1..3d6937ee503 100644 --- a/test/openshell-e2e-qualification.test.ts +++ b/test/openshell-e2e-qualification.test.ts @@ -41,6 +41,7 @@ function prFile(filename: string, status = "modified") { function validReceipt(overrides: Record = {}) { return { allowDgxSparkRunnerQueue: false, + allowJetsonDispatch: false, allowJetsonRunnerQueue: false, baseSha: BASE_SHA, candidateRepository: REPOSITORY, diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index f5aef666837..203fe95bd93 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -45,6 +45,7 @@ function runEvidence( return { dispatch: { allowDgxSparkRunnerQueue: false, + allowJetsonDispatch: false, allowJetsonRunnerQueue: false, ...(receiptVersion === 2 ? { @@ -274,16 +275,34 @@ describe("release E2E evidence", () => { }); it.each([ - ["allowJetsonRunnerQueue", "runs[0].dispatch.allowJetsonRunnerQueue must equal false"], - ["allowDgxSparkRunnerQueue", "runs[0].dispatch.allowDgxSparkRunnerQueue must equal false"], - ])("rejects release evidence that opts into %s", (field, message) => { + [2 as const, "allowJetsonDispatch", "runs[0].dispatch.allowJetsonDispatch must equal false"], + [1 as const, "allowJetsonDispatch", "runs[0].dispatch.allowJetsonDispatch must equal false"], + [ + 1 as const, + "allowJetsonRunnerQueue", + "runs[0].dispatch.allowJetsonRunnerQueue must equal false", + ], + [ + 1 as const, + "allowDgxSparkRunnerQueue", + "runs[0].dispatch.allowDgxSparkRunnerQueue must equal false", + ], + ])("rejects v%s release evidence that opts into %s", (receiptVersion, field, message) => { const plan = preflight(); - const evidence = runEvidence(plan, "default"); + const evidence = runEvidence(plan, "default", { receiptVersion }); (evidence.dispatch as Record)[field] = true; expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow(message); }); + it("keeps allowJetsonDispatch optional for v1 release evidence", () => { + const plan = preflight(); + const evidence = runEvidence(plan, "default", { receiptVersion: 1 }); + delete (evidence.dispatch as Record).allowJetsonDispatch; + + expect(buildReleaseE2eLedger(plan, [evidence]).missingCount).toBe(0); + }); + it("reports a failed matrix row without collapsing its successful siblings", () => { const plan = preflight(); const failedId = 'hermes-gpu-startup[scenario="fallback",sandbox_name="e2e-hgpu-fallback"]'; diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index e5d6494e0b2..f6a3da0ab85 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -61,6 +61,8 @@ const OPAQUE_INPUTS = [ "agents/hermes/mcp-config-transaction.py", "test/e2e/lib/ci-compatible-inference.sh", "scripts/setup-jetson.sh", + "tools/e2e/jetson-dispatch-cleanup.sh", + "tools/e2e/colossus-jetson-dispatch-deploy.sh", ".github/workflows/base-image.yaml", "scripts/export-managed-base-image-contract.sh", "scripts/checks/validate-managed-base-index.sh", @@ -130,6 +132,12 @@ describe("Vitest opaque-input watch triggers", () => { "test/e2e/support/hosted-inference.test.ts", ]); expect(triggeredBy("scripts/setup-jetson.sh")).toEqual(["test/setup-jetson.test.ts"]); + expect(triggeredBy("tools/e2e/jetson-dispatch-cleanup.sh")).toEqual([ + "test/e2e/support/jetson-dispatch-worker.test.ts", + ]); + expect(triggeredBy("tools/e2e/colossus-jetson-dispatch-deploy.sh")).toEqual([ + "test/e2e/support/colossus-jetson-dispatch-deploy.test.ts", + ]); expect(triggeredBy(".github/workflows/base-image.yaml")).toEqual([ "test/managed-base-image-contract.test.ts", "test/managed-image-publication-workflow.test.ts", diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 95e79859184..4680dc6501a 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -244,8 +244,8 @@ export function isPrE2eTypedTargetId(value: string): boolean { } export function isPrE2ePlanningJob(value: string): boolean { - // Automatic PR planning cannot confirm an online self-hosted Jetson runner. - // Remove this exclusion after the Colossus-backed runner path can make that confirmation. + // Automatic PR planning cannot attest the external Colossus dispatcher and cleanup path. + // Remove this exclusion after that hardware gate produces trusted planning evidence. return !PR_E2E_PLANNING_OMITTED_JOB_IDS.has(value); } diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index f25830657bd..3ba5eed8ca1 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -39,7 +39,7 @@ const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 = "3578a053cede863f7aa4814d8399b4ca21ea0b77cee712e6d549c684818f11dd"; const CLI_ARTIFACT_WORKFLOW_CONTRACT_SHA256 = - "c4e838dfa11056b3b47c9769b9a8b16fc422f8ceee518600d703e4a47a717826"; + "6b9110770f2a6f5370910054d40eee6935161eb8fc5ee4ed3afec808cf536e89"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { diff --git a/tools/e2e/colossus-jetson-dispatch-deploy.sh b/tools/e2e/colossus-jetson-dispatch-deploy.sh new file mode 100755 index 00000000000..9adae79c93a --- /dev/null +++ b/tools/e2e/colossus-jetson-dispatch-deploy.sh @@ -0,0 +1,711 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH +umask 022 + +fixed_origin=https://github.com/NVIDIA/NemoClaw.git +deploy_root=/opt/nemoclaw-jetson-dispatch +releases_directory="$deploy_root/releases" +current_link="$deploy_root/current" +deploy_lock="$deploy_root/deploy.lock" +cleanup_relative=tools/e2e/jetson-dispatch-cleanup.sh +environment_relative=tools/e2e/colossus-jetson-dispatch.environment +unit_relative=tools/e2e/nemoclaw-jetson-dispatch.service +cleanup_executable=/usr/local/libexec/nemoclaw-jetson-cleanup +cleanup_link_target="$current_link/$cleanup_relative" +service_user=nemoclaw-jetson-dispatch +service_group=nemoclaw-jetson-dispatch +service_home=/var/lib/nemoclaw-jetson-dispatch +state_directory="$service_home/state" +device_lock="$state_directory/device.lock" +ssh_identity_file="$service_home/id_ed25519" +ssh_known_hosts_file="$service_home/known_hosts" +node_executable=/usr/bin/node +service_name=nemoclaw-jetson-dispatch.service +environment_file=/etc/nemoclaw-jetson-dispatch/environment +unit_file="/etc/systemd/system/$service_name" +tunnel_service_name=nemoclaw-jetson-tunnel.service +service_port=8787 + +staging_directory="" +temporary_link="" +cleanup_stage="" +managed_file_stage="" +bootstrap_environment_installed=0 +bootstrap_unit_installed=0 +bootstrap_daemon_reloaded=0 +bootstrap_enable_attempted=0 + +usage() { + echo "Usage: nemoclaw-colossus-jetson-dispatch-deploy --commit " >&2 +} + +fail() { + echo "Colossus Jetson dispatcher deployment failed: $*" >&2 + exit 1 +} + +effective_uid() { + /usr/bin/id -u +} + +git_exec() { + /usr/bin/env \ + -u GIT_ALTERNATE_OBJECT_DIRECTORIES \ + -u GIT_COMMON_DIR \ + -u GIT_CONFIG_PARAMETERS \ + -u GIT_DIR \ + -u GIT_EXEC_PATH \ + -u GIT_OBJECT_DIRECTORY \ + -u GIT_PROXY_COMMAND \ + -u GIT_SSH \ + -u GIT_SSH_COMMAND \ + -u GIT_SSL_NO_VERIFY \ + -u GIT_TEMPLATE_DIR \ + -u GIT_WORK_TREE \ + GIT_CONFIG_COUNT=0 \ + GIT_CONFIG_GLOBAL=/dev/null \ + GIT_CONFIG_NOSYSTEM=1 \ + HOME=/nonexistent \ + /usr/bin/git \ + -c core.hooksPath=/dev/null \ + -c credential.helper= \ + -c http.sslVerify=true \ + -c protocol.ext.allow=never \ + -c protocol.file.allow=never \ + "$@" +} + +systemctl_exec() { + /usr/bin/systemctl "$@" +} + +ss_exec() { + /usr/bin/ss "$@" +} + +curl_exec() { + /usr/bin/curl "$@" +} + +sleep_exec() { + /usr/bin/sleep "$@" +} + +id_exec() { + /usr/bin/id "$@" +} + +ssh_keygen_exec() { + /usr/bin/ssh-keygen "$@" +} + +install_file() { + /usr/bin/install -o root -g root -m "$3" "$1" "$2" +} + +files_match() { + /usr/bin/cmp -s "$1" "$2" +} + +symlink_owner() { + /usr/bin/stat -c '%u:%g' "$1" +} + +install_directory() { + local path="$1" mode="$2" + /usr/bin/install -d -o root -g root -m "$mode" "$path" +} + +move_replace() { + /usr/bin/mv -Tf -- "$1" "$2" +} + +move_directory() { + /usr/bin/mv -- "$1" "$2" +} + +require_root_owned_directory() { + local path="$1" expected_mode="$2" metadata + if [ ! -d "$path" ] || [ -L "$path" ]; then + fail "$path must be a directory, not a symbolic link" + fi + metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" + if [ "$metadata" != "0:0:$expected_mode" ]; then + fail "$path must be owned by root:root with mode $expected_mode" + fi +} + +require_root_owned_file() { + local path="$1" expected_mode="$2" metadata + if [ ! -f "$path" ] || [ -L "$path" ]; then + fail "$path must be a regular file, not a symbolic link" + fi + metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" + if [ "$metadata" != "0:0:$expected_mode" ]; then + fail "$path must be owned by root:root with mode $expected_mode" + fi +} + +root_owned_file_matches() { + local path="$1" expected_mode="$2" metadata + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || return 1 + [ "$metadata" = "0:0:$expected_mode" ] +} + +require_service_owned_file() { + local path="$1" expected_mode="$2" uid gid metadata + uid="$(id_exec -u "$service_user")" || fail "could not inspect the $service_user account" + gid="$(id_exec -g "$service_user")" || fail "could not inspect the $service_group group" + if [ ! -f "$path" ] || [ -L "$path" ]; then + fail "$path must be a regular file, not a symbolic link" + fi + metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" + [ "$metadata" = "$uid:$gid:$expected_mode" ] \ + || fail "$path must be owned by $service_user:$service_group with mode $expected_mode" +} + +require_service_owned_directory() { + local path="$1" expected_mode="$2" uid gid metadata + uid="$(id_exec -u "$service_user")" || fail "could not inspect the $service_user account" + gid="$(id_exec -g "$service_user")" || fail "could not inspect the $service_group group" + if [ ! -d "$path" ] || [ -L "$path" ]; then + fail "$path must be a directory, not a symbolic link" + fi + metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" + [ "$metadata" = "$uid:$gid:$expected_mode" ] \ + || fail "$path must be owned by $service_user:$service_group with mode $expected_mode" +} + +ssh_identity_is_valid() { + ssh_keygen_exec -y -P '' -f "$ssh_identity_file" >/dev/null 2>&1 +} + +known_hosts_has_jetson() { + ssh_keygen_exec -F 192.168.55.1 -f "$ssh_known_hosts_file" >/dev/null 2>&1 +} + +node_version_is_supported() { + "$node_executable" -e ' + const [major, minor] = process.versions.node.split(".").map(Number); + if (major < 22 || (major === 22 && minor < 19)) process.exit(1); + ' +} + +validate_dispatcher_prerequisites() { + local uid gid + uid="$(id_exec -u "$service_user")" || fail "the $service_user account is required" + gid="$(id_exec -g "$service_user")" || fail "the $service_group group is required" + [[ "$uid" =~ ^[1-9][0-9]*$ ]] || fail "$service_user must not use UID 0" + [[ "$gid" =~ ^[1-9][0-9]*$ ]] || fail "$service_group must not use GID 0" + + require_service_owned_directory "$service_home" 700 + require_service_owned_file "$ssh_identity_file" 600 + ssh_identity_is_valid \ + || fail "$ssh_identity_file is not a readable OpenSSH private key" + require_service_owned_file "$ssh_known_hosts_file" 600 + known_hosts_has_jetson \ + || fail "$ssh_known_hosts_file does not contain the pinned Jetson host key" + + [ -x "$node_executable" ] || fail "$node_executable is required" + node_version_is_supported || fail "$node_executable must be Node.js 22.19.0 or later" +} + +cleanup_temporary_paths() { + if [ -n "$staging_directory" ] && [[ "$staging_directory" == "$releases_directory"/.* ]]; then + rm -rf -- "$staging_directory" + fi + if [ -n "$temporary_link" ] && [[ "$temporary_link" == "$deploy_root"/.current.* ]]; then + rm -f -- "$temporary_link" + fi + if [ -n "$cleanup_stage" ] && [[ "$cleanup_stage" == "${cleanup_executable%/*}"/.nemoclaw-jetson-cleanup.* ]]; then + rm -f -- "$cleanup_stage" + fi + if [ -n "$managed_file_stage" ]; then + case "$managed_file_stage" in + "${environment_file%/*}"/.environment.* | "${unit_file%/*}"/.nemoclaw-jetson-dispatch.service.*) + rm -f -- "$managed_file_stage" + ;; + esac + fi +} + +trap cleanup_temporary_paths EXIT + +require_root() { + [ "$(effective_uid)" = 0 ] || fail "this command requires root" +} + +parse_commit() { + if [ "$#" -ne 2 ] || [ "$1" != "--commit" ] || [[ ! "$2" =~ ^[a-f0-9]{40}$ ]]; then + usage + exit 2 + fi + printf '%s\n' "$2" +} + +print_stage() { + printf '[%s/5] %s\n' "$1" "$2" +} + +ensure_layout() { + install_directory "$deploy_root" 0755 + install_directory "$releases_directory" 0755 + install_directory "${cleanup_executable%/*}" 0755 + install_directory "${environment_file%/*}" 0755 + require_root_owned_directory "$deploy_root" 755 + require_root_owned_directory "$releases_directory" 755 + require_root_owned_directory "${cleanup_executable%/*}" 755 + require_root_owned_directory "${environment_file%/*}" 755 + require_root_owned_directory "${unit_file%/*}" 755 +} + +acquire_deploy_lock() { + if [ -e "$deploy_lock" ] || [ -L "$deploy_lock" ]; then + require_root_owned_file "$deploy_lock" 600 + else + (umask 077 && : >"$deploy_lock") + chown root:root "$deploy_lock" || fail "could not set deployment-lock ownership" + chmod 0600 "$deploy_lock" || fail "could not set deployment-lock permissions" + fi + exec 9<>"$deploy_lock" + /usr/bin/flock -n 9 || fail "another deployment is running" +} + +verify_release() { + local release="$1" sha="$2" observed + require_root_owned_directory "$release" 755 + if ! observed="$(git_exec -C "$release" config --get remote.origin.url)"; then + fail "could not read the release origin" + fi + [ "$observed" = "$fixed_origin" ] || fail "release origin is not $fixed_origin" + if ! observed="$(git_exec -C "$release" rev-parse --verify HEAD)"; then + fail "could not read the release commit" + fi + [ "$observed" = "$sha" ] || fail "release commit does not match $sha" + if ! observed="$(git_exec -C "$release" status --porcelain=v1 --untracked-files=all)"; then + fail "could not inspect the release checkout" + fi + [ -z "$observed" ] || fail "release checkout is modified" + require_root_owned_file "$release/$cleanup_relative" 755 + require_root_owned_file "$release/$environment_relative" 644 + require_root_owned_file "$release/$unit_relative" 644 +} + +prepare_release() { + local sha="$1" + local release="$releases_directory/$sha" + if [ -e "$release" ] || [ -L "$release" ]; then + verify_release "$release" "$sha" + printf '%s\n' "$release" + return + fi + + staging_directory="$(/usr/bin/mktemp -d "$releases_directory/.${sha}.XXXXXX")" + chmod 0755 "$staging_directory" || fail "could not set release-directory permissions" + git_exec -C "$staging_directory" init --quiet || fail "could not initialize the release checkout" + if ! git_exec -C "$staging_directory" remote add origin "$fixed_origin"; then + fail "could not set the fixed release origin" + fi + if ! git_exec -C "$staging_directory" fetch --depth=1 --no-tags origin "$sha"; then + fail "could not fetch commit $sha from the fixed origin" + fi + if ! git_exec -C "$staging_directory" checkout --detach --quiet FETCH_HEAD; then + fail "could not check out commit $sha" + fi + verify_release "$staging_directory" "$sha" + move_directory "$staging_directory" "$release" || fail "could not install release $sha" + staging_directory="" + verify_release "$release" "$sha" + printf '%s\n' "$release" +} + +service_load_state() { + systemctl_exec show --property=LoadState --value "$service_name" +} + +service_active_state() { + systemctl_exec show --property=ActiveState --value "$service_name" +} + +device_lock_is_absent() { + [ ! -e "$device_lock" ] && [ ! -L "$device_lock" ] +} + +require_device_lock_absent() { + device_lock_is_absent || fail "cleanup left $device_lock; recover cleanup before deployment" +} + +stop_installed_service() { + local active_state + systemctl_exec stop "$service_name" || fail "could not stop $service_name" + if ! active_state="$(service_active_state)"; then + fail "could not inspect $service_name after stop" + fi + [ "$active_state" = inactive ] || fail "$service_name has unexpected state after stop: $active_state" + require_device_lock_absent +} + +selected_release() { + local target sha + if [ ! -e "$current_link" ] && [ ! -L "$current_link" ]; then + return 1 + fi + [ -L "$current_link" ] || fail "$current_link must be a symbolic link" + target="$(/usr/bin/readlink "$current_link")" || fail "could not read $current_link" + sha="${target##*/}" + if [[ ! "$sha" =~ ^[a-f0-9]{40}$ ]] || [ "$target" != "$releases_directory/$sha" ]; then + fail "$current_link does not select one managed release" + fi + verify_release "$target" "$sha" + printf '%s\n' "$target" +} + +cleanup_link_matches() { + local metadata target + [ -L "$cleanup_executable" ] || return 1 + target="$(/usr/bin/readlink "$cleanup_executable")" || return 1 + [ "$target" = "$cleanup_link_target" ] || return 1 + metadata="$(symlink_owner "$cleanup_executable")" || return 1 + [ "$metadata" = 0:0 ] +} + +validate_prior_cleanup_selection() { + local previous_release="$1" + if [ -n "$previous_release" ]; then + cleanup_link_matches || fail "$cleanup_executable does not select the managed current release" + return + fi + if [ ! -e "$cleanup_executable" ] && [ ! -L "$cleanup_executable" ]; then + return + fi + cleanup_link_matches || fail "$cleanup_executable exists without one managed current release" +} + +require_public_ingress_disabled() { + public_ingress_is_disabled || fail "$public_ingress_error" +} + +public_ingress_is_disabled() { + local load_state active_state unit_file_state + public_ingress_error="" + if ! load_state="$(systemctl_exec show --property=LoadState --value "$tunnel_service_name")"; then + public_ingress_error="could not inspect $tunnel_service_name" + return 1 + fi + if [ "$load_state" = not-found ]; then + return + fi + if [ "$load_state" != loaded ]; then + public_ingress_error="$tunnel_service_name has unsupported load state: $load_state" + return 1 + fi + if ! active_state="$(systemctl_exec show --property=ActiveState --value "$tunnel_service_name")"; then + public_ingress_error="could not inspect $tunnel_service_name activity" + return 1 + fi + if ! unit_file_state="$(systemctl_exec show --property=UnitFileState --value "$tunnel_service_name")"; then + public_ingress_error="could not inspect $tunnel_service_name enablement" + return 1 + fi + if [ "$active_state" != inactive ] || [ "$unit_file_state" != disabled ]; then + public_ingress_error="$tunnel_service_name must be disabled and inactive" + return 1 + fi +} + +require_bootstrap_destinations_absent() { + if [ -e "$environment_file" ] || [ -L "$environment_file" ]; then + fail "$environment_file exists while $service_name is not loaded" + fi + if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then + fail "$unit_file exists while $service_name is not loaded" + fi +} + +atomic_install_cleanup_link() { + cleanup_link_matches && return 0 + if [ -e "$cleanup_executable" ] || [ -L "$cleanup_executable" ]; then + return 1 + fi + cleanup_stage="$(/usr/bin/mktemp "${cleanup_executable%/*}/.nemoclaw-jetson-cleanup.XXXXXX")" + rm -f -- "$cleanup_stage" + ln -s -- "$cleanup_link_target" "$cleanup_stage" || return 1 + move_replace "$cleanup_stage" "$cleanup_executable" || { + rm -f -- "$cleanup_stage" + cleanup_stage="" + return 1 + } + cleanup_stage="" + cleanup_link_matches +} + +atomic_select_release() { + local release="$1" + temporary_link="$deploy_root/.current.${release##*/}.$$" + if [ -e "$temporary_link" ] || [ -L "$temporary_link" ]; then + return 1 + fi + ln -s -- "$release" "$temporary_link" || { + temporary_link="" + return 1 + } + move_replace "$temporary_link" "$current_link" || { + rm -f -- "$temporary_link" + temporary_link="" + return 1 + } + temporary_link="" + [ -L "$current_link" ] && [ "$(/usr/bin/readlink "$current_link")" = "$release" ] +} + +install_managed_file() { + local source="$1" destination="$2" mode="$3" prefix="$4" + managed_file_stage="$(/usr/bin/mktemp "${destination%/*}/.${prefix}.XXXXXX")" || return 1 + if ! install_file "$source" "$managed_file_stage" "$mode"; then + rm -f -- "$managed_file_stage" + managed_file_stage="" + return 1 + fi + if ! root_owned_file_matches "$managed_file_stage" "$mode" \ + || ! files_match "$source" "$managed_file_stage" \ + || ! move_replace "$managed_file_stage" "$destination"; then + rm -f -- "$managed_file_stage" + managed_file_stage="" + return 1 + fi + managed_file_stage="" + root_owned_file_matches "$destination" "$mode" && files_match "$source" "$destination" +} + +install_bootstrap_files() { + local release="$1" + install_managed_file "$release/$environment_relative" "$environment_file" 600 environment \ + || return 1 + bootstrap_environment_installed=1 + install_managed_file "$release/$unit_relative" "$unit_file" 644 nemoclaw-jetson-dispatch.service \ + || return 1 + bootstrap_unit_installed=1 +} + +loopback_service_responds() { + local listeners http_code + systemctl_exec is-active --quiet "$service_name" || return 1 + listeners="$(ss_exec -H -ltn "sport = :$service_port")" || return 1 + [ -n "$listeners" ] || return 1 + printf '%s\n' "$listeners" | /usr/bin/awk -v expected="127.0.0.1:$service_port" ' + NF < 4 || $4 != expected { invalid = 1 } + END { exit invalid || NR == 0 } + ' || return 1 + http_code="$( + curl_exec \ + --disable \ + --noproxy '*' \ + --silent \ + --show-error \ + --output /dev/null \ + --write-out '%{http_code}' \ + --max-time 5 \ + --request POST \ + --header 'Content-Type: application/json' \ + --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ + "http://127.0.0.1:$service_port/v1/jobs" + )" || return 1 + [ "$http_code" = 401 ] +} + +verify_loopback_service() { + local _ + for _ in {1..20}; do + loopback_service_responds && return 0 + sleep_exec 0.25 + done + return 1 +} + +start_and_verify_service() { + public_ingress_is_disabled || return 1 + systemctl_exec start "$service_name" || return 1 + if ! public_ingress_is_disabled; then + stop_unverified_service || return 1 + return 1 + fi + verify_loopback_service || return 1 + if ! public_ingress_is_disabled; then + stop_unverified_service || return 1 + return 1 + fi +} + +enable_and_verify_initial_service() { + local load_state + systemctl_exec daemon-reload || return 1 + bootstrap_daemon_reloaded=1 + load_state="$(service_load_state)" || return 1 + [ "$load_state" = loaded ] || return 1 + public_ingress_is_disabled || return 1 + bootstrap_enable_attempted=1 + systemctl_exec enable --now "$service_name" || return 1 + public_ingress_is_disabled || return 1 + verify_loopback_service || return 1 + public_ingress_is_disabled +} + +stop_unverified_service() { + local active_state + systemctl_exec stop "$service_name" || return 1 + active_state="$(service_active_state)" || return 1 + [ "$active_state" = inactive ] || return 1 + device_lock_is_absent +} + +restore_cleanup_selection() { + local cleanup_was_present="$1" + if [ "$cleanup_was_present" = 1 ]; then + cleanup_link_matches + else + rm -f -- "$cleanup_executable" + fi +} + +restore_release_selection() { + local previous_release="$1" + if [ -n "$previous_release" ]; then + atomic_select_release "$previous_release" + return + fi + if [ -L "$current_link" ]; then + rm -f -- "$current_link" + elif [ -e "$current_link" ]; then + return 1 + fi +} + +rollback_deployment() { + local previous_release="$1" cleanup_was_present="$2" service_installed="$3" rollback_failed=0 active_state + if [ "$service_installed" = 1 ]; then + systemctl_exec stop "$service_name" || return 1 + active_state="$(service_active_state)" || return 1 + [ "$active_state" = inactive ] || return 1 + device_lock_is_absent || return 1 + fi + restore_release_selection "$previous_release" || rollback_failed=1 + restore_cleanup_selection "$cleanup_was_present" || rollback_failed=1 + if [ "$service_installed" = 1 ] && [ -n "$previous_release" ]; then + start_and_verify_service || rollback_failed=1 + fi + [ "$rollback_failed" = 0 ] +} + +rollback_initial_deployment() { + local previous_release="$1" cleanup_was_present="$2" active_state load_state rollback_failed=0 + if [ "$bootstrap_enable_attempted" = 1 ]; then + systemctl_exec disable --now "$service_name" || return 1 + active_state="$(service_active_state)" || return 1 + [ "$active_state" = inactive ] || return 1 + device_lock_is_absent || return 1 + fi + + restore_release_selection "$previous_release" || rollback_failed=1 + restore_cleanup_selection "$cleanup_was_present" || rollback_failed=1 + + if [ "$bootstrap_unit_installed" = 1 ]; then + root_owned_file_matches "$unit_file" 644 || return 1 + rm -f -- "$unit_file" || return 1 + fi + if [ "$bootstrap_environment_installed" = 1 ]; then + root_owned_file_matches "$environment_file" 600 || return 1 + rm -f -- "$environment_file" || return 1 + fi + if [ "$bootstrap_daemon_reloaded" = 1 ]; then + systemctl_exec daemon-reload || return 1 + load_state="$(service_load_state)" || return 1 + [ "$load_state" = not-found ] || return 1 + fi + [ "$rollback_failed" = 0 ] +} + +activate_release() { + local release="$1" + atomic_install_cleanup_link || return 1 + atomic_select_release "$release" || return 1 + cleanup_link_matches || return 1 + [ "$(/usr/bin/readlink "$current_link")" = "$release" ] || return 1 +} + +main() { + local sha release load_state service_installed=0 previous_release="" cleanup_was_present=0 + require_root + sha="$(parse_commit "$@")" + print_stage 1 "Validate request and prepared host" + validate_dispatcher_prerequisites + ensure_layout + acquire_deploy_lock + + load_state="$(service_load_state)" || fail "could not inspect $service_name" + if [ "$load_state" = loaded ]; then + service_installed=1 + print_stage 2 "Stop and verify the dispatcher" + require_public_ingress_disabled + stop_installed_service + elif [ "$load_state" != not-found ]; then + fail "$service_name has unsupported load state: $load_state" + else + print_stage 2 "Verify initial deployment state" + require_device_lock_absent + require_public_ingress_disabled + require_bootstrap_destinations_absent + fi + + if [ -e "$current_link" ] || [ -L "$current_link" ]; then + previous_release="$(selected_release)" || fail "could not validate $current_link" + fi + validate_prior_cleanup_selection "$previous_release" + [ -n "$previous_release" ] && cleanup_was_present=1 + if [ "$service_installed" = 1 ] && [ -z "$previous_release" ]; then + fail "$service_name is installed without one managed current release" + fi + print_stage 3 "Fetch and verify the release" + release="$(prepare_release "$sha")" + + print_stage 4 "Select dispatcher and cleanup code" + if [ "$service_installed" = 1 ]; then + require_public_ingress_disabled + fi + if ! activate_release "$release"; then + if rollback_deployment "$previous_release" "$cleanup_was_present" "$service_installed"; then + fail "release $sha activation or verification failed; the previous deployment state was restored" + fi + fail "release $sha activation or verification failed, and rollback did not restore a verified service" + fi + + if [ "$service_installed" = 1 ]; then + print_stage 5 "Start and verify the dispatcher" + if ! start_and_verify_service; then + if rollback_deployment "$previous_release" "$cleanup_was_present" "$service_installed"; then + fail "release $sha activation or verification failed; the previous deployment state was restored" + fi + fail "release $sha activation or verification failed, and rollback did not restore a verified service" + fi + echo "Deployed Colossus Jetson dispatcher release $sha and verified its loopback service" + else + print_stage 5 "Install, enable, and verify the dispatcher" + if ! install_bootstrap_files "$release" || ! enable_and_verify_initial_service; then + if rollback_initial_deployment "$previous_release" "$cleanup_was_present"; then + fail "release $sha initial deployment failed; the release, unit, and environment were rolled back" + fi + fail "release $sha initial deployment failed, and rollback did not restore the prepared host" + fi + echo "Deployed Colossus Jetson dispatcher release $sha and verified its loopback service; public ingress remains disabled" + fi +} + +if [[ "${BASH_SOURCE[0]}" = "$0" ]]; then + main "$@" +fi diff --git a/tools/e2e/colossus-jetson-dispatch.environment b/tools/e2e/colossus-jetson-dispatch.environment new file mode 100644 index 00000000000..a3fa0a8e618 --- /dev/null +++ b/tools/e2e/colossus-jetson-dispatch.environment @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +JETSON_DISPATCH_STATE_DIRECTORY=/var/lib/nemoclaw-jetson-dispatch/state +JETSON_DISPATCH_GITHUB_REPOSITORY_ID=1182547092 +JETSON_DISPATCH_PORT=8787 +JETSON_DISPATCH_EXECUTION_TIMEOUT_SECONDS=3000 +JETSON_DISPATCH_CLEANUP_TIMEOUT_SECONDS=300 +JETSON_DISPATCH_TEST_TIMEOUT_SECONDS=2700 +JETSON_DISPATCH_SSH_DESTINATION=nvidia@192.168.55.1 +JETSON_DISPATCH_SSH_IDENTITY_FILE=/var/lib/nemoclaw-jetson-dispatch/id_ed25519 +JETSON_DISPATCH_SSH_KNOWN_HOSTS_FILE=/var/lib/nemoclaw-jetson-dispatch/known_hosts +JETSON_DISPATCH_CLEANUP_EXECUTABLE=/usr/local/libexec/nemoclaw-jetson-cleanup diff --git a/tools/e2e/jetson-dispatch-cleanup.sh b/tools/e2e/jetson-dispatch-cleanup.sh new file mode 100755 index 00000000000..44b1563e47e --- /dev/null +++ b/tools/e2e/jetson-dispatch-cleanup.sh @@ -0,0 +1,635 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [ "$#" -ne 0 ]; then + echo "Jetson cleanup accepts no arguments" >&2 + exit 2 +fi + +state_directory=/var/lib/nemoclaw-jetson-dispatch/state +lock_file="$state_directory/device.lock" +identity_file=/var/lib/nemoclaw-jetson-dispatch/id_ed25519 +known_hosts_file=/var/lib/nemoclaw-jetson-dispatch/known_hosts +destination=nvidia@192.168.55.1 + +mapfile -t lock_lines <"$lock_file" +if [ "${#lock_lines[@]}" -ne 1 ]; then + echo "Jetson cleanup lock contains trailing data" >&2 + exit 1 +fi +job_id="${lock_lines[0]}" +if ! [[ "$job_id" =~ ^[a-f0-9]{64}$ ]]; then + echo "Jetson cleanup lock contains an invalid job ID" >&2 + exit 1 +fi + +ssh_args=( + -F /dev/null + -T + -o BatchMode=yes + -o ConnectTimeout=15 + -o IdentitiesOnly=yes + -o ServerAliveCountMax=2 + -o ServerAliveInterval=15 + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=$known_hosts_file" + -i "$identity_file" +) + +discovery_output="$( + ssh "${ssh_args[@]}" "$destination" bash -s -- "$job_id" <<'JETSON_DISCOVERY' +set -euo pipefail + +job_id="$1" +[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] + +workspace_root=/var/tmp/nemoclaw-jetson-e2e +workspace="$workspace_root/$job_id" +job_home="$workspace/home" +sandbox_name=e2e-jetson-nvmap +gateway_container=openshell-cluster-nemoclaw +gateway_volume=openshell-cluster-nemoclaw +image_repository=nemoclaw-sandbox-local +service_directory="/tmp/nemoclaw-services-$sandbox_name" + +require_plain_directory_if_present() { + local directory="$1" + if [ -L "$directory" ] || { [ -e "$directory" ] && [ ! -d "$directory" ]; }; then + echo "Refusing an untrusted cleanup directory: $directory" >&2 + exit 1 + fi +} + +require_plain_directory_if_present "$workspace_root" +require_plain_directory_if_present "$workspace" +require_plain_directory_if_present "$job_home" +require_plain_directory_if_present "$job_home/.nemoclaw" +require_plain_directory_if_present "$job_home/.local" +require_plain_directory_if_present "$job_home/.local/state" +require_plain_directory_if_present "$job_home/.local/state/nemoclaw" +require_plain_directory_if_present "$job_home/.local/state/nemoclaw/openshell-docker-gateway" +require_plain_directory_if_present "$service_directory" + +recorded_volumes=() +record_volume() { + local volume="$1" recorded + [[ "$volume" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$ ]] || { + echo "Refusing an invalid recorded Docker volume name" >&2 + exit 1 + } + for recorded in "${recorded_volumes[@]}"; do + [ "$recorded" = "$volume" ] && return 0 + done + recorded_volumes+=("$volume") +} + +record_container_volumes() { + local container="$1" volume volume_output + volume_output="$( + docker container inspect --format \ + '{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}' \ + "$container" + )" || { + echo "Unable to inspect a test-owned Docker container" >&2 + exit 1 + } + while IFS= read -r volume; do + [ -n "$volume" ] || continue + record_volume "$volume" + done <<<"$volume_output" +} + +sandbox_container_output="$( + docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name" +)" || { + echo "Unable to list test-owned Docker containers" >&2 + exit 1 +} +sandbox_containers=() +if [ -n "$sandbox_container_output" ]; then + mapfile -t sandbox_containers <<<"$sandbox_container_output" +fi +for container in "${sandbox_containers[@]}"; do + [[ "$container" =~ ^[a-f0-9]{12,64}$ ]] || { + echo "Refusing an invalid recorded Docker container ID" >&2 + exit 1 + } + container_image="$(docker container inspect --format '{{.Config.Image}}' "$container")" + case "$container_image" in + "$image_repository:$sandbox_name-"*) ;; + *) echo "Refusing a sandbox container whose image is not test-owned" >&2; exit 1 ;; + esac + record_container_volumes "$container" +done +container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { + echo "Unable to list Docker containers" >&2 + exit 1 +} +gateway_container_ids="$( + printf '%s\n' "$container_rows" | awk -F '\t' -v name="$gateway_container" '$2 == name { print $1 }' +)" +if [ -n "$gateway_container_ids" ]; then + if [[ "$gateway_container_ids" == *$'\n'* ]] || ! [[ "$gateway_container_ids" =~ ^[a-f0-9]{12,64}$ ]]; then + echo "Refusing an ambiguous test-owned gateway container" >&2 + exit 1 + fi + record_container_volumes "$gateway_container_ids" +fi +record_volume "$gateway_volume" + +read_recorded_pid() { + local pid_file="$1" + if [ ! -e "$pid_file" ]; then + return 0 + fi + if [ -L "$pid_file" ] || [ ! -f "$pid_file" ]; then + echo "Refusing an untrusted cleanup PID file: $pid_file" >&2 + exit 1 + fi + local pid + IFS= read -r pid <"$pid_file" + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || { + echo "Refusing an invalid cleanup PID file: $pid_file" >&2 + exit 1 + } + printf '%s\n' "$pid" +} + +read_proc_uid() { + awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' "$1/status" 2>/dev/null +} + +read_proc_environment() { + dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' +} + +read_proc_command() { + dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' +} + +handle_proc_read_failure() { + local proc_dir="$1" field="$2" process_uid directory_uid + [ -d "$proc_dir" ] || return 0 + if process_uid="$(read_proc_uid "$proc_dir")"; then + [ "$process_uid" = "$(id -u)" ] || return 0 + else + [ -d "$proc_dir" ] || return 0 + if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then + [ -d "$proc_dir" ] || return 0 + echo "Unable to verify the owner of a live process after a failed $field read" >&2 + exit 1 + fi + [ "$directory_uid" = "$(id -u)" ] || return 0 + fi + echo "Unable to inspect $field for a live same-user process" >&2 + exit 1 +} + +validate_recorded_pid() { + local pid="$1" marker="$2" + [ -n "$pid" ] || return 0 + kill -0 "$pid" 2>/dev/null || return 0 + local process_uid cmdline environment proc_dir="/proc/$pid" + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Refusing to record a live process whose owner cannot be verified" >&2 + exit 1 + fi + [ "$process_uid" = "$(id -u)" ] || { + echo "Refusing to record a process owned by another user" >&2 + exit 1 + } + if ! cmdline="$(read_proc_command "$proc_dir")"; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Refusing to record a live process whose command cannot be verified" >&2 + exit 1 + fi + case "$cmdline" in + *"$marker"*) ;; + *) echo "Refusing to record a PID with an unexpected command" >&2; exit 1 ;; + esac + if ! environment="$(read_proc_environment "$proc_dir")"; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Refusing to record a live process whose environment cannot be verified" >&2 + exit 1 + fi + if ! printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home"; then + echo "Refusing to record a process outside the job home" >&2 + exit 1 + fi +} + +auth_proxy_pid="$(read_recorded_pid "$job_home/.nemoclaw/ollama-auth-proxy.pid")" +gateway_pid="$(read_recorded_pid "$job_home/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid")" +cloudflared_pid="$(read_recorded_pid "$service_directory/cloudflared.pid")" +validate_recorded_pid "$auth_proxy_pid" ollama-auth-proxy. +validate_recorded_pid "$gateway_pid" openshell-gateway +validate_recorded_pid "$cloudflared_pid" cloudflared + +printf '%s\n' nemoclaw-cleanup-evidence-v1-begin +for volume in "${recorded_volumes[@]}"; do + printf 'volume\t%s\n' "$volume" +done +for pid in "$auth_proxy_pid" "$gateway_pid" "$cloudflared_pid"; do + if [ -n "$pid" ]; then + printf 'processId\t%s\n' "$pid" + fi +done +for proc_dir in /proc/[0-9]*; do + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" owner + continue + fi + [ "$process_uid" = "$(id -u)" ] || continue + if ! environment="$(read_proc_environment "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" environment + continue + fi + printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue + if ! cmdline="$(read_proc_command "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" command + continue + fi + case "$cmdline" in + *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) + printf 'processId\t%s\n' "${proc_dir##*/}" + ;; + esac +done +printf '%s\n' nemoclaw-cleanup-evidence-v1-end +JETSON_DISCOVERY +)" + +canonical_evidence="$( + # shellcheck disable=SC2016 + printf '%s\n' "$discovery_output" | /usr/bin/node -e ' + const fs = require("node:fs"); + const path = require("node:path"); + const stateDirectory = process.argv[1]; + const jobId = process.argv[2]; + if (!/^[a-f0-9]{64}$/.test(jobId)) process.exit(1); + const begin = "nemoclaw-cleanup-evidence-v1-begin"; + const end = "nemoclaw-cleanup-evidence-v1-end"; + const lines = fs.readFileSync(0, "utf8").replace(/\r/g, "").trimEnd().split("\n"); + const beginIndex = lines.indexOf(begin); + const endIndex = lines.indexOf(end); + if (beginIndex < 0 || endIndex < 0 || beginIndex !== lines.lastIndexOf(begin) || endIndex !== lines.lastIndexOf(end) || beginIndex >= endIndex) process.exit(1); + const discovered = { schemaVersion: 1, volumes: [], processIds: [] }; + for (const line of lines.slice(beginIndex + 1, endIndex)) { + const fields = line.split("\t"); + if (fields.length !== 2) process.exit(1); + if (fields[0] === "volume" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(fields[1])) discovered.volumes.push(fields[1]); + else if (fields[0] === "processId" && /^[1-9][0-9]*$/.test(fields[1]) && Number.isSafeInteger(Number(fields[1]))) discovered.processIds.push(Number(fields[1])); + else process.exit(1); + } + const cleanupFile = path.join(stateDirectory, `${jobId}.cleanup.json`); + let previous = { schemaVersion: 1, volumes: [], processIds: [] }; + if (fs.existsSync(cleanupFile)) { + const metadata = fs.lstatSync(cleanupFile); + if (!metadata.isFile() || metadata.nlink !== 1 || (metadata.mode & 0o077) !== 0 || metadata.size > 64 * 1024) process.exit(1); + previous = JSON.parse(fs.readFileSync(cleanupFile, "utf8")); + } + const validate = (record) => { + const keys = Object.keys(record).sort(); + if (JSON.stringify(keys) !== JSON.stringify(["processIds", "schemaVersion", "volumes"]) || record.schemaVersion !== 1 || !Array.isArray(record.volumes) || !Array.isArray(record.processIds)) process.exit(1); + if (!record.volumes.every((value) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(value))) process.exit(1); + if (!record.processIds.every((value) => Number.isSafeInteger(value) && value > 0)) process.exit(1); + }; + validate(previous); + validate(discovered); + const merged = { + schemaVersion: 1, + volumes: [...new Set([...previous.volumes, ...discovered.volumes])].sort(), + processIds: [...new Set([...previous.processIds, ...discovered.processIds])].sort((a, b) => a - b), + }; + const serialized = `${JSON.stringify(merged)}\n`; + if (Buffer.byteLength(serialized) > 64 * 1024) process.exit(1); + const temporaryFile = path.join(stateDirectory, `.${jobId}.${process.pid}.cleanup.tmp`); + let descriptor; + try { + descriptor = fs.openSync(temporaryFile, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | (fs.constants.O_NOFOLLOW ?? 0), 0o600); + fs.fchmodSync(descriptor, 0o600); + fs.writeFileSync(descriptor, serialized, "utf8"); + fs.fsyncSync(descriptor); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } + fs.renameSync(temporaryFile, cleanupFile); + const directoryDescriptor = fs.openSync(stateDirectory, fs.constants.O_RDONLY); + try { fs.fsyncSync(directoryDescriptor); } finally { fs.closeSync(directoryDescriptor); } + process.stdout.write(serialized); + ' "$state_directory" "$job_id" +)" + +cleanup_evidence_base64="$(printf '%s\n' "$canonical_evidence" | base64 --wrap=0)" +ssh "${ssh_args[@]}" "$destination" bash -s -- "$job_id" "$cleanup_evidence_base64" <<'JETSON_CLEANUP' +set -euo pipefail + +job_id="$1" +cleanup_evidence_base64="$2" +[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] + +workspace_root=/var/tmp/nemoclaw-jetson-e2e +workspace="$workspace_root/$job_id" +job_home="$workspace/home" +sandbox_name=e2e-jetson-nvmap +gateway_container=openshell-cluster-nemoclaw +image_repository=nemoclaw-sandbox-local +service_directory="/tmp/nemoclaw-services-$sandbox_name" + +require_plain_directory_if_present() { + local directory="$1" + if [ -L "$directory" ] || { [ -e "$directory" ] && [ ! -d "$directory" ]; }; then + echo "Refusing an untrusted cleanup directory: $directory" >&2 + exit 1 + fi +} + +require_plain_directory_if_present "$workspace_root" +require_plain_directory_if_present "$workspace" +require_plain_directory_if_present "$job_home" +require_plain_directory_if_present "$job_home/.nemoclaw" +require_plain_directory_if_present "$job_home/.local" +require_plain_directory_if_present "$job_home/.local/state" +require_plain_directory_if_present "$job_home/.local/state/nemoclaw" +require_plain_directory_if_present "$job_home/.local/state/nemoclaw/openshell-docker-gateway" +require_plain_directory_if_present "$service_directory" + +cleanup_rows="$( + printf '%s' "$cleanup_evidence_base64" | base64 --decode | node -e ' + const fs = require("node:fs"); + const value = JSON.parse(fs.readFileSync(0, "utf8")); + const keys = Object.keys(value).sort(); + if (JSON.stringify(keys) !== JSON.stringify(["processIds", "schemaVersion", "volumes"]) || value.schemaVersion !== 1 || !Array.isArray(value.volumes) || !Array.isArray(value.processIds)) process.exit(1); + for (const volume of value.volumes) { + if (typeof volume !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(volume)) process.exit(1); + process.stdout.write("volume\t" + volume + "\n"); + } + for (const pid of value.processIds) { + if (!Number.isSafeInteger(pid) || pid < 1) process.exit(1); + process.stdout.write("processId\t" + pid + "\n"); + } + ' +)" +recorded_volumes=() +recorded_process_ids=() +if [ -n "$cleanup_rows" ]; then + while IFS=$'\t' read -r kind identity; do + case "$kind" in + volume) recorded_volumes+=("$identity") ;; + processId) recorded_process_ids+=("$identity") ;; + *) exit 1 ;; + esac + done <<<"$cleanup_rows" +fi + +read_proc_uid() { + awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' "$1/status" 2>/dev/null +} + +read_proc_environment() { + dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' +} + +read_proc_command() { + dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' +} + +handle_proc_read_failure() { + local proc_dir="$1" field="$2" process_uid directory_uid + [ -d "$proc_dir" ] || return 0 + if process_uid="$(read_proc_uid "$proc_dir")"; then + [ "$process_uid" = "$(id -u)" ] || return 0 + else + [ -d "$proc_dir" ] || return 0 + if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then + [ -d "$proc_dir" ] || return 0 + echo "Unable to verify the owner of a live process after a failed $field read" >&2 + exit 1 + fi + [ "$directory_uid" = "$(id -u)" ] || return 0 + fi + echo "Unable to inspect $field for a live same-user process" >&2 + exit 1 +} + +stop_recorded_pid() { + local pid="$1" + kill -0 "$pid" 2>/dev/null || return 0 + local process_uid cmdline environment proc_dir="/proc/$pid" + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Refusing to stop a live process whose owner cannot be verified" >&2 + exit 1 + fi + [ "$process_uid" = "$(id -u)" ] || { + echo "Refusing to stop a process owned by another user" >&2 + exit 1 + } + if ! cmdline="$(read_proc_command "$proc_dir")"; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Refusing to stop a live process whose command cannot be verified" >&2 + exit 1 + fi + case "$cmdline" in + *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) ;; + *) echo "Refusing to stop a recorded PID with an unexpected command" >&2; exit 1 ;; + esac + if ! environment="$(read_proc_environment "$proc_dir")"; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Refusing to stop a live process whose environment cannot be verified" >&2 + exit 1 + fi + if ! printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home"; then + echo "Refusing to stop a process outside the job home" >&2 + exit 1 + fi + if ! kill "$pid" 2>/dev/null; then + kill -0 "$pid" 2>/dev/null || return 0 + echo "Unable to stop a validated cleanup process" >&2 + exit 1 + fi + for _attempt in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "$pid" 2>/dev/null || return 0 + sleep 0.2 + done + kill -KILL "$pid" + for _attempt in 1 2 3 4 5; do + kill -0 "$pid" 2>/dev/null || return 0 + sleep 0.2 + done + echo "Recorded cleanup process $pid did not stop" >&2 + exit 1 +} + +for pid in "${recorded_process_ids[@]}"; do + stop_recorded_pid "$pid" +done + +sandbox_container_output="$( + docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name" +)" || { + echo "Unable to list test-owned Docker containers" >&2 + exit 1 +} +sandbox_containers=() +if [ -n "$sandbox_container_output" ]; then + mapfile -t sandbox_containers <<<"$sandbox_container_output" +fi +for container in "${sandbox_containers[@]}"; do + [[ "$container" =~ ^[a-f0-9]{12,64}$ ]] || exit 1 + container_image="$(docker container inspect --format '{{.Config.Image}}' "$container")" + case "$container_image" in + "$image_repository:$sandbox_name-"*) ;; + *) echo "Refusing a sandbox container whose image is not test-owned" >&2; exit 1 ;; + esac + docker container rm --force "$container" +done +container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { + echo "Unable to list Docker containers" >&2 + exit 1 +} +gateway_container_ids="$( + printf '%s\n' "$container_rows" | awk -F '\t' -v name="$gateway_container" '$2 == name { print $1 }' +)" +if [ -n "$gateway_container_ids" ]; then + if [[ "$gateway_container_ids" == *$'\n'* ]] || ! [[ "$gateway_container_ids" =~ ^[a-f0-9]{12,64}$ ]]; then + echo "Refusing an ambiguous test-owned gateway container" >&2 + exit 1 + fi + docker container rm --force "$gateway_container_ids" +fi +volume_names="$(docker volume ls --format '{{.Name}}')" || { + echo "Unable to list Docker volumes" >&2 + exit 1 +} +for volume in "${recorded_volumes[@]}"; do + if printf '%s\n' "$volume_names" | grep -Fqx "$volume"; then + docker volume rm "$volume" + fi +done + +list_test_owned_images() { + local image_rows repository tag + image_rows="$(docker image ls "$image_repository" --format '{{.Repository}}\t{{.Tag}}')" || { + echo "Unable to list test-owned Docker images" >&2 + return 1 + } + while IFS=$'\t' read -r repository tag; do + [ -n "$repository" ] || continue + if [ "$repository" = "$image_repository" ] && [[ "$tag" =~ ^e2e-jetson-nvmap-[a-z0-9_.-]+$ ]]; then + printf '%s:%s\n' "$repository" "$tag" + fi + done <<<"$image_rows" +} + +job_images_output="$(list_test_owned_images)" +job_images=() +if [ -n "$job_images_output" ]; then + mapfile -t job_images <<<"$job_images_output" +fi +for image in "${job_images[@]}"; do + docker image rm "$image" +done + +if [ -e "$service_directory" ]; then + rm -rf -- "$service_directory" +fi +if [ -e "$workspace" ]; then + rm -rf -- "$workspace" +fi + +test ! -e "$workspace" +test ! -e "$service_directory" +sandbox_container_output="$(docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name")" || { + echo "Unable to verify test-owned Docker container absence" >&2 + exit 1 +} +if [ -n "$sandbox_container_output" ]; then + echo "A test-owned sandbox container remains" >&2 + exit 1 +fi +container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { + echo "Unable to verify Docker container absence" >&2 + exit 1 +} +if printf '%s\n' "$container_rows" | awk -F '\t' -v name="$gateway_container" '$2 == name { found = 1 } END { exit found ? 0 : 1 }'; then + echo "The test-owned gateway container remains" >&2 + exit 1 +fi +volume_names="$(docker volume ls --format '{{.Name}}')" || { + echo "Unable to verify Docker volume absence" >&2 + exit 1 +} +for volume in "${recorded_volumes[@]}"; do + if printf '%s\n' "$volume_names" | grep -Fqx "$volume"; then + echo "A recorded test-owned Docker volume remains" >&2 + exit 1 + fi +done +for pid in "${recorded_process_ids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "A recorded test-owned helper process remains" >&2 + exit 1 + fi +done +for proc_dir in /proc/[0-9]*; do + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" owner + continue + fi + [ "$process_uid" = "$(id -u)" ] || continue + if ! environment="$(read_proc_environment "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" environment + continue + fi + printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue + if ! cmdline="$(read_proc_command "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" command + continue + fi + case "$cmdline" in + *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) + echo "A job-owned helper process remains" >&2 + exit 1 + ;; + esac +done +remaining_job_images="$(list_test_owned_images)" || exit 1 +if [ -n "$remaining_job_images" ]; then + echo "A test-owned Jetson image remains" >&2 + exit 1 +fi +for openshell_component in openshell openshell-gateway openshell-sandbox; do + if PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + command -v "$openshell_component" >/dev/null 2>&1; then + echo "A host-level OpenShell binary remains after cleanup" >&2 + exit 1 + fi + for host_bin in "/usr/local/bin/$openshell_component" "/usr/bin/$openshell_component" "$HOME/.local/bin/$openshell_component"; do + if [ -e "$host_bin" ] || [ -L "$host_bin" ]; then + echo "A host-level OpenShell binary remains after cleanup" >&2 + exit 1 + fi + done +done +JETSON_CLEANUP + +printf '%s\n' nemoclaw-cleanup-evidence-v1-begin +# shellcheck disable=SC2016 +printf '%s\n' "$canonical_evidence" | /usr/bin/node -e ' + const fs = require("node:fs"); + const value = JSON.parse(fs.readFileSync(0, "utf8")); + for (const volume of value.volumes) console.log(`volume\t${volume}`); + for (const pid of value.processIds) console.log(`processId\t${pid}`); +' +printf '%s\n' nemoclaw-cleanup-evidence-v1-end diff --git a/tools/e2e/jetson-dispatch-client.mts b/tools/e2e/jetson-dispatch-client.mts new file mode 100644 index 00000000000..a74632c86b4 --- /dev/null +++ b/tools/e2e/jetson-dispatch-client.mts @@ -0,0 +1,336 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { + JETSON_DISPATCH_AUDIENCE, + JETSON_DISPATCH_TARGET, + parseJetsonDispatchRequest, +} from "./jetson-dispatch-contract.mts"; +import { + decodeJetsonArtifactArchive, + type JetsonDispatchArtifact, + type JetsonDispatchStatus, + MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, +} from "./jetson-dispatch-lifecycle.mts"; +import { writePrivateRegularFile } from "./private-file.mts"; + +const MAX_STATUS_BYTES = 64 * 1024; +const POLL_INTERVAL_MS = 10_000; +const MAX_WAIT_MS = 54 * 60_000; +const MAX_CONSECUTIVE_POLL_FAILURES = 3; +const OIDC_TOKEN_CACHE_MS = 4 * 60_000; +const JOB_ID_PATTERN = /^[a-f0-9]{64}$/u; + +function record(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${name} must be an object`); + } + return value as Record; +} + +function dispatcherBaseUrl(value: string | undefined): URL { + if (!value) throw new Error("JETSON_DISPATCH_URL is required"); + const url = new URL(value); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + (url.pathname !== "" && url.pathname !== "/") + ) { + throw new Error("JETSON_DISPATCH_URL must be an HTTPS origin without credentials or a path"); + } + url.pathname = "/"; + return url; +} + +export function createGitHubOidcTokenProvider( + options: { fetchImpl?: typeof fetch; now?: () => number } = {}, +): (env?: NodeJS.ProcessEnv) => Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const now = options.now ?? Date.now; + let cached: + | { expiresAtMs: number; requestToken: string; requestUrlValue: string; value: string } + | undefined; + return async (env: NodeJS.ProcessEnv = process.env): Promise => { + const requestUrlValue = env.ACTIONS_ID_TOKEN_REQUEST_URL; + const requestToken = env.ACTIONS_ID_TOKEN_REQUEST_TOKEN; + if (!requestUrlValue || !requestToken) { + throw new Error("GitHub OIDC environment is unavailable"); + } + if ( + cached && + cached.requestUrlValue === requestUrlValue && + cached.requestToken === requestToken && + now() < cached.expiresAtMs + ) { + return cached.value; + } + const requestUrl = new URL(requestUrlValue); + if (requestUrl.protocol !== "https:") { + throw new Error("GitHub OIDC request URL must use HTTPS"); + } + requestUrl.searchParams.set("audience", JETSON_DISPATCH_AUDIENCE); + const response = await fetchImpl(requestUrl, { + headers: { Authorization: `Bearer ${requestToken}` }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error(`GitHub OIDC token request returned HTTP ${response.status}`); + } + const payload = record(await response.json(), "GitHub OIDC token response"); + if ( + typeof payload.value !== "string" || + payload.value.length === 0 || + payload.value.length > 16 * 1024 + ) { + throw new Error("GitHub OIDC token response is invalid"); + } + cached = { + expiresAtMs: now() + OIDC_TOKEN_CACHE_MS, + requestToken, + requestUrlValue, + value: payload.value, + }; + return payload.value; + }; +} + +const githubOidcToken = createGitHubOidcTokenProvider(); + +async function dispatcherRequest(options: { + baseUrl: URL; + method: "DELETE" | "GET" | "POST"; + path: string; + body?: unknown; + maxBytes: number; +}): Promise { + const token = await githubOidcToken(); + const response = await fetch(new URL(options.path, options.baseUrl), { + method: options.method, + headers: { + Accept: "application/json", + Authorization: `Bearer ${token}`, + ...(options.body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + signal: AbortSignal.timeout(15_000), + }); + const contentLength = response.headers.get("content-length"); + if ( + contentLength && + /^[0-9]+$/u.test(contentLength) && + Number(contentLength) > options.maxBytes + ) { + throw new Error("Jetson dispatcher response is too large"); + } + if (!response.body) throw new Error("Jetson dispatcher returned an empty response"); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let responseBytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + responseBytes += value.length; + if (responseBytes > options.maxBytes) { + await reader.cancel(); + throw new Error("Jetson dispatcher response is too large"); + } + chunks.push(value); + } + const bytes = Buffer.concat(chunks, responseBytes); + let payload: unknown; + try { + payload = JSON.parse(bytes.toString("utf8")); + } catch { + throw new Error("Jetson dispatcher returned invalid JSON"); + } + if (!response.ok) { + const error = record(payload, "Jetson dispatcher error").error; + throw new Error( + `Jetson dispatcher returned HTTP ${response.status}: ${typeof error === "string" ? error.slice(0, 500) : "request failed"}`, + ); + } + return payload; +} + +function validateStatus(value: unknown): JetsonDispatchStatus { + const response = record(value, "Jetson dispatcher response"); + const job = record(response.job, "Jetson dispatch job"); + if ( + typeof job.jobId !== "string" || + !JOB_ID_PATTERN.test(job.jobId) || + !["queued", "running", "completed"].includes(String(job.state)) + ) { + throw new Error("Jetson dispatcher returned an invalid job status"); + } + return job as unknown as JetsonDispatchStatus; +} + +function validateArtifact(value: unknown, jobId: string): JetsonDispatchArtifact { + const artifact = record(value, "Jetson dispatch artifact"); + const status = record(artifact.status, "Jetson dispatch artifact status"); + if ( + status.jobId !== jobId || + status.state !== "completed" || + typeof status.conclusion !== "string" || + typeof artifact.log !== "string" + ) { + throw new Error("Jetson dispatcher returned an invalid artifact"); + } + if (artifact.artifactArchiveBase64 !== undefined) { + decodeJetsonArtifactArchive(artifact.artifactArchiveBase64); + } + if (status.conclusion === "success" && artifact.artifactArchiveBase64 === undefined) { + throw new Error("Successful Jetson dispatch did not return its E2E artifact archive"); + } + return artifact as unknown as JetsonDispatchArtifact; +} + +function delay(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +export async function pollJetsonDispatch(options: { + baseUrl: URL; + deadlineMs: number; + initialStatus: JetsonDispatchStatus; + jobId: string; + now?: () => number; + request?: typeof dispatcherRequest; + stopping?: () => boolean; + wait?: typeof delay; +}): Promise { + const now = options.now ?? Date.now; + const request = options.request ?? dispatcherRequest; + const wait = options.wait ?? delay; + let consecutiveFailures = 0; + let status = options.initialStatus; + while (status.state !== "completed") { + if (options.stopping?.()) throw new Error("Jetson dispatch cancellation requested"); + if (now() >= options.deadlineMs) { + await request({ + baseUrl: options.baseUrl, + method: "DELETE", + path: `v1/jobs/${options.jobId}`, + maxBytes: MAX_STATUS_BYTES, + }).catch(() => undefined); + throw new Error("Jetson dispatcher did not complete before the controller deadline"); + } + await wait(POLL_INTERVAL_MS); + try { + status = validateStatus( + await request({ + baseUrl: options.baseUrl, + method: "GET", + path: `v1/jobs/${options.jobId}`, + maxBytes: MAX_STATUS_BYTES, + }), + ); + consecutiveFailures = 0; + console.log(`Jetson dispatch state: ${status.state}`); + } catch (error) { + consecutiveFailures += 1; + if (consecutiveFailures >= MAX_CONSECUTIVE_POLL_FAILURES) { + await request({ + baseUrl: options.baseUrl, + method: "DELETE", + path: `v1/jobs/${options.jobId}`, + maxBytes: MAX_STATUS_BYTES, + }).catch(() => undefined); + throw error; + } + console.warn("Jetson dispatch status request failed; retrying"); + } + } + return status; +} + +async function main(): Promise { + const request = parseJetsonDispatchRequest({ + schemaVersion: 1, + target: JETSON_DISPATCH_TARGET, + candidateSha: process.env.JETSON_DISPATCH_CANDIDATE_SHA, + workflowRunId: process.env.GITHUB_RUN_ID, + workflowRunAttempt: Number(process.env.GITHUB_RUN_ATTEMPT), + }); + const baseUrl = dispatcherBaseUrl(process.env.JETSON_DISPATCH_URL); + const artifactDirectory = process.env.E2E_ARTIFACT_DIR ?? ""; + if (!path.isAbsolute(artifactDirectory)) throw new Error("E2E_ARTIFACT_DIR must be absolute"); + fs.mkdirSync(artifactDirectory, { recursive: true, mode: 0o700 }); + fs.chmodSync(artifactDirectory, 0o700); + + let jobId: string | undefined; + let stopping = false; + const cancel = (): void => { + if (stopping) return; + stopping = true; + if (!jobId) { + process.exitCode = 1; + return; + } + void dispatcherRequest({ + baseUrl, + method: "DELETE", + path: `v1/jobs/${jobId}`, + maxBytes: MAX_STATUS_BYTES, + }).finally(() => { + process.exitCode = 1; + }); + }; + process.on("SIGINT", cancel); + process.on("SIGTERM", cancel); + + const dispatched = validateStatus( + await dispatcherRequest({ + baseUrl, + method: "POST", + path: "v1/jobs", + body: request, + maxBytes: MAX_STATUS_BYTES, + }), + ); + jobId = dispatched.jobId; + console.log(`Jetson dispatch accepted as ${jobId}`); + const deadline = Date.now() + MAX_WAIT_MS; + await pollJetsonDispatch({ + baseUrl, + deadlineMs: deadline, + initialStatus: dispatched, + jobId, + stopping: () => stopping, + }); + + const artifactValue = await dispatcherRequest({ + baseUrl, + method: "GET", + path: `v1/jobs/${jobId}/artifact`, + maxBytes: MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, + }); + const artifact = validateArtifact(artifactValue, jobId); + const { artifactArchiveBase64, ...artifactReceipt } = artifact; + writePrivateRegularFile( + path.join(artifactDirectory, "jetson-dispatch.json"), + `${JSON.stringify(artifactReceipt, null, 2)}\n`, + ); + if (artifactArchiveBase64 !== undefined) { + writePrivateRegularFile( + path.join(artifactDirectory, "jetson-e2e-artifacts.tar.gz"), + decodeJetsonArtifactArchive(artifactArchiveBase64), + ); + } + console.log(`Jetson dispatch conclusion: ${artifact.status.conclusion}`); + if (artifact.status.conclusion !== "success") process.exitCode = 1; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : "Jetson dispatch client failed"); + process.exitCode = 1; + }); +} diff --git a/tools/e2e/jetson-dispatch-contract.mts b/tools/e2e/jetson-dispatch-contract.mts new file mode 100644 index 00000000000..7a8c668d19a --- /dev/null +++ b/tools/e2e/jetson-dispatch-contract.mts @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createPublicKey, createVerify, type JsonWebKeyInput, type KeyObject } from "node:crypto"; + +export const JETSON_DISPATCH_AUDIENCE = "nemoclaw-jetson-dispatch"; +export const JETSON_DISPATCH_TARGET = "jetson-nvmap-gpu"; +export const JETSON_DISPATCH_REPOSITORY = "NVIDIA/NemoClaw"; +export const JETSON_DISPATCH_WORKFLOW_REF = + "NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main"; + +const GITHUB_OIDC_ISSUER = "https://token.actions.githubusercontent.com"; +const GITHUB_OIDC_JWKS_URL = `${GITHUB_OIDC_ISSUER}/.well-known/jwks`; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const POSITIVE_INTEGER_PATTERN = /^[1-9][0-9]*$/u; +const MAX_TOKEN_BYTES = 16 * 1024; +const MAX_TOKEN_LIFETIME_SECONDS = 15 * 60; +const CLOCK_SKEW_SECONDS = 30; +const JWKS_CACHE_MS = 5 * 60_000; +const UNKNOWN_KEY_REFRESH_INTERVAL_MS = 60_000; + +export interface JetsonDispatchRequest { + schemaVersion: 1; + target: typeof JETSON_DISPATCH_TARGET; + candidateSha: string; + workflowRunId: string; + workflowRunAttempt: number; +} + +export interface GitHubOidcIdentity { + repository: typeof JETSON_DISPATCH_REPOSITORY; + repositoryId: string; + runId: string; + runAttempt: number; + workflowSha: string; + tokenId: string; +} + +export interface GitHubOidcPolicy { + repositoryId: string; +} + +export type SigningKeyResolver = (keyId: string) => Promise; + +function record(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${name} must be an object`); + } + return value as Record; +} + +function positiveInteger(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new Error(`${name} must be a positive safe integer`); + } + return value as number; +} + +function positiveIntegerString(value: unknown, name: string): string { + if (typeof value !== "string" || !POSITIVE_INTEGER_PATTERN.test(value)) { + throw new Error(`${name} must be a positive decimal integer`); + } + return value; +} + +export function parseJetsonDispatchRequest(value: unknown): JetsonDispatchRequest { + const request = record(value, "dispatch request"); + const expectedKeys = [ + "candidateSha", + "schemaVersion", + "target", + "workflowRunAttempt", + "workflowRunId", + ]; + const actualKeys = Object.keys(request).sort(); + if ( + actualKeys.length !== expectedKeys.length || + actualKeys.some((key, index) => key !== expectedKeys[index]) + ) { + throw new Error("dispatch request fields must match the fixed Jetson contract"); + } + if (request.schemaVersion !== 1) { + throw new Error("dispatch request schemaVersion must be 1"); + } + if (request.target !== JETSON_DISPATCH_TARGET) { + throw new Error(`dispatch target must be ${JETSON_DISPATCH_TARGET}`); + } + if (typeof request.candidateSha !== "string" || !SHA_PATTERN.test(request.candidateSha)) { + throw new Error("candidateSha must be a lowercase 40-character commit SHA"); + } + const workflowRunId = positiveIntegerString(request.workflowRunId, "workflowRunId"); + const workflowRunAttempt = positiveInteger(request.workflowRunAttempt, "workflowRunAttempt"); + return { + schemaVersion: 1, + target: JETSON_DISPATCH_TARGET, + candidateSha: request.candidateSha, + workflowRunId, + workflowRunAttempt, + }; +} + +function decodeJwtPart(encoded: string, name: string): Record { + if (!/^[A-Za-z0-9_-]+$/u.test(encoded)) throw new Error(`OIDC ${name} encoding is invalid`); + let value: unknown; + try { + value = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")); + } catch { + throw new Error(`OIDC ${name} is not valid JSON`); + } + return record(value, `OIDC ${name}`); +} + +function stringClaim(claims: Record, name: string): string { + const value = claims[name]; + if (typeof value !== "string" || value.length === 0) { + throw new Error(`OIDC ${name} claim is invalid`); + } + return value; +} + +function numericDateClaim(claims: Record, name: string): number { + const value = claims[name]; + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new Error(`OIDC ${name} claim is invalid`); + } + return value as number; +} + +export async function verifyGitHubOidcToken(options: { + token: string; + request: JetsonDispatchRequest; + policy: GitHubOidcPolicy; + resolveSigningKey: SigningKeyResolver; + nowMs?: number; +}): Promise { + if ( + typeof options.token !== "string" || + options.token.length === 0 || + Buffer.byteLength(options.token) > MAX_TOKEN_BYTES + ) { + throw new Error("OIDC token size is invalid"); + } + if (!POSITIVE_INTEGER_PATTERN.test(options.policy.repositoryId)) { + throw new Error("trusted repository ID must be a positive decimal integer"); + } + const parts = options.token.split("."); + if (parts.length !== 3 || parts.some((part) => part.length === 0)) { + throw new Error("OIDC token must be a compact JWT"); + } + const [encodedHeader, encodedClaims, encodedSignature] = parts as [string, string, string]; + const header = decodeJwtPart(encodedHeader, "header"); + if (header.alg !== "RS256" || header.typ !== "JWT") { + throw new Error("OIDC token must use an RS256 JWT header"); + } + const keyId = stringClaim(header, "kid"); + if (keyId.length > 256 || /[\u0000-\u001f\u007f]/u.test(keyId)) { + throw new Error("OIDC kid header is invalid"); + } + + const claims = decodeJwtPart(encodedClaims, "claims"); + if (!/^[A-Za-z0-9_-]+$/u.test(encodedSignature)) { + throw new Error("OIDC signature encoding is invalid"); + } + const signature = Buffer.from(encodedSignature, "base64url"); + const verifier = createVerify("RSA-SHA256"); + verifier.update(`${encodedHeader}.${encodedClaims}`, "ascii"); + verifier.end(); + const signingKey = await options.resolveSigningKey(keyId); + if (!verifier.verify(signingKey, signature)) { + throw new Error("OIDC signature verification failed"); + } + + const nowSeconds = Math.floor((options.nowMs ?? Date.now()) / 1000); + const issuedAt = numericDateClaim(claims, "iat"); + const notBefore = numericDateClaim(claims, "nbf"); + const expiresAt = numericDateClaim(claims, "exp"); + if ( + issuedAt > nowSeconds + CLOCK_SKEW_SECONDS || + notBefore > nowSeconds + CLOCK_SKEW_SECONDS || + expiresAt <= nowSeconds - CLOCK_SKEW_SECONDS || + expiresAt <= issuedAt || + expiresAt - issuedAt > MAX_TOKEN_LIFETIME_SECONDS + ) { + throw new Error("OIDC token is outside its allowed validity window"); + } + + const workflowSha = stringClaim(claims, "workflow_sha"); + const workflowRunId = positiveIntegerString(claims.run_id, "OIDC run_id claim"); + const workflowRunAttempt = positiveIntegerString(claims.run_attempt, "OIDC run_attempt claim"); + if ( + stringClaim(claims, "iss") !== GITHUB_OIDC_ISSUER || + stringClaim(claims, "aud") !== JETSON_DISPATCH_AUDIENCE || + stringClaim(claims, "repository") !== JETSON_DISPATCH_REPOSITORY || + stringClaim(claims, "repository_id") !== options.policy.repositoryId || + stringClaim(claims, "workflow_ref") !== JETSON_DISPATCH_WORKFLOW_REF || + stringClaim(claims, "ref") !== "refs/heads/main" || + stringClaim(claims, "event_name") !== "workflow_dispatch" || + stringClaim(claims, "runner_environment") !== "github-hosted" || + !SHA_PATTERN.test(workflowSha) || + stringClaim(claims, "sha") !== workflowSha || + workflowRunId !== options.request.workflowRunId || + Number(workflowRunAttempt) !== options.request.workflowRunAttempt + ) { + throw new Error("OIDC claims do not match the trusted Jetson controller"); + } + + return { + repository: JETSON_DISPATCH_REPOSITORY, + repositoryId: options.policy.repositoryId, + runId: workflowRunId, + runAttempt: options.request.workflowRunAttempt, + workflowSha, + tokenId: stringClaim(claims, "jti"), + }; +} + +type FetchLike = typeof fetch; + +export function createGitHubJwksResolver(fetchImpl: FetchLike = fetch): SigningKeyResolver { + let cachedAt = 0; + let cachedKeys = new Map(); + let refreshInFlight: Promise | undefined; + + async function refresh(): Promise { + const response = await fetchImpl(GITHUB_OIDC_JWKS_URL, { + headers: { Accept: "application/json" }, + signal: AbortSignal.timeout(5_000), + }); + if (!response.ok) throw new Error(`GitHub OIDC JWKS returned HTTP ${response.status}`); + const payload = record(await response.json(), "GitHub OIDC JWKS response"); + if (!Array.isArray(payload.keys) || payload.keys.length === 0 || payload.keys.length > 20) { + throw new Error("GitHub OIDC JWKS response has an invalid key set"); + } + const nextKeys = new Map(); + for (const value of payload.keys) { + const key = record(value, "GitHub OIDC JWK"); + if ( + key.kty !== "RSA" || + key.use !== "sig" || + key.alg !== "RS256" || + typeof key.kid !== "string" || + key.kid.length === 0 || + typeof key.n !== "string" || + typeof key.e !== "string" + ) { + continue; + } + nextKeys.set(key.kid, createPublicKey({ key, format: "jwk" } as JsonWebKeyInput)); + } + if (nextKeys.size === 0) throw new Error("GitHub OIDC JWKS contains no RS256 signing key"); + cachedKeys = nextKeys; + cachedAt = Date.now(); + } + + async function refreshOnce(): Promise { + refreshInFlight ??= refresh().finally(() => { + refreshInFlight = undefined; + }); + await refreshInFlight; + } + + return async (keyId) => { + const now = Date.now(); + const refreshedExpiredCache = cachedAt === 0 || now - cachedAt > JWKS_CACHE_MS; + if (refreshedExpiredCache) await refreshOnce(); + let key = cachedKeys.get(keyId); + if (!key && !refreshedExpiredCache && now - cachedAt > UNKNOWN_KEY_REFRESH_INTERVAL_MS) { + await refreshOnce(); + key = cachedKeys.get(keyId); + } + if (!key) throw new Error("OIDC signing key is unknown"); + return key; + }; +} diff --git a/tools/e2e/jetson-dispatch-lifecycle.mts b/tools/e2e/jetson-dispatch-lifecycle.mts new file mode 100644 index 00000000000..35cd03a0a79 --- /dev/null +++ b/tools/e2e/jetson-dispatch-lifecycle.mts @@ -0,0 +1,657 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { + type GitHubOidcIdentity, + type JetsonDispatchRequest, + parseJetsonDispatchRequest, +} from "./jetson-dispatch-contract.mts"; +import { + createPrivateRegularFile, + readPrivateRegularFile, + writePrivateRegularFile, +} from "./private-file.mts"; + +export const MAX_JETSON_DISPATCH_LOG_BYTES = 4 * 1024 * 1024; +const MAX_RETAINED_JOB_STATUSES = 128; +const MAX_JETSON_DISPATCH_STATUS_BYTES = 16 * 1024; +const JOB_ID_PATTERN = /^[a-f0-9]{64}$/u; +const STATUS_FILE_PATTERN = /^([a-f0-9]{64})\.json$/u; +export const MAX_JETSON_ARTIFACT_ARCHIVE_BYTES = 1024 * 1024; +const MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS = + Math.ceil(MAX_JETSON_ARTIFACT_ARCHIVE_BYTES / 3) * 4; +const MAX_JETSON_DISPATCH_ARTIFACT_JSON_OVERHEAD_BYTES = 256 * 1024; +export const MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES = + MAX_JETSON_DISPATCH_LOG_BYTES * 6 + + MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS + + MAX_JETSON_DISPATCH_ARTIFACT_JSON_OVERHEAD_BYTES; + +export type JetsonDispatchConclusion = + | "cancelled" + | "cleanup-failed" + | "failure" + | "success" + | "timed-out"; + +export interface JetsonDeviceIdentity { + model: string; + jetpackVersion: string; + jetsonLinuxRelease: string; + kernel: string; +} + +export interface JetsonWorkerResult { + device: JetsonDeviceIdentity; + log: string; + artifactArchiveBase64?: string; +} + +export interface JetsonDispatchWorker { + run( + request: JetsonDispatchRequest, + options: { jobId: string; signal: AbortSignal }, + ): Promise; + cleanup(options: { jobId: string; signal: AbortSignal }): Promise; +} + +export interface JetsonDispatchStatus { + schemaVersion: 1; + jobId: string; + request: JetsonDispatchRequest; + state: "queued" | "running" | "completed"; + conclusion?: JetsonDispatchConclusion; + createdAt: string; + startedAt?: string; + completedAt?: string; + device?: JetsonDeviceIdentity; + cleanup: "pending" | "succeeded" | "failed"; + error?: string; +} + +export interface JetsonDispatchArtifact { + status: JetsonDispatchStatus; + log: string; + artifactArchiveBase64?: string; +} + +export class JetsonDispatchBusyError extends Error {} + +export class JetsonDispatchNotFoundError extends Error {} + +function safeError(error: unknown): string { + const message = error instanceof Error ? error.message : "Jetson dispatch failed"; + return message.replace(/[\u0000-\u001f\u007f]/gu, " ").slice(0, 500); +} + +function appendBoundedError(previous: string | undefined, newest: string): string { + const latest = newest.replace(/[\u0000-\u001f\u007f]/gu, " ").slice(0, 500); + if (!previous) return latest; + const separator = "; "; + const available = 500 - separator.length - latest.length; + if (available <= 0) return latest; + const earlier = previous.replace(/[\u0000-\u001f\u007f]/gu, " "); + const boundedEarlier = + earlier.length <= available + ? earlier + : available > 3 + ? `${earlier.slice(0, available - 3)}...` + : earlier.slice(0, available); + return `${boundedEarlier}${separator}${latest}`; +} + +function workerErrorLog(error: unknown): string | undefined { + if (!error || typeof error !== "object" || !("log" in error)) return undefined; + const log = (error as { log?: unknown }).log; + if (typeof log !== "string" || Buffer.byteLength(log) > MAX_JETSON_DISPATCH_LOG_BYTES) { + return undefined; + } + return log; +} + +export function decodeJetsonArtifactArchive(value: unknown): Buffer { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value) + ) { + throw new Error("Jetson artifact archive must be bounded canonical base64"); + } + const decoded = Buffer.from(value, "base64"); + if (decoded.length > MAX_JETSON_ARTIFACT_ARCHIVE_BYTES || decoded.toString("base64") !== value) { + throw new Error("Jetson artifact archive must be bounded canonical base64"); + } + return decoded; +} + +function workerErrorArtifactArchive(error: unknown): string | undefined { + if (!error || typeof error !== "object" || !("artifactArchiveBase64" in error)) { + return undefined; + } + const value = (error as { artifactArchiveBase64?: unknown }).artifactArchiveBase64; + try { + decodeJetsonArtifactArchive(value); + return value as string; + } catch { + return undefined; + } +} + +function ensurePrivateDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new Error(`${directory} must be a private directory`); + } + const currentUid = process.getuid?.(); + if (currentUid !== undefined && stat.uid !== currentUid) { + throw new Error(`${directory} must be owned by the dispatcher user`); + } + fs.chmodSync(directory, 0o700); +} + +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync( + directory, + fs.constants.O_RDONLY | (fs.constants.O_DIRECTORY ?? 0) | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + if (!fs.fstatSync(descriptor).isDirectory()) { + throw new Error(`${directory} must be a directory`); + } + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function dispatchJobId(request: JetsonDispatchRequest): string { + return createHash("sha256") + .update( + `${request.schemaVersion}:${request.target}:${request.candidateSha}:${request.workflowRunId}:${request.workflowRunAttempt}`, + "utf8", + ) + .digest("hex"); +} + +function parseTimestamp(value: unknown, name: string): string { + if ( + typeof value !== "string" || + value.length !== 24 || + Number.isNaN(Date.parse(value)) || + new Date(value).toISOString() !== value + ) { + throw new Error(`${name} must be an ISO timestamp`); + } + return value; +} + +function parseDeviceIdentity(value: unknown): JetsonDeviceIdentity | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("device must be an object"); + } + const device = value as Record; + const fields = ["jetpackVersion", "jetsonLinuxRelease", "kernel", "model"]; + if (fields.some((field) => typeof device[field] !== "string")) { + throw new Error("device fields are invalid"); + } + return { + model: device.model as string, + jetpackVersion: device.jetpackVersion as string, + jetsonLinuxRelease: device.jetsonLinuxRelease as string, + kernel: device.kernel as string, + }; +} + +function parsePersistedStatus(contents: string, expectedJobId: string): JetsonDispatchStatus { + let value: unknown; + try { + value = JSON.parse(contents); + } catch { + throw new Error("persisted Jetson dispatch status is not valid JSON"); + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("persisted Jetson dispatch status must be an object"); + } + const status = value as Record; + const request = parseJetsonDispatchRequest(status.request); + if ( + status.schemaVersion !== 1 || + status.jobId !== expectedJobId || + dispatchJobId(request) !== expectedJobId + ) { + throw new Error("persisted Jetson dispatch status does not match its job ID"); + } + const createdAt = parseTimestamp(status.createdAt, "createdAt"); + if (status.state === "queued" || status.state === "running") { + if (status.cleanup !== "pending") { + throw new Error("persisted incomplete Jetson dispatch cleanup is invalid"); + } + return { + schemaVersion: 1, + jobId: expectedJobId, + request, + state: status.state, + createdAt, + ...(status.state === "running" + ? { startedAt: parseTimestamp(status.startedAt, "startedAt") } + : {}), + cleanup: "pending", + }; + } + if (status.state !== "completed") { + throw new Error("persisted Jetson dispatch status has an invalid state"); + } + if ( + !["cancelled", "cleanup-failed", "failure", "success", "timed-out"].includes( + status.conclusion as string, + ) || + !["failed", "succeeded"].includes(status.cleanup as string) || + (status.error !== undefined && typeof status.error !== "string") + ) { + throw new Error("persisted Jetson dispatch result fields are invalid"); + } + const startedAt = parseTimestamp(status.startedAt, "startedAt"); + const completedAt = parseTimestamp(status.completedAt, "completedAt"); + const device = parseDeviceIdentity(status.device); + return { + schemaVersion: 1, + jobId: expectedJobId, + request, + state: "completed", + conclusion: status.conclusion as JetsonDispatchConclusion, + createdAt, + startedAt, + completedAt, + ...(device === undefined ? {} : { device }), + cleanup: status.cleanup as "failed" | "succeeded", + ...(status.error === undefined ? {} : { error: status.error as string }), + }; +} + +function withTimeout( + timeoutMs: number, + operation: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error("operation timed out")), timeoutMs); + timer.unref(); + return operation(controller.signal).finally(() => clearTimeout(timer)); +} + +export class JetsonDispatchCoordinator { + readonly #stateDirectory: string; + readonly #lockPath: string; + readonly #worker: JetsonDispatchWorker; + readonly #executionTimeoutMs: number; + readonly #cleanupTimeoutMs: number; + readonly #jobs = new Map(); + #activeJobId: string | undefined; + #activeAbort: AbortController | undefined; + #activeRun: Promise | undefined; + #cancelRequested = false; + #initialized = false; + #lockRecoveryRequired = false; + + constructor(options: { + stateDirectory: string; + worker: JetsonDispatchWorker; + executionTimeoutMs: number; + cleanupTimeoutMs: number; + }) { + if (!path.isAbsolute(options.stateDirectory)) { + throw new Error("Jetson dispatcher state directory must be absolute"); + } + if ( + !Number.isSafeInteger(options.executionTimeoutMs) || + options.executionTimeoutMs < 60_000 || + options.executionTimeoutMs > 55 * 60_000 + ) { + throw new Error("Jetson execution timeout must be between 1 and 55 minutes"); + } + if ( + !Number.isSafeInteger(options.cleanupTimeoutMs) || + options.cleanupTimeoutMs < 10_000 || + options.cleanupTimeoutMs > 10 * 60_000 + ) { + throw new Error("Jetson cleanup timeout must be between 10 seconds and 10 minutes"); + } + this.#stateDirectory = options.stateDirectory; + this.#lockPath = path.join(options.stateDirectory, "device.lock"); + this.#worker = options.worker; + this.#executionTimeoutMs = options.executionTimeoutMs; + this.#cleanupTimeoutMs = options.cleanupTimeoutMs; + } + + async initialize(): Promise { + if (this.#initialized) return; + ensurePrivateDirectory(this.#stateDirectory); + const staleLock = readPrivateRegularFile(this.#lockPath, { + allowMissing: true, + maxBytes: 1024, + }); + if (staleLock !== null) { + if (!/^[a-f0-9]{64}\n?$/u.test(staleLock)) { + throw new Error("Jetson device lock contains an invalid job ID"); + } + const jobId = staleLock.endsWith("\n") ? staleLock.slice(0, -1) : staleLock; + await withTimeout(this.#cleanupTimeoutMs, (signal) => + this.#worker.cleanup({ jobId, signal }), + ); + const recoveredStatus = this.#readStatus(jobId); + if (recoveredStatus && recoveredStatus.state !== "completed") { + recoveredStatus.state = "completed"; + recoveredStatus.conclusion = "failure"; + recoveredStatus.startedAt ??= recoveredStatus.createdAt; + recoveredStatus.completedAt = new Date().toISOString(); + recoveredStatus.cleanup = "succeeded"; + recoveredStatus.error = "Jetson dispatcher restarted before terminal status was persisted"; + this.#persist(recoveredStatus); + } + this.#removeDeviceLock(jobId); + } + this.#restoreCompletedStatuses(); + this.#initialized = true; + } + + dispatch(request: JetsonDispatchRequest, identity: GitHubOidcIdentity): JetsonDispatchStatus { + this.#requireInitialized(); + if (this.#lockRecoveryRequired) { + throw new JetsonDispatchBusyError("Jetson device lock requires recovery"); + } + if ( + identity.runId !== request.workflowRunId || + identity.runAttempt !== request.workflowRunAttempt + ) { + throw new Error("dispatch identity does not match the workflow run"); + } + const jobId = dispatchJobId(request); + const existing = this.#findStatus(jobId); + if (existing) return structuredClone(existing); + if (this.#activeJobId) { + throw new JetsonDispatchBusyError("Jetson device is already running another job"); + } + this.#evictOldestCompletedJob(); + + try { + createPrivateRegularFile(this.#lockPath, `${jobId}\n`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new JetsonDispatchBusyError("Jetson device lock requires recovery"); + } + throw error; + } + const status: JetsonDispatchStatus = { + schemaVersion: 1, + jobId, + request: structuredClone(request), + state: "queued", + createdAt: new Date().toISOString(), + cleanup: "pending", + }; + this.#clearPriorResultFiles(jobId); + this.#persist(status); + fsyncDirectory(this.#stateDirectory); + this.#jobs.set(jobId, status); + this.#activeJobId = jobId; + this.#activeRun = this.#run(status); + return structuredClone(status); + } + + status(jobId: string): JetsonDispatchStatus { + this.#requireInitialized(); + const status = this.#findStatus(jobId); + if (!status) throw new JetsonDispatchNotFoundError("Jetson dispatch job was not found"); + return structuredClone(status); + } + + request(jobId: string): JetsonDispatchRequest { + return this.status(jobId).request; + } + + cancel(jobId: string): JetsonDispatchStatus { + this.#requireInitialized(); + const status = this.#findStatus(jobId); + if (!status) throw new JetsonDispatchNotFoundError("Jetson dispatch job was not found"); + if (status.state !== "completed" && this.#activeJobId === jobId) { + this.#cancelRequested = true; + this.#activeAbort?.abort(new Error("dispatch cancelled")); + } + return structuredClone(status); + } + + async shutdown(): Promise { + if (this.#activeJobId) this.cancel(this.#activeJobId); + await this.#activeRun; + if (this.#lockRecoveryRequired) { + throw new Error("Jetson device lock state requires operator recovery"); + } + if ( + readPrivateRegularFile(this.#lockPath, { + allowMissing: true, + maxBytes: 1024, + }) !== null + ) { + throw new Error("Jetson device lock still requires cleanup recovery"); + } + } + + artifact(jobId: string): JetsonDispatchArtifact { + const status = this.status(jobId); + if (status.state !== "completed") + throw new JetsonDispatchBusyError("Jetson job is not complete"); + const log = + readPrivateRegularFile(this.#logPath(jobId), { + allowMissing: true, + maxBytes: MAX_JETSON_DISPATCH_LOG_BYTES, + }) ?? ""; + const artifactArchiveBase64 = readPrivateRegularFile(this.#artifactPath(jobId), { + allowMissing: true, + maxBytes: MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS, + }); + if (artifactArchiveBase64 !== null) decodeJetsonArtifactArchive(artifactArchiveBase64); + return { + status, + log, + ...(artifactArchiveBase64 === null ? {} : { artifactArchiveBase64 }), + }; + } + + async #run(status: JetsonDispatchStatus): Promise { + const controller = new AbortController(); + this.#activeAbort = controller; + this.#cancelRequested = false; + let timedOut = false; + let conclusion: JetsonDispatchConclusion = "failure"; + const timer = setTimeout(() => { + timedOut = true; + controller.abort(new Error("Jetson execution timed out")); + }, this.#executionTimeoutMs); + timer.unref(); + + try { + status.state = "running"; + status.startedAt = new Date().toISOString(); + this.#persist(status); + const result = await this.#worker.run(status.request, { + jobId: status.jobId, + signal: controller.signal, + }); + if (Buffer.byteLength(result.log) > MAX_JETSON_DISPATCH_LOG_BYTES) { + throw new Error(`Jetson log exceeds ${MAX_JETSON_DISPATCH_LOG_BYTES} bytes`); + } + writePrivateRegularFile(this.#logPath(status.jobId), result.log); + if (result.artifactArchiveBase64 !== undefined) { + decodeJetsonArtifactArchive(result.artifactArchiveBase64); + writePrivateRegularFile(this.#artifactPath(status.jobId), result.artifactArchiveBase64); + } + status.device = result.device; + conclusion = "success"; + } catch (error) { + status.error = safeError(error); + conclusion = timedOut ? "timed-out" : this.#cancelRequested ? "cancelled" : "failure"; + try { + writePrivateRegularFile(this.#logPath(status.jobId), workerErrorLog(error) ?? ""); + const artifactArchiveBase64 = workerErrorArtifactArchive(error); + if (artifactArchiveBase64 !== undefined) { + writePrivateRegularFile(this.#artifactPath(status.jobId), artifactArchiveBase64); + } + } catch (resultError) { + status.error = appendBoundedError( + status.error, + `Result persistence failed: ${safeError(resultError)}`, + ); + } + } finally { + clearTimeout(timer); + try { + await withTimeout(this.#cleanupTimeoutMs, (signal) => + this.#worker.cleanup({ jobId: status.jobId, signal }), + ); + status.cleanup = "succeeded"; + } catch (error) { + status.cleanup = "failed"; + const cleanupError = `Jetson cleanup failed: ${safeError(error)}`; + status.error = appendBoundedError(status.error, cleanupError); + conclusion = "cleanup-failed"; + } + status.conclusion = conclusion; + status.state = "completed"; + status.completedAt = new Date().toISOString(); + let terminalStatusPersisted = false; + try { + this.#persist(status); + terminalStatusPersisted = true; + } catch (error) { + const persistenceError = `Final status persistence failed: ${safeError(error)}`; + status.error = appendBoundedError(status.error, persistenceError); + if (status.conclusion !== "cleanup-failed") status.conclusion = "failure"; + } + if (status.cleanup === "succeeded" && terminalStatusPersisted) { + try { + this.#removeDeviceLock(status.jobId); + } catch (error) { + const lockError = `Jetson lock removal failed: ${safeError(error)}`; + status.error = appendBoundedError(status.error, lockError); + status.conclusion = "cleanup-failed"; + try { + this.#persist(status); + } catch (persistError) { + const persistenceError = `Lock failure persistence failed: ${safeError(persistError)}`; + status.error = appendBoundedError(status.error, persistenceError); + } + } + } + this.#activeAbort = undefined; + this.#activeRun = undefined; + this.#activeJobId = undefined; + this.#cancelRequested = false; + } + } + + #persist(status: JetsonDispatchStatus): void { + writePrivateRegularFile(this.#statusPath(status.jobId), `${JSON.stringify(status, null, 2)}\n`); + } + + #removeDeviceLock(jobId: string): void { + fs.unlinkSync(this.#lockPath); + try { + fsyncDirectory(this.#stateDirectory); + } catch (error) { + try { + this.#restoreDeviceLock(jobId); + } catch (restoreError) { + this.#lockRecoveryRequired = true; + throw new Error( + `Jetson lock directory persistence failed: ${safeError(error)}; lock restoration failed: ${safeError(restoreError)}`, + ); + } + throw error; + } + } + + #restoreDeviceLock(jobId: string): void { + try { + createPrivateRegularFile(this.#lockPath, `${jobId}\n`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const existing = readPrivateRegularFile(this.#lockPath, { maxBytes: 1024 }); + if (existing !== `${jobId}\n`) { + throw new Error("Jetson device lock changed while durability was restored"); + } + } + fsyncDirectory(this.#stateDirectory); + } + + #statusPath(jobId: string): string { + return path.join(this.#stateDirectory, `${jobId}.json`); + } + + #logPath(jobId: string): string { + return path.join(this.#stateDirectory, `${jobId}.log`); + } + + #artifactPath(jobId: string): string { + return path.join(this.#stateDirectory, `${jobId}.artifacts.tar.gz.b64`); + } + + #clearPriorResultFiles(jobId: string): void { + fs.rmSync(this.#logPath(jobId), { force: true }); + fs.rmSync(this.#artifactPath(jobId), { force: true }); + } + + #findStatus(jobId: string): JetsonDispatchStatus | undefined { + const existing = this.#jobs.get(jobId); + if (existing || !JOB_ID_PATTERN.test(jobId)) return existing; + const restored = this.#readStatus(jobId); + if (restored?.state === "completed") { + this.#evictOldestCompletedJob(); + this.#jobs.set(jobId, restored); + } + return restored?.state === "completed" ? restored : undefined; + } + + #readStatus(jobId: string): JetsonDispatchStatus | null { + try { + const contents = readPrivateRegularFile(this.#statusPath(jobId), { + allowMissing: true, + maxBytes: MAX_JETSON_DISPATCH_STATUS_BYTES, + }); + return contents === null ? null : parsePersistedStatus(contents, jobId); + } catch (error) { + throw new Error(`Jetson dispatch status ${jobId}.json is invalid: ${safeError(error)}`); + } + } + + #restoreCompletedStatuses(): void { + const completed = fs + .readdirSync(this.#stateDirectory) + .flatMap((name) => { + const match = STATUS_FILE_PATTERN.exec(name); + if (!match) return []; + const status = this.#readStatus(match[1]!); + return status?.state === "completed" ? [status] : []; + }) + .sort((left, right) => left.completedAt!.localeCompare(right.completedAt!)) + .slice(-MAX_RETAINED_JOB_STATUSES); + for (const status of completed) this.#jobs.set(status.jobId, status); + } + + #evictOldestCompletedJob(): void { + if (this.#jobs.size < MAX_RETAINED_JOB_STATUSES) return; + for (const [jobId, status] of this.#jobs) { + if (status.state === "completed") { + this.#jobs.delete(jobId); + return; + } + } + } + + #requireInitialized(): void { + if (!this.#initialized) throw new Error("Jetson dispatcher is not initialized"); + } +} diff --git a/tools/e2e/jetson-dispatch-service.mts b/tools/e2e/jetson-dispatch-service.mts new file mode 100644 index 00000000000..738a71fc126 --- /dev/null +++ b/tools/e2e/jetson-dispatch-service.mts @@ -0,0 +1,261 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { + createGitHubJwksResolver, + type GitHubOidcIdentity, + parseJetsonDispatchRequest, + type SigningKeyResolver, + verifyGitHubOidcToken, +} from "./jetson-dispatch-contract.mts"; +import { + JetsonDispatchBusyError, + JetsonDispatchCoordinator, + JetsonDispatchNotFoundError, +} from "./jetson-dispatch-lifecycle.mts"; +import { loadSshJetsonWorkerConfig, SshJetsonDispatchWorker } from "./jetson-dispatch-worker.mts"; + +const MAX_REQUEST_BYTES = 8 * 1024; +const JOB_PATH_PATTERN = /^\/v1\/jobs\/([a-f0-9]{64})(?:\/(artifact))?$/u; + +function positiveIntegerEnvironment( + value: string | undefined, + name: string, + minimum: number, + maximum: number, +): number { + if (!value || !/^[1-9][0-9]*$/u.test(value)) { + throw new Error(`${name} must be a positive integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(`${name} must be between ${minimum} and ${maximum}`); + } + return parsed; +} + +function bearerToken(request: IncomingMessage): string { + const authorization = request.headers.authorization; + const match = + typeof authorization === "string" ? /^Bearer ([A-Za-z0-9._-]+)$/u.exec(authorization) : null; + if (!match) throw new Error("request requires a bearer token"); + return match[1]!; +} + +async function readJsonBody(request: IncomingMessage): Promise { + if (request.headers["content-type"]?.split(";", 1)[0]?.trim() !== "application/json") { + throw new Error("request Content-Type must be application/json"); + } + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > MAX_REQUEST_BYTES) throw new Error("request body is too large"); + chunks.push(buffer); + } + if (bytes === 0) throw new Error("request body is required"); + try { + return JSON.parse(Buffer.concat(chunks).toString("utf8")); + } catch { + throw new Error("request body must be valid JSON"); + } +} + +function sendJson(response: ServerResponse, status: number, value: unknown): void { + const body = `${JSON.stringify(value)}\n`; + response.writeHead(status, { + "Cache-Control": "no-store", + "Content-Length": Buffer.byteLength(body), + "Content-Type": "application/json; charset=utf-8", + "X-Content-Type-Options": "nosniff", + }); + response.end(body); +} + +export function createJetsonDispatchServer(options: { + coordinator: JetsonDispatchCoordinator; + repositoryId: string; + resolveSigningKey: SigningKeyResolver; +}): Server { + async function authenticate( + token: string, + dispatchRequest: ReturnType, + ): Promise { + return verifyGitHubOidcToken({ + token, + request: dispatchRequest, + policy: { repositoryId: options.repositoryId }, + resolveSigningKey: options.resolveSigningKey, + }); + } + + return createServer((request, response) => { + void (async () => { + if (!request.url) { + sendJson(response, 404, { error: "not found" }); + return; + } + let token: string; + try { + token = bearerToken(request); + } catch { + sendJson(response, 401, { error: "request authentication failed" }); + return; + } + const createJob = request.method === "POST" && request.url === "/v1/jobs"; + const match = JOB_PATH_PATTERN.exec(request.url); + if (!createJob && (!match || !["DELETE", "GET"].includes(request.method ?? ""))) { + sendJson(response, 404, { error: "not found" }); + return; + } + + let dispatchRequest; + let jobId: string | undefined; + let artifact = false; + if (createJob) { + try { + dispatchRequest = parseJetsonDispatchRequest(await readJsonBody(request)); + } catch (error) { + sendJson(response, 400, { + error: error instanceof Error ? error.message : "invalid request", + }); + return; + } + } else { + jobId = match![1]!; + artifact = match![2] === "artifact"; + try { + dispatchRequest = options.coordinator.request(jobId); + } catch { + sendJson(response, 401, { error: "request authentication failed" }); + return; + } + } + + let identity; + try { + identity = await authenticate(token, dispatchRequest); + } catch { + sendJson(response, 401, { error: "request authentication failed" }); + return; + } + if (createJob) { + sendJson(response, 202, { job: options.coordinator.dispatch(dispatchRequest, identity) }); + return; + } + + const existingJobId = jobId!; + if (request.method === "DELETE") { + if (artifact) { + sendJson(response, 405, { error: "method not allowed" }); + return; + } + sendJson(response, 202, { job: options.coordinator.cancel(existingJobId) }); + return; + } + sendJson( + response, + 200, + artifact + ? options.coordinator.artifact(existingJobId) + : { job: options.coordinator.status(existingJobId) }, + ); + })().catch((error) => { + if (response.headersSent) { + response.destroy(); + return; + } + if (error instanceof JetsonDispatchBusyError) { + sendJson(response, 409, { error: error.message }); + return; + } + if (error instanceof JetsonDispatchNotFoundError) { + sendJson(response, 404, { error: error.message }); + return; + } + console.error("Jetson dispatch service failed"); + sendJson(response, 500, { error: "Jetson dispatch service failed" }); + }); + }); +} + +async function main(): Promise { + const stateDirectory = process.env.JETSON_DISPATCH_STATE_DIRECTORY ?? ""; + if (!path.isAbsolute(stateDirectory)) { + throw new Error("JETSON_DISPATCH_STATE_DIRECTORY must be absolute"); + } + const repositoryId = process.env.JETSON_DISPATCH_GITHUB_REPOSITORY_ID ?? ""; + if (!/^[1-9][0-9]*$/u.test(repositoryId)) { + throw new Error("JETSON_DISPATCH_GITHUB_REPOSITORY_ID must be a positive integer"); + } + const executionTimeoutSeconds = positiveIntegerEnvironment( + process.env.JETSON_DISPATCH_EXECUTION_TIMEOUT_SECONDS, + "JETSON_DISPATCH_EXECUTION_TIMEOUT_SECONDS", + 120, + 55 * 60, + ); + const cleanupTimeoutSeconds = positiveIntegerEnvironment( + process.env.JETSON_DISPATCH_CLEANUP_TIMEOUT_SECONDS, + "JETSON_DISPATCH_CLEANUP_TIMEOUT_SECONDS", + 10, + 10 * 60, + ); + const workerConfig = loadSshJetsonWorkerConfig({ stateDirectory }); + if (executionTimeoutSeconds < workerConfig.testTimeoutSeconds + 30) { + throw new Error("Jetson execution timeout must exceed the remote test timeout by 30 seconds"); + } + const coordinator = new JetsonDispatchCoordinator({ + stateDirectory, + worker: new SshJetsonDispatchWorker(workerConfig), + executionTimeoutMs: executionTimeoutSeconds * 1_000, + cleanupTimeoutMs: cleanupTimeoutSeconds * 1_000, + }); + await coordinator.initialize(); + const server = createJetsonDispatchServer({ + coordinator, + repositoryId, + resolveSigningKey: createGitHubJwksResolver(), + }); + server.headersTimeout = 10_000; + server.requestTimeout = 15_000; + server.keepAliveTimeout = 5_000; + const port = positiveIntegerEnvironment( + process.env.JETSON_DISPATCH_PORT ?? "8787", + "JETSON_DISPATCH_PORT", + 1, + 65_535, + ); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolve); + }); + console.log(`Jetson dispatch service listening on 127.0.0.1:${port}`); + + let stopping = false; + const stop = (): void => { + if (stopping) return; + stopping = true; + void new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }) + .then(() => coordinator.shutdown()) + .catch((error) => { + console.error(error instanceof Error ? error.message : "Jetson dispatch shutdown failed"); + process.exitCode = 1; + }); + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : "Jetson dispatch service failed"); + process.exitCode = 1; + }); +} diff --git a/tools/e2e/jetson-dispatch-worker.mts b/tools/e2e/jetson-dispatch-worker.mts new file mode 100644 index 00000000000..d0c517bc556 --- /dev/null +++ b/tools/e2e/jetson-dispatch-worker.mts @@ -0,0 +1,1056 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import type { JetsonDispatchRequest } from "./jetson-dispatch-contract.mts"; +import type { + JetsonDeviceIdentity, + JetsonDispatchWorker, + JetsonWorkerResult, +} from "./jetson-dispatch-lifecycle.mts"; +import { + decodeJetsonArtifactArchive, + MAX_JETSON_ARTIFACT_ARCHIVE_BYTES, +} from "./jetson-dispatch-lifecycle.mts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.mts"; + +const MAX_PROCESS_OUTPUT_BYTES = 4 * 1024 * 1024; +const JETSON_SSH_DESTINATION = "nvidia@192.168.55.1"; +const JETSON_CLEANUP_EXECUTABLE = "/usr/local/libexec/nemoclaw-jetson-cleanup"; +const JETSON_CLEANUP_TARGET = + "/opt/nemoclaw-jetson-dispatch/current/tools/e2e/jetson-dispatch-cleanup.sh"; +const SAFE_PROCESS_ENV = { + LANG: "C.UTF-8", + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +}; +const MAX_BASELINE_OUTPUT_BYTES = 4 * 1024; +const MAX_BASELINE_RECORD_BYTES = 8 * 1024; +const MAX_OLLAMA_MODELS = 64; +const MAX_OLLAMA_MODELS_BYTES = 3 * 1024; +const MAX_CLEANUP_EVIDENCE_BYTES = 64 * 1024; +const CLEANUP_EVIDENCE_BEGIN = "nemoclaw-cleanup-evidence-v1-begin"; +const CLEANUP_EVIDENCE_END = "nemoclaw-cleanup-evidence-v1-end"; + +interface JetsonCleanupEvidence { + schemaVersion: 1; + volumes: string[]; + processIds: number[]; +} + +const DEVICE_IDENTITY_SCRIPT = String.raw`set -eu +clean_line() { tr -d '\000\r\n\t'; } +printf 'model\t' +if [ -r /proc/device-tree/model ]; then clean_line /dev/null 2>&1; then + jetpack_version="$(dpkg-query -W -f='${"${Version}"}' nvidia-jetpack 2>/dev/null || true)" + if [ -n "$jetpack_version" ]; then printf '%s' "$jetpack_version" | clean_line; else printf 'unavailable'; fi +else + printf 'unavailable' +fi +printf '\njetsonLinuxRelease\t' +if [ -r /etc/nv_tegra_release ]; then sed -n '1p' /etc/nv_tegra_release | clean_line; else printf 'unavailable'; fi +printf '\nkernel\t' +uname -r | clean_line +printf '\n' +`; + +const PRESERVED_BASELINE_SCRIPT = String.raw`set -euo pipefail +export LC_ALL=C +clean_line() { tr -d '\000\r\n\t'; } +node_path="$(command -v node)" +npm_path="$(command -v npm)" +ollama_path="$(command -v ollama)" +for openshell_component in openshell openshell-gateway openshell-sandbox; do + if command -v "$openshell_component" >/dev/null 2>&1; then + echo "Jetson dispatch requires OpenShell to be absent before the job" >&2 + exit 1 + fi + for host_bin in "/usr/local/bin/$openshell_component" "/usr/bin/$openshell_component" "$HOME/.local/bin/$openshell_component"; do + if [ -e "$host_bin" ] || [ -L "$host_bin" ]; then + echo "Jetson dispatch requires host-level OpenShell binaries to be absent" >&2 + exit 1 + fi + done +done +node_version="$(node --version)" +node -e ' + const [major, minor] = process.versions.node.split(".").map(Number); + if (major < 22 || (major === 22 && minor < 19)) process.exit(1); +' +npm_version="$(npm --version)" +npm_major="${"${npm_version%%.*}"}" +[[ "$npm_major" =~ ^[0-9]+$ ]] && (( npm_major >= 10 )) +test -c /dev/nvmap +docker_runtimes="$(docker info --format '{{json .Runtimes}}')" +case "$docker_runtimes" in + *nvidia*) ;; + *) echo "Docker does not report the NVIDIA runtime" >&2; exit 1 ;; +esac +ollama_models_base64="$( + ollama list | + awk 'NR == 1 { if ($1 != "NAME" || $2 != "ID") exit 1; next } { if (NF < 2) exit 1; print $1 "\t" $2 } END { if (NR == 0) exit 1 }' | + LC_ALL=C sort | + base64 --wrap=0 +)" +test_owned_images="$(docker image ls nemoclaw-sandbox-local --format '{{.Repository}}\t{{.Tag}}' | awk '$1 == "nemoclaw-sandbox-local" && index($2, "e2e-jetson-nvmap-") == 1 { print $1 ":" $2 }')" +[ -z "$test_owned_images" ] || { echo "A test-owned Jetson image remains before dispatch" >&2; exit 1; } +printf 'nodePath\t'; printf '%s' "$node_path" | clean_line +printf '\nnodeVersion\t'; printf '%s' "$node_version" | clean_line +printf '\nnpmPath\t'; printf '%s' "$npm_path" | clean_line +printf '\nnpmVersion\t'; printf '%s' "$npm_version" | clean_line +printf '\nollamaPath\t'; printf '%s' "$ollama_path" | clean_line +printf '\nollamaModelsBase64\t'; printf '%s' "$ollama_models_base64" | clean_line +printf '\nopenshellState\tabsent' +printf '\n' +`; + +const CLEANUP_VERIFICATION_SCRIPT = + String.raw`set -euo pipefail +job_id="$1" +cleanup_evidence_base64="$2" +[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] +workspace="/var/tmp/nemoclaw-jetson-e2e/$job_id" +job_home="$workspace/home" +sandbox_name=e2e-jetson-nvmap +gateway_name=nemoclaw +recorded_volumes=() +recorded_process_ids=() +cleanup_rows="$( + printf '%s' "$cleanup_evidence_base64" | base64 --decode | node -e ' + const fs = require("node:fs"); + const value = JSON.parse(fs.readFileSync(0, "utf8")); + if (value.schemaVersion !== 1 || !Array.isArray(value.volumes) || !Array.isArray(value.processIds)) process.exit(1); + for (const volume of value.volumes) { + if (typeof volume !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(volume)) process.exit(1); + process.stdout.write("volume\t" + volume + "\n"); + } + for (const pid of value.processIds) { + if (!Number.isSafeInteger(pid) || pid < 1) process.exit(1); + process.stdout.write("processId\t" + pid + "\n"); + } + ' +)" +if [ -n "$cleanup_rows" ]; then + while IFS=$'\t' read -r kind identity; do + case "$kind" in + volume) recorded_volumes+=("$identity") ;; + processId) recorded_process_ids+=("$identity") ;; + *) echo "Cleanup evidence contains an invalid identity" >&2; exit 1 ;; + esac + done <<<"$cleanup_rows" +fi +test ! -e "$workspace" +test ! -L "$workspace" +sandbox_container_output="$(docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name")" +if [ -n "$sandbox_container_output" ]; then + echo "A test-owned OpenShell sandbox container remains" >&2 + exit 1 +fi +container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" +if printf '%s\n' "$container_rows" | awk -F '\t' -v name="openshell-cluster-$gateway_name" '$2 == name { found = 1 } END { exit found ? 0 : 1 }'; then + echo "The test-owned OpenShell gateway container remains" >&2 + exit 1 +fi +volume_names="$(docker volume ls --format '{{.Name}}')" +if printf '%s\n' "$volume_names" | grep -Fqx "openshell-cluster-$gateway_name"; then + echo "The test-owned OpenShell gateway volume remains" >&2 + exit 1 +fi +for volume in "${"${recorded_volumes[@]}"}"; do + if printf '%s\n' "$volume_names" | grep -Fqx "$volume"; then + echo "A recorded test-owned Docker volume remains" >&2 + exit 1 + fi +done +for pid in "${"${recorded_process_ids[@]}"}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "A recorded test-owned helper process remains" >&2 + exit 1 + fi +done +read_proc_uid() { + awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' "$1/status" 2>/dev/null +} +read_proc_environment() { + dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' +} +read_proc_command() { + dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' +} +handle_proc_read_failure() { + local proc_dir="$1" field="$2" process_uid directory_uid + [ -d "$proc_dir" ] || return 0 + if process_uid="$(read_proc_uid "$proc_dir")"; then + [ "$process_uid" = "$(id -u)" ] || return 0 + else + [ -d "$proc_dir" ] || return 0 + if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then + [ -d "$proc_dir" ] || return 0 + echo "Unable to verify the owner of a live process after a failed $field read" >&2 + exit 1 + fi + [ "$directory_uid" = "$(id -u)" ] || return 0 + fi + echo "Unable to inspect $field for a live same-user process" >&2 + exit 1 +} +for proc_dir in /proc/[0-9]*; do + if ! process_uid="$(read_proc_uid "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" owner + continue + fi + [ "$process_uid" = "$(id -u)" ] || continue + if ! environment="$(read_proc_environment "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" environment + continue + fi + printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue + if ! cmdline="$(read_proc_command "$proc_dir")"; then + handle_proc_read_failure "$proc_dir" command + continue + fi + case "$cmdline" in + *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) + echo "A job-owned helper process remains" >&2 + exit 1 + ;; + esac +done +` + PRESERVED_BASELINE_SCRIPT; + +const JETSON_TEST_SCRIPT = String.raw`set -euo pipefail +candidate_sha="$1" +job_id="$2" +timeout_seconds="$3" +[[ "$candidate_sha" =~ ^[a-f0-9]{40}$ ]] +[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] +[[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] + +workspace_root=/var/tmp/nemoclaw-jetson-e2e +if [ -L "$workspace_root" ]; then + echo "Jetson E2E workspace root must not be a symbolic link" >&2 + exit 1 +fi +install -d -m 0700 "$workspace_root" +workspace="$workspace_root/$job_id" +if [ -e "$workspace" ]; then + echo "Jetson E2E workspace already exists; cleanup recovery is required" >&2 + exit 1 +fi +install -d -m 0700 "$workspace" +install -d -m 0700 \ + "$workspace/home" \ + "$workspace/home/.cache" \ + "$workspace/home/.config" \ + "$workspace/home/.local/bin" \ + "$workspace/home/.local/share" \ + "$workspace/home/.local/state" \ + "$workspace/npm-prefix" \ + "$workspace/runtime" \ + "$workspace/tmp" + +export HOME="$workspace/home" +export TMPDIR="$workspace/tmp" +export XDG_CACHE_HOME="$workspace/home/.cache" +export XDG_CONFIG_HOME="$workspace/home/.config" +export XDG_DATA_HOME="$workspace/home/.local/share" +export XDG_STATE_HOME="$workspace/home/.local/state" +export XDG_BIN_HOME="$workspace/home/.local/bin" +export XDG_RUNTIME_DIR="$workspace/runtime" +export npm_config_prefix="$workspace/npm-prefix" +export PATH="$XDG_BIN_HOME:$workspace/npm-prefix/bin:$PATH" +export NEMOCLAW_JETSON_WORKSPACE="$workspace" +unset DBUS_SESSION_BUS_ADDRESS NEMOCLAW_OPENSHELL_BIN NEMOCLAW_OPENSHELL_GATEWAY_BIN + +if [ -w /usr/local/bin ] || [ -w /usr/bin ]; then + echo "Jetson job user must not be able to install OpenShell into a host binary directory" >&2 + exit 1 +fi + +git init --quiet "$workspace/repository" +git -C "$workspace/repository" remote add origin https://github.com/NVIDIA/NemoClaw.git +git -C "$workspace/repository" fetch --quiet --depth=1 --no-tags origin "$candidate_sha" +fetched_sha="$(git -C "$workspace/repository" rev-parse FETCH_HEAD)" +[[ "$fetched_sha" == "$candidate_sha" ]] +git -C "$workspace/repository" checkout --quiet --detach "$candidate_sha" +cd "$workspace/repository" + +node -e ' + const [major, minor] = process.versions.node.split(".").map(Number); + if (major < 22 || (major === 22 && minor < 19)) process.exit(1); +' || { echo "Jetson E2E requires Node.js 22.19.0 or later" >&2; exit 1; } +npm_major="$(npm --version | cut -d . -f 1)" +[[ "$npm_major" =~ ^[0-9]+$ ]] && (( npm_major >= 10 )) || { + echo "Jetson E2E requires npm 10 or later" >&2 + exit 1 +} +npm ci --ignore-scripts +npm run build:cli + +export E2E_JOB=1 +export E2E_TARGET_ID=jetson-nvmap-gpu +export E2E_ARTIFACT_DIR="$workspace/e2e-artifacts/live/jetson-nvmap-gpu" +export NEMOCLAW_CLI_BIN="$workspace/repository/bin/nemoclaw.js" +export NEMOCLAW_RUN_LIVE_E2E=1 +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_SANDBOX_NAME=e2e-jetson-nvmap +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_PROVIDER=ollama +export OPENSHELL_GATEWAY=nemoclaw + +timeout --foreground --signal=TERM --kill-after=30s "${"${timeout_seconds}"}s" \ + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/jetson-nvmap-gpu.test.ts + +for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox; do + installed_path="$(command -v "$installed_command")" + canonical_path="$(realpath -e "$installed_path")" + case "$canonical_path" in + "$workspace"/*) ;; + *) echo "$installed_command resolved outside the Jetson job workspace" >&2; exit 1 ;; + esac +done +`; + +const COLLECT_ARTIFACT_SCRIPT = String.raw`set -euo pipefail +job_id="$1" +[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] +workspace="/var/tmp/nemoclaw-jetson-e2e/$job_id" + +artifact_directory="$workspace/e2e-artifacts/live/jetson-nvmap-gpu" +if [ ! -d "$artifact_directory" ] || [ -L "$artifact_directory" ]; then + exit 0 +fi +artifact_bytes="$(du --apparent-size --block-size=1 --summarize "$artifact_directory" | cut -f1)" +[[ "$artifact_bytes" =~ ^[0-9]+$ ]] +if (( artifact_bytes > ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} )); then + echo "Jetson E2E artifacts exceed ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} bytes" >&2 + exit 1 +fi +archive="$workspace/jetson-e2e-artifacts.tar.gz" +tar --one-file-system --create --gzip --file "$archive" --directory "$artifact_directory" -- . +archive_bytes="$(stat --format='%s' "$archive")" +if (( archive_bytes > ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} )); then + echo "Jetson E2E artifact archive exceeds ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} bytes" >&2 + exit 1 +fi +base64 --wrap=0 "$archive" +`; + +export interface SshJetsonWorkerConfig { + cleanupExecutable: string; + destination: string; + identityFile: string; + knownHostsFile: string; + stateDirectory: string; + testTimeoutSeconds: number; +} + +function requireSecureFile( + file: string, + options: { executable?: boolean; private?: boolean }, +): void { + if (!path.isAbsolute(file) || /[\u0000-\u001f\u007f]/u.test(file)) { + throw new Error(`${file} must be an absolute path without control characters`); + } + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`${file} must be a regular file`); + if ((stat.mode & 0o022) !== 0) throw new Error(`${file} must not be group- or world-writable`); + if (options.private && (stat.mode & 0o077) !== 0) { + throw new Error(`${file} must be readable only by its owner`); + } + const currentUid = process.getuid?.(); + if (currentUid !== undefined && stat.uid !== currentUid && stat.uid !== 0) { + throw new Error(`${file} must be owned by root or the dispatcher user`); + } + if (options.executable && (stat.mode & 0o111) === 0) { + throw new Error(`${file} must be executable`); + } +} + +function requireSecureCleanupExecutable(file: string): void { + if (file !== JETSON_CLEANUP_EXECUTABLE) { + requireSecureFile(file, { executable: true }); + return; + } + const linkStat = fs.lstatSync(file); + if (!linkStat.isSymbolicLink()) { + requireSecureFile(file, { executable: true }); + return; + } + if (linkStat.uid !== 0) { + throw new Error(`${file} symbolic link must be owned by root`); + } + if (fs.readlinkSync(file) !== JETSON_CLEANUP_TARGET) { + throw new Error(`${file} must select the managed current-release cleanup program`); + } + requireSecureFile(JETSON_CLEANUP_TARGET, { executable: true }); +} + +function positiveIntegerEnvironment( + value: string | undefined, + name: string, + minimum: number, + maximum: number, +): number { + if (!value || !/^[1-9][0-9]*$/u.test(value)) { + throw new Error(`${name} must be a positive integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(`${name} must be between ${minimum} and ${maximum}`); + } + return parsed; +} + +export function loadSshJetsonWorkerConfig( + options: { stateDirectory: string }, + env: NodeJS.ProcessEnv = process.env, +): SshJetsonWorkerConfig { + if (!path.isAbsolute(options.stateDirectory)) { + throw new Error("Jetson dispatcher state directory must be absolute"); + } + const destination = env.JETSON_DISPATCH_SSH_DESTINATION ?? ""; + if (destination !== JETSON_SSH_DESTINATION) { + throw new Error(`JETSON_DISPATCH_SSH_DESTINATION must be ${JETSON_SSH_DESTINATION}`); + } + const identityFile = env.JETSON_DISPATCH_SSH_IDENTITY_FILE ?? ""; + const knownHostsFile = env.JETSON_DISPATCH_SSH_KNOWN_HOSTS_FILE ?? ""; + const cleanupExecutable = env.JETSON_DISPATCH_CLEANUP_EXECUTABLE ?? ""; + requireSecureFile(identityFile, { private: true }); + requireSecureFile(knownHostsFile, {}); + requireSecureCleanupExecutable(cleanupExecutable); + return { + cleanupExecutable, + destination, + identityFile, + knownHostsFile, + stateDirectory: options.stateDirectory, + testTimeoutSeconds: positiveIntegerEnvironment( + env.JETSON_DISPATCH_TEST_TIMEOUT_SECONDS, + "JETSON_DISPATCH_TEST_TIMEOUT_SECONDS", + 60, + 50 * 60, + ), + }; +} + +interface ProcessResult { + stdout: string; + stderr: string; +} + +export class ProcessFailure extends Error { + readonly result: ProcessResult; + + constructor(message: string, result: ProcessResult) { + super(message); + this.result = result; + } +} + +function runProcess(options: { + executable: string; + args: string[]; + input?: string; + signal: AbortSignal; +}): Promise { + return new Promise((resolve, reject) => { + const child = spawn(options.executable, options.args, { + env: SAFE_PROCESS_ENV, + stdio: ["pipe", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let outputBytes = 0; + let settled = false; + let killTimer: NodeJS.Timeout | undefined; + + const finish = (error?: Error, result?: ProcessResult): void => { + if (settled) return; + settled = true; + options.signal.removeEventListener("abort", abort); + if (error) reject(error); + else resolve(result!); + }; + const stop = (): void => { + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), 5_000); + killTimer.unref(); + }; + const abort = (): void => stop(); + const collect = (chunks: Buffer[], chunk: Buffer): void => { + if (settled) return; + outputBytes += chunk.length; + if (outputBytes > MAX_PROCESS_OUTPUT_BYTES) { + stop(); + finish(new Error(`Jetson command output exceeds ${MAX_PROCESS_OUTPUT_BYTES} bytes`)); + return; + } + chunks.push(chunk); + }; + + options.signal.addEventListener("abort", abort, { once: true }); + if (options.signal.aborted) abort(); + child.stdout.on("data", (chunk: Buffer) => collect(stdout, chunk)); + child.stderr.on("data", (chunk: Buffer) => collect(stderr, chunk)); + child.stdin.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") { + stop(); + finish(error); + } + }); + child.once("error", (error) => finish(error)); + child.once("close", (code, signal) => { + if (killTimer) clearTimeout(killTimer); + if (settled) return; + const result = { + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }; + if (options.signal.aborted) { + finish(new ProcessFailure("Jetson command was interrupted", result)); + return; + } + if (code !== 0) { + finish( + new ProcessFailure( + `Jetson command exited with ${code ?? `signal ${signal ?? "unknown"}`}: ${result.stderr.slice(-500)}`, + result, + ), + ); + return; + } + finish(undefined, result); + }); + if (options.input !== undefined) child.stdin.end(options.input, "utf8"); + else child.stdin.end(); + }); +} + +function parseDeviceIdentity(output: string): JetsonDeviceIdentity { + if (Buffer.byteLength(output) > 8 * 1024) throw new Error("Jetson identity output is too large"); + const entries = new Map(); + for (const line of output.trimEnd().split("\n")) { + const separator = line.indexOf("\t"); + if (separator < 1) throw new Error("Jetson identity output is malformed"); + const key = line.slice(0, separator); + const value = line.slice(separator + 1); + if (entries.has(key) || value.length === 0 || /[\u0000-\u001f\u007f]/u.test(value)) { + throw new Error("Jetson identity output is malformed"); + } + entries.set(key, value.slice(0, 500)); + } + const expected = ["jetpackVersion", "jetsonLinuxRelease", "kernel", "model"]; + if (entries.size !== expected.length || expected.some((key) => !entries.has(key))) { + throw new Error("Jetson identity output is incomplete"); + } + return { + model: entries.get("model")!, + jetpackVersion: entries.get("jetpackVersion")!, + jetsonLinuxRelease: entries.get("jetsonLinuxRelease")!, + kernel: entries.get("kernel")!, + }; +} + +interface JetsonPreservedBaseline { + nodePath: string; + nodeVersion: string; + npmPath: string; + npmVersion: string; + ollamaPath: string; + ollamaModels: JetsonOllamaModel[]; + openshellState: "absent"; +} + +interface JetsonOllamaModel { + name: string; + id: string; +} + +function parseOllamaModelsBase64(value: string): JetsonOllamaModel[] { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value)) { + throw new Error("Jetson protected-baseline Ollama model data is malformed"); + } + const decoded = Buffer.from(value, "base64"); + if ( + decoded.length > MAX_OLLAMA_MODELS_BYTES || + decoded.toString("base64") !== value || + decoded.includes(0) + ) { + throw new Error("Jetson protected-baseline Ollama model data is malformed or oversized"); + } + const rows = decoded.length === 0 ? [] : decoded.toString("utf8").trimEnd().split("\n"); + if (rows.length > MAX_OLLAMA_MODELS) { + throw new Error("Jetson protected-baseline Ollama model data is malformed or oversized"); + } + const models = rows.map((row) => { + const fields = row.split("\t"); + if ( + fields.length !== 2 || + !/^[A-Za-z0-9][A-Za-z0-9._/:@-]{0,254}$/u.test(fields[0]!) || + !/^[a-f0-9]{12,64}$/u.test(fields[1]!) + ) { + throw new Error("Jetson protected-baseline Ollama model data is malformed"); + } + return { name: fields[0]!, id: fields[1]! }; + }); + const sorted = [...models].sort((left, right) => { + if (left.name !== right.name) return left.name < right.name ? -1 : 1; + if (left.id === right.id) return 0; + return left.id < right.id ? -1 : 1; + }); + const names = new Set(); + if ( + models.some((model) => { + if (names.has(model.name)) return true; + names.add(model.name); + return false; + }) || + models.some( + (model, index) => model.name !== sorted[index]!.name || model.id !== sorted[index]!.id, + ) + ) { + throw new Error("Jetson protected-baseline Ollama model data is duplicate or ambiguous"); + } + return models; +} + +function validateOllamaModels(value: unknown): JetsonOllamaModel[] { + if (!Array.isArray(value) || value.length > MAX_OLLAMA_MODELS) { + throw new Error("Jetson protected-baseline Ollama model record is malformed or oversized"); + } + const serialized = value + .map((model) => { + if (!model || typeof model !== "object" || Array.isArray(model)) { + throw new Error("Jetson protected-baseline Ollama model record is malformed"); + } + const record = model as Record; + if ( + Object.keys(record).sort().join(",") !== "id,name" || + typeof record.name !== "string" || + typeof record.id !== "string" + ) { + throw new Error("Jetson protected-baseline Ollama model record is malformed"); + } + return `${record.name}\t${record.id}`; + }) + .join("\n"); + if (Buffer.byteLength(serialized) > MAX_OLLAMA_MODELS_BYTES) { + throw new Error("Jetson protected-baseline Ollama model record is malformed or oversized"); + } + return parseOllamaModelsBase64(Buffer.from(serialized).toString("base64")); +} + +function parsePreservedBaseline(output: string): JetsonPreservedBaseline { + if (Buffer.byteLength(output) > MAX_BASELINE_OUTPUT_BYTES) { + throw new Error("Jetson protected-baseline output is too large"); + } + const entries = new Map(); + for (const line of output.trimEnd().split("\n")) { + const separator = line.indexOf("\t"); + if (separator < 1) throw new Error("Jetson protected-baseline output is malformed"); + const key = line.slice(0, separator); + const value = line.slice(separator + 1); + if ( + entries.has(key) || + (value.length === 0 && key !== "ollamaModelsBase64") || + /[\u0000-\u001f\u007f]/u.test(value) + ) { + throw new Error("Jetson protected-baseline output is malformed"); + } + entries.set(key, value); + } + const expected = [ + "nodePath", + "nodeVersion", + "npmPath", + "npmVersion", + "ollamaPath", + "ollamaModelsBase64", + "openshellState", + ] as const; + if (entries.size !== expected.length || expected.some((key) => !entries.has(key))) { + throw new Error("Jetson protected-baseline output is incomplete"); + } + const baseline: JetsonPreservedBaseline = { + nodePath: entries.get("nodePath")!, + nodeVersion: entries.get("nodeVersion")!, + npmPath: entries.get("npmPath")!, + npmVersion: entries.get("npmVersion")!, + ollamaPath: entries.get("ollamaPath")!, + ollamaModels: parseOllamaModelsBase64(entries.get("ollamaModelsBase64")!), + openshellState: entries.get("openshellState") as "absent", + }; + if (baseline.openshellState !== "absent") { + throw new Error("Jetson protected-baseline OpenShell state is malformed"); + } + if (Buffer.byteLength(`${JSON.stringify(baseline)}\n`) > MAX_BASELINE_RECORD_BYTES) { + throw new Error("Jetson protected-baseline record is too large"); + } + return baseline; +} + +function parsePreservedBaselineJson(output: string): JetsonPreservedBaseline { + if (Buffer.byteLength(output) > MAX_BASELINE_RECORD_BYTES) { + throw new Error("Jetson protected-baseline record is too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Jetson protected-baseline record is malformed"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Jetson protected-baseline record is malformed"); + } + const record = parsed as Record; + const expected = [ + "nodePath", + "nodeVersion", + "npmPath", + "npmVersion", + "ollamaPath", + "ollamaModels", + "openshellState", + ]; + if ( + Object.keys(record).length !== expected.length || + expected.some((key) => key !== "ollamaModels" && typeof record[key] !== "string") + ) { + throw new Error("Jetson protected-baseline record is incomplete"); + } + const ollamaModels = validateOllamaModels(record.ollamaModels); + const baselineOutput = [ + `nodePath\t${record.nodePath}`, + `nodeVersion\t${record.nodeVersion}`, + `npmPath\t${record.npmPath}`, + `npmVersion\t${record.npmVersion}`, + `ollamaPath\t${record.ollamaPath}`, + `ollamaModelsBase64\t${Buffer.from( + ollamaModels.map((model) => `${model.name}\t${model.id}`).join("\n"), + ).toString("base64")}`, + `openshellState\t${record.openshellState}`, + ].join("\n"); + return parsePreservedBaseline(baselineOutput); +} + +function protectedHostBaselineMatches( + expected: JetsonPreservedBaseline, + actual: JetsonPreservedBaseline, +): boolean { + return ( + expected.nodePath === actual.nodePath && + expected.nodeVersion === actual.nodeVersion && + expected.npmPath === actual.npmPath && + expected.npmVersion === actual.npmVersion && + expected.ollamaPath === actual.ollamaPath && + expected.openshellState === actual.openshellState + ); +} + +function preservesOllamaModels( + expected: JetsonPreservedBaseline, + actual: JetsonPreservedBaseline, +): boolean { + const actualModels = new Map(actual.ollamaModels.map((model) => [model.name, model.id])); + return expected.ollamaModels.every((model) => actualModels.get(model.name) === model.id); +} + +function validateCleanupEvidence(value: unknown): JetsonCleanupEvidence { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Jetson cleanup evidence is malformed"); + } + const record = value as Record; + if ( + Object.keys(record).length !== 3 || + record.schemaVersion !== 1 || + !Array.isArray(record.volumes) || + !Array.isArray(record.processIds) + ) { + throw new Error("Jetson cleanup evidence is malformed"); + } + const volumes = record.volumes; + const processIds = record.processIds; + if ( + volumes.some( + (volume) => typeof volume !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(volume), + ) || + processIds.some((pid) => typeof pid !== "number" || !Number.isSafeInteger(pid) || pid < 1) + ) { + throw new Error("Jetson cleanup evidence contains an invalid resource identity"); + } + return { + schemaVersion: 1, + volumes: [...new Set(volumes as string[])].sort(), + processIds: [...new Set(processIds as number[])].sort((left, right) => left - right), + }; +} + +function parseCleanupEvidenceJson(output: string): JetsonCleanupEvidence { + if (Buffer.byteLength(output) > MAX_CLEANUP_EVIDENCE_BYTES) { + throw new Error("Jetson cleanup evidence is too large"); + } + try { + return validateCleanupEvidence(JSON.parse(output)); + } catch (error) { + if (error instanceof Error && error.message.startsWith("Jetson cleanup evidence")) throw error; + throw new Error("Jetson cleanup evidence is malformed"); + } +} + +function parseCleanupCommandEvidence(output: string): JetsonCleanupEvidence { + if (Buffer.byteLength(output) > MAX_PROCESS_OUTPUT_BYTES) { + throw new Error("Jetson cleanup output is too large"); + } + const lines = output.replace(/\r/gu, "").split("\n"); + while (lines.at(-1) === "") lines.pop(); + const beginIndex = lines.indexOf(CLEANUP_EVIDENCE_BEGIN); + const endIndex = lines.indexOf(CLEANUP_EVIDENCE_END); + if ( + beginIndex < 0 || + endIndex < 0 || + beginIndex !== lines.lastIndexOf(CLEANUP_EVIDENCE_BEGIN) || + endIndex !== lines.lastIndexOf(CLEANUP_EVIDENCE_END) || + beginIndex >= endIndex + ) { + throw new Error("Jetson cleanup did not return bounded resource evidence"); + } + const volumes: string[] = []; + const processIds: number[] = []; + for (const line of lines.slice(beginIndex + 1, endIndex)) { + const fields = line.split("\t"); + if (fields.length !== 2) throw new Error("Jetson cleanup evidence is malformed"); + const [kind, identity] = fields; + if (kind === "volume") { + volumes.push(identity!); + } else if (kind === "processId" && /^[1-9][0-9]*$/u.test(identity!)) { + processIds.push(Number(identity)); + } else { + throw new Error("Jetson cleanup evidence is malformed"); + } + } + return validateCleanupEvidence({ schemaVersion: 1, volumes, processIds }); +} + +export class SshJetsonDispatchWorker implements JetsonDispatchWorker { + readonly #config: SshJetsonWorkerConfig; + readonly #runProcess: typeof runProcess; + + constructor(config: SshJetsonWorkerConfig, processRunner: typeof runProcess = runProcess) { + this.#config = config; + this.#runProcess = processRunner; + } + + async run( + request: JetsonDispatchRequest, + options: { jobId: string; signal: AbortSignal }, + ): Promise { + fs.rmSync(this.#baselinePath(options.jobId), { force: true }); + const baselineResult = await this.#runProcess({ + executable: "ssh", + args: [...this.#sshArgs(), this.#config.destination, "bash", "-s"], + input: PRESERVED_BASELINE_SCRIPT, + signal: options.signal, + }); + const baseline = parsePreservedBaseline(baselineResult.stdout); + writePrivateRegularFile(this.#baselinePath(options.jobId), `${JSON.stringify(baseline)}\n`); + const identityResult = await this.#runProcess({ + executable: "ssh", + args: [...this.#sshArgs(), this.#config.destination, "bash", "-s"], + input: DEVICE_IDENTITY_SCRIPT, + signal: options.signal, + }); + const device = parseDeviceIdentity(identityResult.stdout); + let testResult: ProcessResult | undefined; + let testError: unknown; + try { + testResult = await this.#runProcess({ + executable: "ssh", + args: [ + ...this.#sshArgs(), + this.#config.destination, + "bash", + "-s", + "--", + request.candidateSha, + options.jobId, + String(this.#config.testTimeoutSeconds), + ], + input: JETSON_TEST_SCRIPT, + signal: options.signal, + }); + } catch (error) { + testError = error; + } + let artifactArchiveBase64: string | undefined; + let artifactCollectionError: unknown; + if (!options.signal.aborted) { + try { + artifactArchiveBase64 = await this.#collectArtifact(options.jobId, options.signal); + } catch (error) { + artifactCollectionError = error; + } + } + if (testError instanceof ProcessFailure) { + const collectionFailure = + artifactCollectionError instanceof Error + ? `; artifact collection failed: ${artifactCollectionError.message}` + : ""; + const failure = new Error(`${testError.message}${collectionFailure}`) as Error & { + artifactArchiveBase64?: string; + log: string; + }; + failure.log = [ + "=== Jetson identity ===", + identityResult.stdout.trimEnd(), + "=== Jetson E2E stdout ===", + testError.result.stdout.trimEnd(), + "=== Jetson E2E stderr ===", + testError.result.stderr.trimEnd(), + "", + ].join("\n"); + if (artifactArchiveBase64 !== undefined) { + failure.artifactArchiveBase64 = artifactArchiveBase64; + } + throw failure; + } + if (testError) throw testError; + if (!testResult) throw new Error("Jetson E2E did not return a process result"); + if (artifactCollectionError) { + const collectionMessage = + artifactCollectionError instanceof Error + ? artifactCollectionError.message + : "artifact collection failed"; + const failure = new Error(collectionMessage) as Error & { log: string }; + failure.log = [ + "=== Jetson identity ===", + identityResult.stdout.trimEnd(), + "=== Jetson E2E stdout ===", + testResult.stdout.trimEnd(), + "=== Jetson E2E stderr ===", + testResult.stderr.trimEnd(), + "=== Jetson artifact collection error ===", + collectionMessage, + "", + ].join("\n"); + throw failure; + } + if (artifactArchiveBase64 === undefined) { + throw new Error("Successful Jetson E2E did not produce a bounded artifact archive"); + } + return { + artifactArchiveBase64, + device, + log: [ + "=== Jetson identity ===", + identityResult.stdout.trimEnd(), + "=== Jetson E2E stdout ===", + testResult.stdout.trimEnd(), + "=== Jetson E2E stderr ===", + testResult.stderr.trimEnd(), + "", + ].join("\n"), + }; + } + + async #collectArtifact(jobId: string, signal: AbortSignal): Promise { + const result = await this.#runProcess({ + executable: "ssh", + args: [...this.#sshArgs(), this.#config.destination, "bash", "-s", "--", jobId], + input: COLLECT_ARTIFACT_SCRIPT, + signal, + }); + if (result.stdout.length === 0) return undefined; + decodeJetsonArtifactArchive(result.stdout); + return result.stdout; + } + + async cleanup(options: { jobId: string; signal: AbortSignal }): Promise { + const cleanupResult = await this.#runProcess({ + executable: this.#config.cleanupExecutable, + args: [], + signal: options.signal, + }); + const reportedEvidence = parseCleanupCommandEvidence(cleanupResult.stdout); + const evidencePath = this.#cleanupEvidencePath(options.jobId); + const persistedRaw = readPrivateRegularFile(evidencePath, { + allowMissing: true, + maxBytes: MAX_CLEANUP_EVIDENCE_BYTES, + }); + if (persistedRaw === null) throw new Error("Jetson cleanup evidence is missing"); + const persistedEvidence = parseCleanupEvidenceJson(persistedRaw); + if (JSON.stringify(reportedEvidence) !== JSON.stringify(persistedEvidence)) { + throw new Error("Jetson cleanup output differs from its durable resource evidence"); + } + const serializedEvidence = `${JSON.stringify(persistedEvidence)}\n`; + const expectedRaw = readPrivateRegularFile(this.#baselinePath(options.jobId), { + allowMissing: true, + maxBytes: MAX_BASELINE_RECORD_BYTES, + }); + const verification = await this.#runProcess({ + executable: "ssh", + args: [ + ...this.#sshArgs(), + this.#config.destination, + "bash", + "-s", + "--", + options.jobId, + Buffer.from(serializedEvidence).toString("base64"), + ], + input: CLEANUP_VERIFICATION_SCRIPT, + signal: options.signal, + }); + const actual = parsePreservedBaseline(verification.stdout); + if (expectedRaw === null) return; + const expected = parsePreservedBaselineJson(expectedRaw); + if (!protectedHostBaselineMatches(expected, actual)) { + throw new Error("Jetson protected tool baseline differs after cleanup"); + } + if (!preservesOllamaModels(expected, actual)) { + throw new Error("Jetson cleanup did not preserve every pre-existing Ollama model"); + } + } + + #baselinePath(jobId: string): string { + if (!/^[a-f0-9]{64}$/u.test(jobId)) throw new Error("Jetson job ID is invalid"); + return path.join(this.#config.stateDirectory, `${jobId}.baseline.json`); + } + + #cleanupEvidencePath(jobId: string): string { + if (!/^[a-f0-9]{64}$/u.test(jobId)) throw new Error("Jetson job ID is invalid"); + return path.join(this.#config.stateDirectory, `${jobId}.cleanup.json`); + } + + #sshArgs(): string[] { + return [ + "-F", + "/dev/null", + "-T", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=15", + "-o", + "IdentitiesOnly=yes", + "-o", + "ServerAliveCountMax=2", + "-o", + "ServerAliveInterval=15", + "-o", + "StrictHostKeyChecking=yes", + "-o", + `UserKnownHostsFile=${this.#config.knownHostsFile}`, + "-i", + this.#config.identityFile, + ]; + } +} diff --git a/tools/e2e/nemoclaw-jetson-dispatch.service b/tools/e2e/nemoclaw-jetson-dispatch.service new file mode 100644 index 00000000000..9c5834aae6b --- /dev/null +++ b/tools/e2e/nemoclaw-jetson-dispatch.service @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[Unit] +Description=NemoClaw Jetson dispatcher +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=nemoclaw-jetson-dispatch +Group=nemoclaw-jetson-dispatch +UMask=0077 +WorkingDirectory=/opt/nemoclaw-jetson-dispatch/current +EnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment +ExecStart=/usr/bin/node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-service.mts +Restart=on-failure +RestartSec=5 +TimeoutStopSec=360 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/var/lib/nemoclaw-jetson-dispatch + +[Install] +WantedBy=multi-user.target diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index a4338ad7016..4ca631b2e3e 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -241,8 +241,11 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow ) { errors.push("Manual PR E2E concurrency must be scoped to its pull request"); } - if (workflow.concurrency?.["cancel-in-progress"] !== "${{ inputs.checkout_sha != '' }}") { - errors.push("Manual PR E2E concurrency must cancel obsolete runs"); + if ( + workflow.concurrency?.["cancel-in-progress"] !== + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + ) { + errors.push("Manual PR E2E concurrency must not cancel an active Jetson dispatch"); } const matrixJob = workflow.jobs["generate-matrix"] ?? {}; @@ -271,6 +274,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } const authEnvironment = { ACTOR: "${{ github.actor }}", + ALLOW_JETSON_DISPATCH: "${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}", BASE_SHA: "${{ inputs.base_sha }}", CHECKOUT_REPOSITORY: "${{ inputs.checkout_repository }}", CHECKOUT_SHA: "${{ inputs.checkout_sha }}", @@ -293,10 +297,15 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } const authSource = String(authentication.run ?? ""); const acceptedJobCases = [ - "::false", - ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS.map((jobId) => `${jobId}::false`), + "::false:false", + ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS.map((jobId) => `${jobId}::false:false`), + ":jetson-nvmap-gpu:false:true", ].join(" | "); - const acceptedJobNames = PR_E2E_MANUAL_CONTROLLER_JOB_IDS.join(", or "); + const acceptedNames = [ + ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS, + "jetson-nvmap-gpu with its dispatch flag", + ]; + const acceptedJobNames = `${acceptedNames.slice(0, -1).join(", ")}, or ${acceptedNames.at(-1)}`; for (const fragment of [ '"$WORKFLOW_EVENT" == "workflow_dispatch"', '"$WORKFLOW_REF" == "refs/heads/main"', @@ -374,6 +383,11 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.name === "Checkout trusted llama.cpp qualification" && step.with?.repository === "${{ github.repository }}" && step.with?.ref === "${{ inputs.workflow_sha || github.workflow_sha }}"; + const trustedJetsonControllerCheckout = + jobName === "jetson-nvmap-gpu" && + step.name === "Check out trusted Jetson controller" && + step.with?.repository === "NVIDIA/NemoClaw" && + step.with?.ref === "${{ github.workflow_sha }}"; const trustedCheckout = trustedHermesFixtureCheckout || trustedReportHelperCheckout || @@ -381,7 +395,8 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow trustedPublicationCheckout || trustedManagedImageRuntimeCheckout || trustedLlamaCppPlanCheckout || - trustedLlamaCppQualificationCheckout; + trustedLlamaCppQualificationCheckout || + trustedJetsonControllerCheckout; if ( step.uses?.startsWith("actions/checkout@") && step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" && diff --git a/tools/e2e/private-file.mts b/tools/e2e/private-file.mts index 18e62303187..115533cc5d8 100644 --- a/tools/e2e/private-file.mts +++ b/tools/e2e/private-file.mts @@ -76,7 +76,7 @@ export function readPrivateRegularFile( } } -export function writePrivateRegularFile(file: string, contents: string): void { +export function writePrivateRegularFile(file: string, contents: string | Uint8Array): void { const descriptor = openPrivateFileForWrite(file); try { const stat = fs.fstatSync(descriptor); @@ -85,7 +85,10 @@ export function writePrivateRegularFile(file: string, contents: string): void { } fs.fchmodSync(descriptor, 0o600); fs.ftruncateSync(descriptor, 0); - fs.writeFileSync(descriptor, contents, "utf8"); + fs.writeFileSync( + descriptor, + typeof contents === "string" ? Buffer.from(contents, "utf8") : contents, + ); fs.fsyncSync(descriptor); } finally { fs.closeSync(descriptor); diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index c43e8420008..e1fcdf1bb15 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -95,6 +95,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json", }, ], + [ + "jetson-nvmap-gpu", + { + name: "e2e-jetson-nvmap-gpu", + path: "${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/", + }, + ], [ "retired-selector-compatibility", { @@ -415,6 +422,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): return ( jobName === "staging-brev-launchable" || jobName === "generate-matrix" || + jobName === "jetson-nvmap-gpu" || jobName === "live" || jobName === RETIRED_SELECTOR_COMPATIBILITY_JOB || env.E2E_JOB === "1" || diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 9c5210d4271..12d565143e7 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -3958,63 +3958,130 @@ function validateBedrockRuntimeCompatibleAnthropicJob( requireRunDoesNotContain(errors, runVitest, "${{ inputs."); } -function validateAllowJetsonRunnerQueueInput( - errors: string[], - dispatchInputs: WorkflowRecord, -): void { - const input = requireInput(errors, dispatchInputs, "allow_jetson_runner_queue"); +function validateAllowJetsonDispatchInput(errors: string[], dispatchInputs: WorkflowRecord): void { + const input = requireInput(errors, dispatchInputs, "allow_jetson_dispatch"); if (input.type !== "boolean") { - errors.push("workflow_dispatch allow_jetson_runner_queue input must be boolean"); + errors.push("workflow_dispatch allow_jetson_dispatch input must be boolean"); } if (input.default !== false) { - errors.push("workflow_dispatch allow_jetson_runner_queue input must default to false"); + errors.push("workflow_dispatch allow_jetson_dispatch input must default to false"); } const description = stringValue(input.description); if ( - !description.includes("repository administrator confirmation") || - !description.includes("Jetson runner") || - !description.includes("authoritative") || - !description.includes("NVIDIA/NemoClaw Settings -> Actions -> Runners") || - !description.includes("timeout-minutes") + !description.includes("Colossus dispatcher") || + !description.includes("Cloudflare Tunnel") || + !description.includes("cleanup helper") || + !description.includes("JETSON_DISPATCH_URL") || + !description.includes("test/e2e/docs/jetson-colossus-dispatch.md") ) { errors.push( - "workflow_dispatch allow_jetson_runner_queue input must require repository administrator confirmation from the authoritative NVIDIA/NemoClaw Settings -> Actions -> Runners inventory and document queued timeout behavior", + "workflow_dispatch allow_jetson_dispatch input must require the Colossus dispatcher, tunnel, cleanup helper, repository URL variable, and deployment checks", ); } } -function validateJetsonJobOptInBoundary(errors: string[], jobs: WorkflowRecord): void { +function validateJetsonControllerBoundary(errors: string[], jobs: WorkflowRecord): void { const job = asRecord(jobs["jetson-nvmap-gpu"]); if (job.needs !== "generate-matrix") { errors.push("jetson-nvmap-gpu job must depend on generate-matrix"); } const trustedSelector = - "${{ inputs.allow_jetson_runner_queue && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && ((github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',jetson-nvmap-gpu,') || contains(format(',{0},', inputs.targets), ',jetson-nvmap-gpu,')) }}"; + "${{ inputs.allow_jetson_dispatch && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && ((inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',jetson-nvmap-gpu,') || contains(format(',{0},', inputs.targets), ',jetson-nvmap-gpu,')) }}"; if (job.if !== trustedSelector) { errors.push( - "jetson-nvmap-gpu job must require allow_jetson_runner_queue before runner assignment and retain trusted-main selectors", + "jetson-nvmap-gpu job must require the dispatch opt-in, trusted main workflow dispatch, same-repository candidate, and target selectors", ); } - const configuredRunsOn = - "${{ vars.JETSON_E2E_RUNNER_LABEL || 'linux-arm64-gpu-jetson-orin-latest-1' }}"; - if (job["runs-on"] !== configuredRunsOn) { - errors.push("jetson-nvmap-gpu job must use the configured runner only after job-level opt-in"); + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("jetson-nvmap-gpu controller must use a GitHub-hosted runner"); + } + if (job["timeout-minutes"] !== 60) + errors.push("jetson-nvmap-gpu controller timeout must be 60 minutes"); + if ( + !isDeepStrictEqual(asRecord(job.concurrency), { + group: "jetson-nvmap-gpu-colossus", + queue: "max", + "cancel-in-progress": false, + }) + ) { + errors.push( + "jetson-nvmap-gpu concurrency must queue every dispatch on one fixed device without cancellation", + ); + } + if (!isDeepStrictEqual(asRecord(job.permissions), { contents: "read", "id-token": "write" })) { + errors.push("jetson-nvmap-gpu controller must grant only contents:read and id-token:write"); + } + if (Object.keys(asRecord(job.env)).length !== 0) { + errors.push("jetson-nvmap-gpu controller must not define a job-level environment"); } const steps = asSteps(job.steps); - if (steps.some((step) => step.name === "Guard Jetson runner dispatch")) { - errors.push("jetson-nvmap-gpu must enforce opt-in before runner assignment, not in a step"); + const checkout = namedStep(steps, "Check out trusted Jetson controller"); + if (!checkout || !stringValue(checkout.uses).startsWith("actions/checkout@")) { + errors.push("jetson-nvmap-gpu controller must check out the trusted workflow source"); + } else { + requireFullShaAction(errors, checkout, "jetson-nvmap-gpu trusted checkout"); + if ( + !isDeepStrictEqual(asRecord(checkout.with), { + repository: "NVIDIA/NemoClaw", + ref: "${{ github.workflow_sha }}", + "persist-credentials": false, + }) + ) { + errors.push( + "jetson-nvmap-gpu checkout must use the trusted workflow SHA without credentials", + ); + } + } + const setupNode = namedStep(steps, "Set up Node for Jetson controller"); + if (!setupNode || !stringValue(setupNode.uses).startsWith("actions/setup-node@")) { + errors.push("jetson-nvmap-gpu controller must set up Node.js"); + } else { + requireFullShaAction(errors, setupNode, "jetson-nvmap-gpu Node setup"); + if (asRecord(setupNode.with)["node-version"] !== 22) { + errors.push("jetson-nvmap-gpu controller must use Node.js 22"); + } + } + const dispatch = namedStep(steps, "Dispatch exact commit to Jetson through Colossus"); + if ( + dispatch?.run !== + "node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-client.mts" || + !isDeepStrictEqual(asRecord(dispatch?.env), { + E2E_ARTIFACT_DIR: "${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu", + JETSON_DISPATCH_CANDIDATE_SHA: "${{ inputs.checkout_sha || github.sha }}", + JETSON_DISPATCH_URL: "${{ vars.JETSON_DISPATCH_URL }}", + }) + ) { + errors.push( + "jetson-nvmap-gpu controller must dispatch only the exact candidate and configured URL", + ); + } + const upload = namedStep(steps, "Upload Jetson nvmap GPU artifacts"); + if ( + upload?.if !== "always()" || + upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || + !isDeepStrictEqual(asRecord(upload.with), { + name: "e2e-jetson-nvmap-gpu", + path: "${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/", + }) + ) { + errors.push("jetson-nvmap-gpu controller must upload its bounded dispatch artifact"); + } + if (steps.length !== 4) { + errors.push( + "jetson-nvmap-gpu controller must contain only checkout, Node setup, dispatch, and upload", + ); } } -export function validateJetsonRunnerDispatchBoundary(workflow: unknown): string[] { +export function validateJetsonDispatchBoundary(workflow: unknown): string[] { const workflowRecord = asRecord(workflow); const triggers = asRecord(workflowRecord.on ?? workflowRecord[true as unknown as string]); const workflowDispatch = asRecord(triggers.workflow_dispatch); const errors: string[] = []; - validateAllowJetsonRunnerQueueInput(errors, asRecord(workflowDispatch.inputs)); - validateJetsonJobOptInBoundary(errors, asRecord(workflowRecord.jobs)); + validateAllowJetsonDispatchInput(errors, asRecord(workflowDispatch.inputs)); + validateJetsonControllerBoundary(errors, asRecord(workflowRecord.jobs)); return errors; } @@ -4054,8 +4121,11 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): if (concurrency.group !== expectedGroup) { errors.push("workflow concurrency must isolate each full dispatch with github.run_id"); } - if (concurrency["cancel-in-progress"] !== "${{ inputs.checkout_sha != '' }}") { - errors.push("workflow concurrency must cancel only superseded PR gate runs"); + if ( + concurrency["cancel-in-progress"] !== + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + ) { + errors.push("workflow concurrency must not cancel an active Jetson dispatch"); } } @@ -4291,7 +4361,8 @@ function validateTrustedE2eDispatchReceipt( const dispatchReceiptEnv = asRecord(dispatchReceipt?.env); const expectedDispatchReceiptEnv = { ALLOW_DGX_SPARK_RUNNER_QUEUE: "${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}", - ALLOW_JETSON_RUNNER_QUEUE: "${{ inputs.allow_jetson_runner_queue && 'true' || 'false' }}", + ALLOW_JETSON_DISPATCH: "${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}", + ALLOW_JETSON_RUNNER_QUEUE: "false", BASE_SHA: "${{ inputs.checkout_sha != '' && inputs.base_sha || github.sha }}", CANDIDATE_REPOSITORY: "${{ inputs.checkout_repository || github.repository }}", CANDIDATE_SHA: "${{ inputs.checkout_sha || github.sha }}", @@ -4329,6 +4400,7 @@ function validateTrustedE2eDispatchReceipt( "jobs: $jobs", "targets: $targets", "allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue", + "allowJetsonDispatch: $allowJetsonDispatch", "allowJetsonRunnerQueue: $allowJetsonRunnerQueue", "includeStagingBrevLaunchable: $includeStagingBrevLaunchable", 'emptySelectors: ($jobs == "" and $targets == "")', @@ -4446,7 +4518,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (workflow["run-name"] !== expectedRunName) { errors.push("workflow run-name must expose the unique manual-dispatch correlation ID"); } - errors.push(...validateJetsonRunnerDispatchBoundary(workflow)); + errors.push(...validateJetsonDispatchBoundary(workflow)); const { errors: inventoryErrors, inventory: freeStandingInventory } = deriveFreeStandingJobsInventoryFromJobs(jobs); errors.push(...inventoryErrors); @@ -4532,7 +4604,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { `matrix='[${defaultMappings}]'`, `test_matrix='[${defaultTestMappings}]'`, ";;", - "inference-routing: | managed-image-protected-runtime:)", + "inference-routing: | managed-image-protected-runtime: | :jetson-nvmap-gpu)", "matrix='[]'", ";;", `:${deepAgentsTarget})`, diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index f3356f82f6f..bbe2c25cce3 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -10,6 +10,7 @@ import { type CredentialFreeTestMatrixRow, discoverCredentialFreeTests, } from "./credential-free-tests.mts"; +import { JETSON_DISPATCH_TARGET } from "./jetson-dispatch-contract.mts"; import { selectedRetiredControllerJobs } from "./retired-selector-compatibility.mts"; import { normalizeE2eSelectorIds } from "./selector-aliases.mts"; import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; @@ -197,6 +198,17 @@ export function buildE2eWorkflowPlan(selectors: WorkflowPlanSelectors = {}): E2e const targets = selectorIds(selectors.targets, "targets"); const inventory = readFreeStandingJobsInventory(); + const jetsonDispatchSelected = + (jobs.length === 1 && jobs[0] === JETSON_DISPATCH_TARGET && targets.length === 0) || + (targets.length === 1 && targets[0] === JETSON_DISPATCH_TARGET && jobs.length === 0); + if (jetsonDispatchSelected) { + return { + matrix: [], + testMatrix: [], + hermesSelected: false, + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }; + } const credentialFreeTests = discoverCredentialFreeTests(); if (jobs.length > 0) {