diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 8f5adc3dbe7..d60205153fa 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -67,7 +67,7 @@ Choose exactly one mode: - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. - The run skips `jetson-nvmap-gpu` unless its Colossus dispatch flag is `true`. + The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. - For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. @@ -243,12 +243,11 @@ repository `maintain` or `admin` permission before the Launchable path's source checkout. That role check is the authorization. A user permitted to dispatch this workflow may set `allow_jetson_dispatch=true` to add `jetson-nvmap-gpu` to an empty-selector manual run or enable its explicit -selection. Set it only after every deployment check in -[Jetson Dispatch Through Colossus](../../../test/e2e/docs/jetson-colossus-dispatch.md) -passes. -The Jetson lifecycle removes and verifies only its fixed job-owned cleanup -allowlist. Do not treat `cleanup: "succeeded"` as evidence that every possible -candidate host change was reversed. +selection. Set it only after the operator-owned service is available and +compatible with HTTP contract +version `1.0.0`, and `JETSON_DISPATCH_URL` contains its verified HTTPS origin. +See [Jetson Dispatch Controller](../../../test/e2e/docs/jetson-dispatch.md). +Require the uploaded Jetson receipt to report `cleanup: "succeeded"`. A permitted dispatcher may set `allow_dgx_spark_runner_queue=true` to add `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` to an empty-selector manual run or enable explicit qualification selection. Set it diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index deedc25bd5b..75e09573ec5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -34,7 +34,7 @@ on: - internal-nvidia - public-nvidia allow_jetson_dispatch: - description: "Set true for jetson-nvmap-gpu only after the Colossus dispatcher, Cloudflare Tunnel, cleanup helper, and JETSON_DISPATCH_URL repository variable have passed the deployment checks in test/e2e/docs/jetson-colossus-dispatch.md." + description: "Set true for jetson-nvmap-gpu only after the operator-owned dispatch backend is available and JETSON_DISPATCH_URL contains its verified HTTPS origin. Refer to test/e2e/docs/jetson-dispatch.md." required: false default: false type: boolean @@ -4924,7 +4924,7 @@ jobs: needs: generate-matrix if: ${{ inputs.allow_jetson_dispatch && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && ((inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',jetson-nvmap-gpu,') || contains(format(',{0},', inputs.targets), ',jetson-nvmap-gpu,')) }} concurrency: - group: jetson-nvmap-gpu-colossus + group: jetson-nvmap-gpu-dispatch queue: max cancel-in-progress: false runs-on: ubuntu-latest @@ -4945,7 +4945,7 @@ jobs: with: node-version: 22 - - name: Dispatch exact commit to Jetson through Colossus + - name: Dispatch exact commit to Jetson through operator backend env: E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu JETSON_DISPATCH_CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} diff --git a/test/e2e/README.md b/test/e2e/README.md index 1397870f5e1..ea133c5da2e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -456,12 +456,11 @@ runs remain queued instead of replacing one another. The Jetson nvmap and DGX Spark llama.cpp jobs remain excluded from ordinary and full runs unless their independent opt-in flags are `true`. -Set `allow_jetson_dispatch=true` to select `jetson-nvmap-gpu` only after every -deployment check in -[Jetson Dispatch Through Colossus](docs/jetson-colossus-dispatch.md) passes. -The Colossus lifecycle removes and verifies only the fixed job-owned cleanup -allowlist. Its cleanup evidence does not attest that every possible candidate -host change was reversed. +Set `allow_jetson_dispatch=true` to select `jetson-nvmap-gpu` after the +operator-owned dispatch service is available at the repository variable +`JETSON_DISPATCH_URL`. Refer to the +[Jetson dispatch controller](docs/jetson-dispatch.md) for the trusted workflow, +HTTP contract, and evidence boundary that NemoClaw owns. Set `allow_dgx_spark_runner_queue=true` to select both `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification`. GitHub can pause the qualification job for the @@ -798,7 +797,7 @@ For a PR revision run, a repository maintainer or administrator leaves `jobs` an - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. -The run skips `jetson-nvmap-gpu` unless its Colossus dispatch flag is `true`. +The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. The trusted workflow definition remains on `main` and binds the candidate head to the current PR base SHA. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 8f17b23e4ca..e079e1a1092 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -314,10 +314,10 @@ test/e2e/ `jobs` value is `managed-image-protected-runtime`; `targets` must remain empty. Refer to [NemoClaw E2E CI](../README.md). -- [Jetson Dispatch Through Colossus](jetson-colossus-dispatch.md) defines how - maintainers deploy and verify the temporary authenticated controller, - dispatcher, SSH, bounded cleanup, and stale-lock recovery path for - `jetson-nvmap-gpu`. +- [Jetson dispatch controller](jetson-dispatch.md) defines the NemoClaw-owned + HTTP contract, trusted GitHub controller, repository configuration, and + evidence for `jetson-nvmap-gpu`. The service behind that contract is + operator-owned infrastructure. - `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist with diff --git a/test/e2e/docs/jetson-colossus-dispatch.md b/test/e2e/docs/jetson-colossus-dispatch.md deleted file mode 100644 index 87ad9e1d0b1..00000000000 --- a/test/e2e/docs/jetson-colossus-dispatch.md +++ /dev/null @@ -1,1236 +0,0 @@ - - - -# Jetson Dispatch Through Colossus - -This temporary path runs the `jetson-nvmap-gpu` end-to-end (E2E) job on one Jetson behind a Colossus host. -GitHub Actions controls the job through an authenticated HTTPS endpoint. -Colossus retains the Cloudflare Tunnel credential, Jetson SSH key, SSH host key, device lock, and cleanup capability. -Candidate code runs only on the Jetson. - -Do not set `allow_jetson_dispatch=true` until every deployment check on this page passes. - -## Trust Boundary - -The deployment has these input and credential boundaries: - -| System | Receives | Must not receive | -| --- | --- | --- | -| GitHub-hosted controller | Candidate commit SHA, workflow run identity, public dispatch URL, and short-lived GitHub OpenID Connect (OIDC) token | Jetson SSH key, tunnel credential, and cleanup privilege | -| Colossus dispatcher | Validated workflow identity, fixed target, and candidate commit SHA | Request-controlled command, SSH host, repository, path, or cleanup command | -| Jetson | Candidate checkout and the fixed live E2E command | GitHub OIDC token, tunnel credential, and Colossus SSH private key | - -The dispatcher accepts only `NVIDIA/NemoClaw` and the trusted `main` E2E workflow. -It requires a GitHub-hosted controller, one repository ID, one workflow run identity, and the `jetson-nvmap-gpu` target. -It also requires a lowercase 40-character candidate commit SHA. -The temporary path rejects fork repositories. - -The credentials have these lifecycles: - -| Credential | Location and access | Lifetime and removal | -| --- | --- | --- | -| GitHub OIDC token | GitHub-hosted controller process memory and the authenticated request | The client reuses a token in process memory for at most four minutes and then requests another. It does not write the token to disk. | -| Cloudflare account certificate | The administrator account that runs `cloudflared tunnel login` | Remove `cert.pem` from Colossus after tunnel creation and DNS routing. Reauthenticate before later management changes. | -| Cloudflare Tunnel credential | `/etc/cloudflared/TUNNEL_UUID.json`, readable only by the tunnel service account | Keep it for this temporary deployment. Revoke the tunnel and remove the file when the deployment ends. | -| Jetson SSH private key | `/var/lib/nemoclaw-jetson-dispatch/id_ed25519`, readable only by the dispatcher service account | Keep it for this temporary deployment. Remove the matching Jetson public key and private key when the deployment ends. | - -The dispatcher rejects an OIDC token whose issued-to-expiry window exceeds 15 minutes. -The GitHub repository variable contains only the public dispatch URL. -Do not put a credential in `JETSON_DISPATCH_URL`. - -## Prepare the Dedicated Jetson - -Use a dedicated Jetson without production data or credentials. -The E2E job gives candidate code Docker access, which can control the dedicated host. -Cleanup removes only the fixed job-owned resources defined below. -It does not attest that cleanup reversed every possible host change made by candidate code. - -Run these checks as the `nvidia` account that the dispatcher uses: - -```bash -set -euo pipefail -uname -m -tr -d '\0' /dev/null 2>&1; then - echo 'OpenShell must be absent from the prepared Jetson' >&2 - exit 1 - fi - for host_bin in \ - "/usr/local/bin/$openshell_component" \ - "/usr/bin/$openshell_component" \ - "$HOME/.local/bin/$openshell_component"; do - test ! -e "$host_bin" && test ! -L "$host_bin" - done -done -ollama list -docker info --format '{{json .Runtimes}}' -test -c /dev/nvmap -if sudo -n true 2>/dev/null; then echo 'unexpected passwordless sudo'; exit 1; fi -``` - -The architecture must be `aarch64`. -Node.js must be version 22.19.0 or later, and npm must have major version 10 or later. -OpenShell must be absent from the host `PATH` and the three checked host binary directories. -Do not preinstall OpenShell on the Jetson. -The `ollama list` command must succeed. -Docker must expose the NVIDIA runtime required by the existing Jetson live E2E test. -Preinstall Ollama so candidate code does not invoke its host installer. -The `nvidia` account must not have passwordless `sudo` access. - -The worker creates `/var/tmp/nemoclaw-jetson-e2e/` and sets these job-local paths: - -- `HOME=/home` -- `TMPDIR=/tmp` -- `XDG_CACHE_HOME=/home/.cache` -- `XDG_CONFIG_HOME=/home/.config` -- `XDG_DATA_HOME=/home/.local/share` -- `XDG_STATE_HOME=/home/.local/state` -- `XDG_BIN_HOME=/home/.local/bin` -- `XDG_RUNTIME_DIR=/runtime` -- `npm_config_prefix=/npm-prefix` -- `PATH=$XDG_BIN_HOME:/npm-prefix/bin:$PATH` - -The worker unsets `DBUS_SESSION_BUS_ADDRESS` so onboarding uses OpenShell's existing standalone fallback when the job-local systemd user service cannot load. -The worker must not set `NEMOCLAW_DEFER_OPENSHELL_INSTALL`. -It must not invoke `scripts/install-openshell.sh`. -The existing live E2E runs `bash install.sh --non-interactive`. -NemoClaw onboarding owns the compatible pinned OpenShell installation in the job workspace. -After onboarding, `nemoclaw`, `openshell`, `openshell-gateway`, and `openshell-sandbox` must resolve canonically inside the job workspace. -The worker rejects a symbolic link or resolved path that leaves that workspace. - -Reserve these names and paths for this E2E target: - -- The current dispatcher-created `/var/tmp/nemoclaw-jetson-e2e/` workspace. -- The `/tmp/nemoclaw-services-e2e-jetson-nvmap` helper-service directory. -- The `e2e-jetson-nvmap` NemoClaw and OpenShell sandbox name. -- The `nemoclaw` OpenShell gateway name and forwards for the named sandbox. -- The recorded `ollama-auth-proxy`, OpenShell Docker gateway, and helper `cloudflared` processes. -- OpenShell-managed Docker containers labeled for `e2e-jetson-nvmap`. -- The volumes recorded from those labeled containers. -- The `openshell-cluster-nemoclaw` gateway container, volume, and recorded attached volumes. -- `nemoclaw-sandbox-local` images whose tag begins with `e2e-jetson-nvmap-`. - -Do not run unrelated work under these reserved names. -The cleanup program must act only on this allowlist. - -Before candidate execution, the worker records this protected tool and model baseline: - -- The resolved Node.js path and version. -- The resolved npm path and version. -- The resolved Ollama path and the sorted pre-existing Ollama model names and IDs. -- The required absence of host-level `openshell`, `openshell-gateway`, and `openshell-sandbox` binaries. - -The Ollama list accepts at most 64 sorted, unique model name and ID rows whose decoded content is at most 3 KiB. -The SSH probe output is limited to 4 KiB, while the persisted baseline record and its reader are limited to 8 KiB. -Before candidate execution, the worker serializes the baseline and rejects it if persistence would exceed that 8 KiB read bound. -The dispatcher stores the recorded values in a private `.baseline.json` state file. -After cleanup, the worker repeats the probes and requires every pre-existing Ollama model name and ID to remain. -Additional models installed by the job, such as `qwen3.5:9b`, are allowed only while the post-cleanup list stays within the same row and size bounds. -An oversized post-cleanup list fails cleanup and retains the device lock without removing any model. -Before and after candidate execution, the worker also requires `/dev/nvmap` and the Docker NVIDIA runtime to be available. -The worker keeps the record through cleanup and device-lock release so startup recovery can repeat the same comparison. -The host retention policy may remove it only after no device lock exists. -Before a replay of the same job ID, the worker removes the earlier record and writes a new record before it invokes candidate code. -If the initial baseline probe fails, the worker never invokes candidate code; cleanup verifies current prerequisites and allowlisted-resource absence without a before-and-after comparison. - -These other host resources also remain outside the cleanup allowlist: - -- Node.js, npm, and the Docker engine. -- The Ollama binary, service, models, configuration, and unrelated `ollama serve` processes. -- The NVIDIA container runtime and Docker images that the test does not own exclusively. -- The `/dev/nvmap` character device and its permissions. -- JetPack, Jetson Linux, CUDA, NVIDIA packages, other `apt` packages, SDK Manager, and downloaded flashing files. -- User accounts, SSH keys, and user files outside the current job workspace and named NemoClaw resources. -- Docker resources without the exact label, repository tag, or name association defined above. -- Processes other than the recorded job-home helper processes defined above. -- Colossus credentials, service configuration, and job evidence. - -The cleanup program must not change or remove any resource outside its allowlist. -It never removes Ollama models. -Review its command construction and target resolution before deployment. -Do not use broad process termination, Docker pruning, wildcard paths, or host-wide package removal. - -## Prepare the Colossus Dispatcher Account and Deployment Command - -Run these commands on Colossus. -The dispatcher requires `/usr/bin/node` 22.19.0 or later. - -```bash -set -euo pipefail -/usr/bin/node --version -/usr/bin/node -e ' - const [major, minor] = process.versions.node.split(".").map(Number); - if (major < 22 || (major === 22 && minor < 19)) process.exit(1); -' -sudo useradd --system --create-home \ - --home-dir /var/lib/nemoclaw-jetson-dispatch \ - --shell /usr/sbin/nologin nemoclaw-jetson-dispatch -sudo install -d -o nemoclaw-jetson-dispatch -g nemoclaw-jetson-dispatch -m 0700 \ - /var/lib/nemoclaw-jetson-dispatch -``` - -Bootstrap the deployment command from a separately reviewed checkout of `https://github.com/NVIDIA/NemoClaw.git`. -Replace the placeholder with the full lowercase 40-character SHA of that reviewed checkout: - -```bash -set -euo pipefail -REVIEWED_BOOTSTRAP_SHA=0000000000000000000000000000000000000000 -test "$(git remote get-url origin)" = https://github.com/NVIDIA/NemoClaw.git -test "$(git rev-parse HEAD)" = "$REVIEWED_BOOTSTRAP_SHA" -test -z "$(git status --short)" -sudo install -o root -g root -m 0755 \ - tools/e2e/colossus-jetson-dispatch-deploy.sh \ - /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy -test "$(sudo stat --format='%F:%u:%g:%a' -- \ - /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy)" = \ - 'regular file:0:0:755' -``` - -The deployment command requires root and accepts only `--commit `. -It rejects any additional argument. -The command does not update its installed copy. -Install a reviewed deployment-command revision separately when that command changes. - -The Colossus service runs the dispatcher files from the selected reviewed release. -The GitHub-hosted controller runs the client from the trusted workflow commit. -The Colossus service never checks out candidate code. - -Create a dedicated SSH key owned by the service account: - -```bash -sudo -u nemoclaw-jetson-dispatch ssh-keygen -t ed25519 -N '' \ - -f /var/lib/nemoclaw-jetson-dispatch/id_ed25519 -``` - -Add the public key to the Jetson `nvidia` account. -Restrict it to the Colossus source IP address and disable SSH session features that the worker does not use. -For the default USB link, the entry has this shape: - -```text -from="192.168.55.100",restrict ssh-ed25519 AAAA... -``` - -Capture the Jetson host key and verify its fingerprint through the serial console or another trusted channel: - -```bash -set -euo pipefail -umask 077 -known_hosts_tmp="$(mktemp /tmp/nemoclaw-jetson-known-hosts.XXXXXX)" -trap 'rm -f -- "$known_hosts_tmp"' EXIT -ssh-keyscan -T 10 -H 192.168.55.1 >"$known_hosts_tmp" -ssh-keygen -lf "$known_hosts_tmp" -sudo install -o nemoclaw-jetson-dispatch -g nemoclaw-jetson-dispatch -m 0600 \ - "$known_hosts_tmp" /var/lib/nemoclaw-jetson-dispatch/known_hosts -rm -f -- "$known_hosts_tmp" -trap - EXIT -``` - -The pinned `known_hosts` file validates the SSH host identity separately from the persisted cleanup baseline. -Do not accept a changed host key without reconciling it through the serial console or another trusted channel. - -## Define the Cleanup Program - -The deployment command installs `/usr/local/libexec/nemoclaw-jetson-cleanup` as a root-owned stable symbolic link to `/opt/nemoclaw-jetson-dispatch/current/tools/e2e/jetson-dispatch-cleanup.sh`. -The selected target file is owned by `root:root` with mode `0755`. -One atomic `current` switch selects both the dispatcher code and cleanup program, so an interruption cannot select them from different releases. -The deployed worker configuration must name that exact cleanup path. -The worker accepts the symbolic link only when root owns it, its target is the exact managed `current` cleanup program, and that target passes the regular executable ownership and write-permission checks. -It rejects arbitrary cleanup symbolic links and an unsafe managed target. -The deployment command accepts no cleanup path, resource selector, or cleanup-scope override. -Review a release that changes the cleanup allowlist separately before deployment. - -It must accept no arguments and must not be group- or world-writable. -It must derive the lowercase 64-character job ID from the private dispatcher `device.lock`. -It must reject a missing or malformed lock instead of selecting a broader target. -The bundled program fixes the state directory, SSH files, SSH destination, and cleanup names listed on this page. -Changing one of those values requires a corresponding reviewed source change. -The dispatcher invokes it after success, failure, cancellation, and timeout. -The dispatcher also invokes it during startup when `device.lock` remains from an interrupted service process. - -Before destructive cleanup, the program discovers and validates the job-owned Docker volume and process identities. -It accepts a helper or forward PID only when its environment has the exact job `HOME` and its command has one of these markers: - -- `ollama-auth-proxy.` -- `openshell-gateway` -- `openshell-forward` -- `openshell forward` -- `cloudflared` - -Discovery, worker absence verification, and the teardown probe fail closed when the owner, environment, or command of a live same-user process cannot be read from `/proc`. -A process that disappears during inspection or is identified as owned by a different user is ignored. -It merges them into a mode-`0600` private `.cleanup.json` record on Colossus. -The record survives retries so stale-lock recovery can clean and verify the same identities. -The cleanup program must not execute OpenShell or any other executable installed in the job workspace. - -The destructive phase must perform these bounded actions: - -1. Stop recorded helper and forward PIDs after verifying the process owner, exact job `HOME`, and command marker. -2. Remove the exact labeled sandbox containers and exact gateway container. -3. Remove the recorded volumes and reserved test image tags. -4. Remove the helper-service directory and `/var/tmp/nemoclaw-jetson-e2e/` workspace for the locked job. -5. Independently verify that the allowlisted resources are absent, host OpenShell remains absent, the protected tools match, and every pre-existing Ollama model name and ID remains. - -The cleanup program must be idempotent. -It must treat an already absent allowlisted resource as success. -It must exit nonzero when target ownership is ambiguous, cleanup fails, or absence verification is inconclusive. -It must not remove `/var/lib/nemoclaw-jetson-dispatch/state/device.lock`. -The dispatcher removes that lock only after cleanup and absence verification succeed. - -The private cleanup record uses this path: - -```text -/var/lib/nemoclaw-jetson-dispatch/state/.cleanup.json -``` - -The file has this exact schema: - -```json -{ - "schemaVersion": 1, - "volumes": ["example-volume"], - "processIds": [1234] -} -``` - -Either identity array can be empty. -The helper merges new identities with the existing record and cleans every recorded identity. -Any helper failure or interruption keeps the device lock for startup recovery. -The dispatcher retains the cleanup record after it removes the device lock. - -After the helper succeeds, the worker independently verifies these conditions over pinned-host-key SSH: - -- The validated job workspace is absent and is not a symbolic link. -- No OpenShell-managed container has the `e2e-jetson-nvmap` sandbox label. -- The `openshell-cluster-nemoclaw` gateway container and volume are absent. -- No `nemoclaw-sandbox-local` image has a tag that begins with `e2e-jetson-nvmap-`. -- Every Docker volume and process ID in the merged cleanup record is absent. -- Every recorded Node.js, npm, and Ollama tool value matches, and every pre-existing Ollama model name and ID remains. -- No host-level `openshell`, `openshell-gateway`, or `openshell-sandbox` binary resolves or exists in a checked host binary path. -- `/dev/nvmap` exists, and Docker still reports the NVIDIA runtime. - -The worker reports cleanup failure when any helper or independent verification step fails. - -The dispatcher runs the cleanup program as `nemoclaw-jetson-dispatch`. -The managed service unit prevents privilege elevation. -Do not remove that protection or grant the dispatcher account passwordless `sudo` access. - -## Configure the Dispatcher Service - -Review these fixed files in the selected commit before deployment: - -- `tools/e2e/colossus-jetson-dispatch.environment` -- `tools/e2e/nemoclaw-jetson-dispatch.service` - -The environment fixes the repository ID, state path, timeouts, Jetson SSH destination, pinned SSH files, and cleanup executable. -The unit fixes the service account, selected-release working directory, Node.js command, restart policy, and filesystem restrictions. -It sets `TimeoutStopSec=360` so systemd allows the bounded cleanup interval during service stop. - -For an initial deployment, the dispatcher service must have `LoadState=not-found`. -The device lock, destination environment file, and destination unit file must be absent. -The Cloudflare Tunnel service must be absent or both disabled and inactive. - -Replace the placeholder with the full lowercase 40-character SHA of the reviewed release, then run one command: - -```bash -sudo nemoclaw-colossus-jetson-dispatch-deploy --commit 0000000000000000000000000000000000000000 -``` - -The command reports five stages: - -1. Validate the request, service-owned home, dispatcher account, SSH key, pinned host key, and Node.js version. -2. Verify the absent service, device lock, destination files, and public ingress. -3. Fetch and verify the exact commit from `https://github.com/NVIDIA/NemoClaw.git`. -4. Select the dispatcher and cleanup code with one atomic `current` switch. -5. Install, enable, start, and verify the dispatcher service. - -The command accepts an absent cleanup link or verifies the exact managed link before release selection. -It stores each verified release at `/opt/nemoclaw-jetson-dispatch/releases/`. -Each release directory must be owned by `root:root` with mode `0755`. -The command verifies the fixed origin, exact commit, unmodified worktree, cleanup program, environment, and unit before selection. -It installs `/etc/nemoclaw-jetson-dispatch/environment` as `root:root` mode `0600`. -It installs `/etc/systemd/system/nemoclaw-jetson-dispatch.service` as `root:root` mode `0644`. -It runs `systemctl daemon-reload` and `systemctl enable --now nemoclaw-jetson-dispatch.service`. -It requires the Cloudflare Tunnel service to be absent or disabled and inactive immediately before the dispatcher starts, immediately after it starts, and after loopback verification. -Before success, it also requires the dispatcher to listen only on `127.0.0.1:8787` and an anonymous job request to return HTTP `401`. - -If initial startup or verification fails after enablement, the command attempts to disable and stop the service before rollback. -It restores the prior release and cleanup selection only after proving exact `ActiveState=inactive` and an absent device lock. -An earlier selection or configuration-installation failure restores the prior selection and removes only files or links created by the attempt. -Rollback removes any environment and unit that the attempt installed, reloads systemd, and requires `LoadState=not-found`. -If it cannot establish these conditions, it reports rollback failure. -If initial activation detects public ingress and rollback cannot disable and stop the dispatcher, its state is inconclusive. -Immediately disable or delete public ingress through the approved Cloudflare administrator path and prove that the public endpoint is unavailable before local recovery. -The command never enables or starts the Cloudflare Tunnel service. -It accepts no cleanup override and does not change the cleanup allowlist. - -After the command returns, test cleanup-program access and the SSH path as the service account: - -```bash -sudo -u nemoclaw-jetson-dispatch test -x \ - /usr/local/libexec/nemoclaw-jetson-cleanup -sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -o BatchMode=yes \ - -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ - -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ - -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ - nvidia@192.168.55.1 'uname -m' -``` - -Do not invoke the cleanup program manually without the dispatcher-created device lock and baseline record. -The proof-job procedure below exercises cleanup and verifies its result. - -For independent verification or recovery, repeat the command's loopback checks: - -```bash -set -euo pipefail -test "$(sudo systemctl show --property=ActiveState --value \ - nemoclaw-jetson-dispatch.service)" = active -listeners="$(sudo ss -H -ltn 'sport = :8787')" -test -n "$listeners" -printf '%s\n' "$listeners" | awk '$4 != "127.0.0.1:8787" { exit 1 }' -HTTP_CODE="$( - curl --disable --noproxy '*' --silent --show-error --output /dev/null \ - --write-out '%{http_code}' --max-time 5 --request POST \ - --header 'Content-Type: application/json' \ - --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ - http://127.0.0.1:8787/v1/jobs -)" -test "$HTTP_CODE" = 401 -``` - -A malformed request returns HTTP `400`, and an unknown route returns HTTP `404`. - -## Deploy a Later Reviewed Commit - -This procedure applies after the tunnel has been published once. -Use the installed deployment command for each later reviewed release: - -```bash -( - set -euo pipefail - tunnel_verified=0 - stop_unverified_tunnel() { - original_status=$? - trap - EXIT - if [ "$tunnel_verified" = 1 ]; then - exit "$original_status" - fi - - set +e - sudo systemctl stop nemoclaw-jetson-tunnel.service - stop_status=$? - observed_state="$(sudo systemctl show --property=ActiveState --value \ - nemoclaw-jetson-tunnel.service)" - inspection_status=$? - set -e - - exit_status=$original_status - if [ "$exit_status" -eq 0 ]; then - exit_status=1 - fi - if [ "$inspection_status" -ne 0 ]; then - state_report=unknown - elif [ -n "$observed_state" ]; then - state_report=$observed_state - else - state_report=empty - fi - if [ "$stop_status" -ne 0 ] || [ "$inspection_status" -ne 0 ] || \ - [ "$observed_state" != inactive ]; then - printf 'PUBLIC INGRESS CONTAINMENT FAILED: tunnel stop status=%s; ActiveState=%s; inspection status=%s\n' \ - "$stop_status" "$state_report" "$inspection_status" >&2 - fi - exit "$exit_status" - } - trap stop_unverified_tunnel EXIT - - REVIEWED_COMMIT_SHA=0000000000000000000000000000000000000000 - sudo systemctl stop nemoclaw-jetson-tunnel.service - test "$(sudo systemctl show --property=ActiveState --value \ - nemoclaw-jetson-tunnel.service)" = inactive - sudo /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy \ - --commit "$REVIEWED_COMMIT_SHA" - sudo systemctl start nemoclaw-jetson-tunnel.service - test "$(sudo systemctl show --property=ActiveState --value \ - nemoclaw-jetson-tunnel.service)" = active - PUBLIC_HTTP_CODE="$( - curl --disable --noproxy '*' --silent --show-error --output /dev/null \ - --write-out '%{http_code}' --max-time 10 --request POST \ - --header 'Content-Type: application/json' \ - --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ - https://jetson-e2e.example.com/v1/jobs - )" - test "$PUBLIC_HTTP_CODE" = 401 - tunnel_verified=1 -) -``` - -The tunnel stop and exact `ActiveState=inactive` check happen before the deployer can change the dispatcher release. -The subshell installs an `EXIT` trap before tunnel operations and leaves its `tunnel_verified` sentinel unset until every check succeeds. -On any earlier exit, the trap disables itself, records the tunnel stop status, and independently inspects `ActiveState`. -Only a successful stop, successful inspection, and exact `ActiveState=inactive` confirm local public-ingress containment. -The trap preserves the original nonzero status, or returns `1` for an otherwise successful unverified exit. -When it cannot confirm containment, it reports `PUBLIC INGRESS CONTAINMENT FAILED` with the stop status and observed or unknown state. -The deployer completes only after the dispatcher starts and its local loopback bind and anonymous HTTP `401` checks pass. -Only after that successful return does the block start the tunnel, require `ActiveState=active`, and repeat a config-free, direct anonymous HTTP `401` check through the public endpoint. -The tunnel remains running only when the block sets `tunnel_verified=1` after all three checks succeed. - -It accepts only systemd `LoadState` values `loaded` and `not-found`. -Both cases require `device.lock` to be absent before release preparation. -A loaded service is eligible only when `current` selects one managed release and the stable cleanup link follows `current`. -For a loaded service, the command performs these actions: - -1. Require the Cloudflare Tunnel service to be disabled and inactive, then stop the dispatcher. -2. Query dispatcher `ActiveState` and require the exact value `inactive`. -3. Refuse deployment when `/var/lib/nemoclaw-jetson-dispatch/state/device.lock` remains. -4. Validate the managed `current` release and stable cleanup link, then prepare and verify the requested release. -5. Require the tunnel to remain disabled and inactive, then atomically switch `current`, selecting the requested dispatcher code and cleanup program together. -6. Require the tunnel to remain disabled and inactive immediately before and after starting the dispatcher. -7. Verify that only `127.0.0.1:8787` listens and make a config-free, direct request with `curl --disable --noproxy '*'`; the anonymous job request must return HTTP `401`. -8. Require the tunnel to remain disabled and inactive after loopback verification. - -If the stop or post-stop inspection cannot prove `inactive`, dispatcher state is inconclusive and the command does not prepare or select a release. -If `device.lock` remains after a proven stop, the command exits with the service stopped and does not prepare or select a release. -Do not remove the lock. -Complete the recovery procedure on this page before you rerun deployment. -If the loaded service's pre-stop public-ingress check fails, the previously verified dispatcher remains running and unchanged. - -An interruption before the atomic `current` switch leaves the previous dispatcher and cleanup program selected; an interruption after it selects both from the requested release. -If activation, start, or loopback verification fails for a loaded service, rollback must first stop the service successfully, require `ActiveState=inactive`, and require `device.lock` to be absent. -Only then does it atomically restore the prior `current` selection, which restores both the dispatcher and cleanup program. -If rollback cannot prove those three conditions after the requested pair is selected, it leaves that new code and cleanup pair selected and reports rollback failure. -When an earlier selected release exists, it restarts that release and repeats the loopback verification. -That rollback restart passes the same tunnel checks before and after start and after loopback verification. -If public ingress activates after a dispatcher start, the command attempts to stop it and does not restart either release while ingress remains active. -If that stop, the `ActiveState` inspection, or the device-lock check fails, dispatcher state is inconclusive. -When no prior selection exists, rollback removes the new `current` and stable cleanup link. -If rollback cannot restore a verified service, the command exits with an explicit rollback failure. -When the trap confirms `ActiveState=inactive`, keep the tunnel stopped while you complete dispatcher recovery. -If it reports `PUBLIC INGRESS CONTAINMENT FAILED`, do not assume that the tunnel stopped. -Immediately disable or delete the public ingress through the approved Cloudflare administrator path and confirm that the public endpoint is unavailable. -Keep the dispatcher stopped until the tunnel is confirmed disabled and inactive and the public endpoint is unavailable. -Then rerun the complete later-deployment subshell instead of starting the dispatcher manually. -The deployer repeats the ingress gates and local bind and authentication verification before the subshell can publish the tunnel again. -Its exit trap keeps the tunnel stopped unless `ActiveState=active` and the config-free, direct public HTTP `401` proof both succeed. - -A later deployment does not modify the dispatcher environment, systemd unit, SSH credentials, or Cloudflare credentials. -It does not update itself. -Normal deployment requires the selected release to preserve the documented cleanup allowlist. -A cleanup-scope change requires separate review and a corresponding runbook update before deployment. -After every later deployment, repeat the authenticated successful, controlled-failure, and cancellation proof jobs under [Configure GitHub and Run a Proof Job](#configure-github-and-run-a-proof-job). - -## Publish the Dispatcher With Cloudflare Tunnel - -Install `cloudflared` from the approved package source for Colossus. -The next commands create a public DNS route to the authenticated dispatcher. -Do not create the route until the loopback authentication check returns HTTP `401` for an anonymous request. -Use an administrator account to create one named tunnel and route one DNS hostname: - -```bash -cloudflared tunnel login -cloudflared tunnel create nemoclaw-jetson-dispatch -cloudflared tunnel route dns nemoclaw-jetson-dispatch jetson-e2e.example.com -``` - -Record the tunnel UUID from the create command. -Substitute it for `TUNNEL_UUID` in each later command and file. -Create a dedicated `cloudflared` service account if the approved package did not create one: - -```bash -sudo useradd --system --create-home --home-dir /var/lib/cloudflared \ - --shell /usr/sbin/nologin cloudflared -``` - -Install only the tunnel credential for that account: - -```bash -sudo install -d -o cloudflared -g cloudflared -m 0700 /etc/cloudflared -sudo install -o cloudflared -g cloudflared -m 0600 \ - "$HOME/.cloudflared/TUNNEL_UUID.json" /etc/cloudflared/TUNNEL_UUID.json -sudo -u cloudflared test -r /etc/cloudflared/TUNNEL_UUID.json -rm "$HOME/.cloudflared/cert.pem" "$HOME/.cloudflared/TUNNEL_UUID.json" -``` - -The removed `cert.pem` authorizes Cloudflare account management. -Do not give it to the tunnel service account. - -Create `/etc/cloudflared/jetson-dispatch.yml` with mode `0600` and the tunnel service account as owner: - -```yaml -tunnel: TUNNEL_UUID -credentials-file: /etc/cloudflared/TUNNEL_UUID.json -ingress: - - hostname: jetson-e2e.example.com - service: http://127.0.0.1:8787 - - service: http_status:404 -``` - -Run `cloudflared` as its own restricted service account. -Install `/etc/systemd/system/nemoclaw-jetson-tunnel.service`: - -```ini -[Unit] -Description=NemoClaw Jetson Cloudflare Tunnel -After=network-online.target nemoclaw-jetson-dispatch.service -Wants=network-online.target - -[Service] -Type=simple -User=cloudflared -Group=cloudflared -ExecStart=/usr/bin/cloudflared --no-autoupdate --config /etc/cloudflared/jetson-dispatch.yml tunnel run -Restart=on-failure -RestartSec=5 -NoNewPrivileges=true -PrivateTmp=true -ProtectHome=true -ProtectSystem=strict - -[Install] -WantedBy=multi-user.target -``` - -Start the tunnel service and inspect its status: - -```bash -sudo systemctl daemon-reload -sudo systemctl enable --now nemoclaw-jetson-tunnel.service -sudo systemctl status nemoclaw-jetson-tunnel.service -``` - -The origin service validates the GitHub OIDC signature and claims on every job, status, cancel, and artifact request. -Cloudflare Tunnel does not replace origin authentication. -Apply a Cloudflare rate limit to `/v1/jobs*` if the managed zone supports one. -Without that rate limit, origin authentication still rejects unauthorized work, but public requests can consume dispatcher connections. - -Confirm the public endpoint also rejects an anonymous request with HTTP `401`. -Confirm Colossus has outbound HTTPS access to GitHub's OIDC key endpoint: - -```bash -PUBLIC_HTTP_CODE="$( - curl --disable --noproxy '*' --silent --show-error --output /dev/null \ - --write-out '%{http_code}' --max-time 10 --request POST \ - --header 'Content-Type: application/json' \ - --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ - https://jetson-e2e.example.com/v1/jobs -)" -test "$PUBLIC_HTTP_CODE" = 401 -curl --fail-with-body \ - https://token.actions.githubusercontent.com/.well-known/jwks -``` - -## Configure GitHub and Run a Proof Job - -The next command updates a repository Actions variable. -Set the public HTTPS origin as a repository Actions variable: - -```bash -gh variable set JETSON_DISPATCH_URL --repo NVIDIA/NemoClaw \ - --body 'https://jetson-e2e.example.com' -``` - -Resolve the non-fork pull request (PR) and trusted workflow identities. -Replace `1234` with the PR number: - -```bash -set -euo pipefail -PR_NUMBER=1234 -PR_JSON="$(gh api "repos/NVIDIA/NemoClaw/pulls/$PR_NUMBER")" -CANDIDATE_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" -CANDIDATE_SHA="$(jq -r .head.sha <<<"$PR_JSON")" -BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" -git fetch --prune origin main -WORKFLOW_SHA="$(git rev-parse origin/main)" -test "$CANDIDATE_REPOSITORY" = NVIDIA/NemoClaw -[[ "$CANDIDATE_SHA" =~ ^[a-f0-9]{40}$ ]] -[[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] -[[ "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] -``` - -Select only the Jetson target from the trusted `main` workflow: - -```bash -gh workflow run .github/workflows/e2e.yaml --repo NVIDIA/NemoClaw --ref main \ - -f targets=jetson-nvmap-gpu \ - -f jobs= \ - -f inference_mode=mock \ - -f include_staging_brev_launchable=false \ - -f allow_jetson_dispatch=true \ - -f allow_dgx_spark_runner_queue=false \ - -f "pr_number=$PR_NUMBER" \ - -f "checkout_sha=$CANDIDATE_SHA" \ - -f "checkout_repository=$CANDIDATE_REPOSITORY" \ - -f "base_sha=$BASE_SHA" \ - -f "workflow_sha=$WORKFLOW_SHA" \ - -f review_reason='Reviewed the commit under review for the isolated Jetson E2E.' -``` - -The `jetson-nvmap-gpu` job uses the fixed `jetson-nvmap-gpu-colossus` concurrency group. -`queue: max` queues every dispatch, and `cancel-in-progress: false` prevents automatic cancellation. - -The controller must run on `ubuntu-latest`. -The GitHub controller log must show one `Jetson dispatch accepted as ` line. -The Colossus journal must not contain a bearer token. -The uploaded `e2e-jetson-nvmap-gpu` artifact must contain `jetson-dispatch.json` with these results: - -- The requested candidate commit SHA and workflow run ID and attempt. -- The Jetson model, JetPack package version or `unavailable`, Jetson Linux release, and kernel. -- The test conclusion and bounded log. -- `status.cleanup: "succeeded"`. -- `status.conclusion: "success"` for the successful proof job. - -Download the trusted `e2e-dispatch--` receipt artifact and the `e2e-jetson-nvmap-gpu` artifact. -Set `RUN_ID` and `RUN_ATTEMPT` from the selected workflow run: - -```bash -EVIDENCE_DIR="$(mktemp -d)" -chmod 700 "$EVIDENCE_DIR" -RUN_ID=123456789 -RUN_ATTEMPT=1 -gh run download "$RUN_ID" --repo NVIDIA/NemoClaw \ - --name "e2e-dispatch-$RUN_ID-$RUN_ATTEMPT" \ - --dir "$EVIDENCE_DIR/trusted-dispatch" -gh run download "$RUN_ID" --repo NVIDIA/NemoClaw \ - --name e2e-jetson-nvmap-gpu \ - --dir "$EVIDENCE_DIR/jetson" -DISPATCH_JSON="$EVIDENCE_DIR/trusted-dispatch/dispatch.json" -JETSON_JSON="$EVIDENCE_DIR/jetson/jetson-dispatch.json" -``` - -Require `dispatch.json` to establish the repository, PR number, base SHA, candidate SHA, and trusted workflow SHA independently: - -```bash -jq -e \ - --arg repository NVIDIA/NemoClaw \ - --arg candidateRepository "$CANDIDATE_REPOSITORY" \ - --arg candidateSha "$CANDIDATE_SHA" \ - --arg baseSha "$BASE_SHA" \ - --arg workflowSha "$WORKFLOW_SHA" \ - --arg workflowRunId "$RUN_ID" \ - --argjson prNumber "$PR_NUMBER" \ - --argjson workflowRunAttempt "$RUN_ATTEMPT" ' - .kind == "nemoclaw-e2e-dispatch-v2" and - .repository == $repository and - .candidateRepository == $candidateRepository and - .candidateRepository == .repository and - .candidateSha == $candidateSha and - .prNumber == $prNumber and - .baseSha == $baseSha and - .workflowSha == $workflowSha and - .workflowRunId == $workflowRunId and - .workflowRunAttempt == $workflowRunAttempt and - .allowJetsonDispatch == true - ' "$DISPATCH_JSON" -``` - -Then bind `jetson-dispatch.json` to that trusted receipt by comparing its candidate SHA and workflow run ID and attempt: - -```bash -jq -e --slurpfile dispatch "$DISPATCH_JSON" ' - .status.request.candidateSha == $dispatch[0].candidateSha and - .status.request.workflowRunId == $dispatch[0].workflowRunId and - .status.request.workflowRunAttempt == $dispatch[0].workflowRunAttempt and - .status.cleanup == "succeeded" and - .status.conclusion == "success" -' "$JETSON_JSON" -``` - -It must also contain `jetson-e2e-artifacts.tar.gz`. -The dispatcher creates that archive from the remote E2E artifact directory before it removes the candidate workspace. -It rejects an artifact directory or compressed archive larger than 1 MiB. -During initialization, the dispatcher validates the schema, request, job ID, and state in every private file whose basename matches `^[a-f0-9]{64}\.json$`. -For a completed record, it also validates the terminal fields. -It ignores valid queued or running records and restores only the 128 newest completed statuses to memory. -An invalid matching file fails initialization before the dispatcher accepts work, and the error names its exact `.json` basename. -Status and artifact requests both restore an evicted completed status from its private file. -Only an artifact request returns the persisted log and archive. -A repeated deterministic dispatch returns the persisted completed status without rerunning candidate code or clearing its evidence. -Before worker execution, the dispatcher creates `device.lock`, persists the queued status, and synchronizes the Colossus state directory. -If queued-state persistence or that directory synchronization fails, candidate code does not run and the dispatcher does not unlink the exact live job lock. -Because a failed directory synchronization cannot prove reboot durability, inspect the live state and restart the dispatcher service to invoke startup cleanup without rebooting the Colossus host. -The dispatcher persists terminal status before it removes `device.lock`. -After removing the lock, it synchronizes the state directory. -If terminal-status persistence fails, it reports a completed in-memory failure and retains the lock. -If the lock-removal directory synchronization fails and restoration succeeds, it re-establishes the exact same job lock, synchronizes the directory again, and reports the lock-removal failure. -If restoration fails, lock state is inconclusive; the dispatcher sets an in-memory recovery barrier, blocks later dispatches, and reports that operator recovery is required during shutdown. - -After the workflow completes, independently verify every allowlisted resource is absent. -Require the private `.cleanup.json` file and verify every recorded volume and process ID is absent. -The private cleanup record is Colossus state and is not part of the uploaded artifact. -Then run one controlled failing candidate. -Confirm that its `jetson-dispatch.json` artifact shows `status.cleanup: "succeeded"` and `status.conclusion: "failure"`. -Require its private cleanup record and verify every recorded identity is absent. -For a cancellation proof, cancel a controller after it logs the job ID. -The controller can exit before it downloads an artifact. -Inspect the private Colossus `.json` status for `conclusion: "cancelled"` and `cleanup: "succeeded"`. -Independently verify the cleanup allowlist after that cancellation. - -These checks establish bounded cleanup of the allowlisted resources. -They do not attest that cleanup reversed every possible host change made by candidate code. - -## Recover or Disable the Deployment - -The dispatcher permits one active job. -A process interruption leaves `device.lock` in the state directory. -On startup, the dispatcher invokes the cleanup program before it removes that stale lock or accepts more work. -After successful cleanup, a valid queued or running record for the locked job becomes a durable completed failure with `cleanup: "succeeded"` and error `Jetson dispatcher restarted before terminal status was persisted`. -Only then does the dispatcher remove the lock; if this recovery persistence fails, the lock remains. -A repeated deterministic dispatch returns that recovered failure without rerunning candidate code. -If cleanup fails, startup fails or the completed job reports `conclusion: "cleanup-failed"` with `cleanup: "failed"`. -If cleanup succeeds but lock removal fails, the completed job reports `conclusion: "cleanup-failed"` with `cleanup: "succeeded"`. -The lock remains after a cleanup failure or successful lock restoration. -A lock-restoration failure does not prove whether the lock exists. -Bounded status error text retains the newest cleanup, persistence, or lock-removal error. - -If the service stops before durable persistence completes, destructive cleanup has not started. -If it stops after durable persistence, the cleanup record and device lock remain for startup recovery. -On startup, the helper discovers current identities and merges them with every retained identity before cleanup resumes. -For example, termination after container removal cannot erase the volume identities recorded before that removal. -The startup cleanup passes those retained volumes into deletion and independent verification. -After the helper succeeds, the worker revalidates its output and every retained identity before the dispatcher removes the lock. - -The unit uses `Restart=on-failure`, so a startup cleanup failure otherwise retries every five seconds. -Stop that retry loop before investigation or repair: - -```bash -sudo systemctl stop nemoclaw-jetson-dispatch.service -``` - -An invalid matching `.json` file also fails startup and enters the same retry loop. -Set `JOB_ID` from the exact basename in the startup error. -For this failure, preserve and inspect the named private file under the approved incident-retention policy; treat its JSON as private candidate output. -The following command validates the exact basename, stops the retry loop, requires `ActiveState=inactive`, and accepts either no `device.lock` or the service-owned, mode `0600`, single-link regular lock whose exact contents are `JOB_ID` plus its terminating newline. -A mismatched, nonregular, or symbolic-link lock blocks recovery. -It rejects a symbolic link or non-directory quarantine parent and verifies `root:root` mode `0700` without following links. -It then moves the exact object without overwriting retained evidence and restarts the service. -For a matching lock, startup reruns cleanup and removes the lock only after cleanup succeeds; never remove the lock manually. -After it returns, repeat the loopback-bind and anonymous HTTP `401` checks under [Configure the Dispatcher Service](#configure-the-dispatcher-service) before accepting work. - -```bash -set -euo pipefail -JOB_ID=0000000000000000000000000000000000000000000000000000000000000000 -[[ "$JOB_ID" =~ ^[a-f0-9]{64}$ ]] -state=/var/lib/nemoclaw-jetson-dispatch/state -status_file="$state/$JOB_ID.json" -quarantine=/var/lib/nemoclaw-jetson-status-quarantine -quarantine_file="$quarantine/$JOB_ID.json.invalid" -lock="$state/device.lock" -dispatcher_uid="$(id -u nemoclaw-jetson-dispatch)" -dispatcher_gid="$(id -g nemoclaw-jetson-dispatch)" -sudo systemctl stop nemoclaw-jetson-dispatch.service -test "$(sudo systemctl show --property=ActiveState --value \ - nemoclaw-jetson-dispatch.service)" = inactive -sudo stat -- "$status_file" -sudo test ! -L "$lock" -if sudo test -e "$lock"; then - test "$(sudo stat --format='%F:%u:%g:%a:%h' -- "$lock")" = \ - "regular file:$dispatcher_uid:$dispatcher_gid:600:1" - printf '%s\n' "$JOB_ID" | sudo cmp --silent -- "$lock" - -fi -sudo test ! -L "$quarantine" -if sudo test -e "$quarantine"; then - test "$(sudo stat --format='%F:%u:%g:%a' -- "$quarantine")" = \ - directory:0:0:700 -else - sudo install -d -o root -g root -m 0700 "$quarantine" -fi -test "$(sudo stat --format='%F:%u:%g:%a' -- "$quarantine")" = \ - directory:0:0:700 -sudo test ! -e "$quarantine_file" -sudo test ! -L "$quarantine_file" -sudo mv -- "$status_file" "$quarantine_file" -sudo test ! -e "$status_file" -sudo test ! -L "$status_file" -sudo systemctl start nemoclaw-jetson-dispatch.service -test "$(sudo systemctl show --property=ActiveState --value \ - nemoclaw-jetson-dispatch.service)" = active -``` - -Do not delete `device.lock` to bypass recovery. -For `cleanup: "failed"`, inspect the recorded error before choosing a recovery action. -Repair the cleanup program or allowlisted resource state only when that named operation failed. -If the protected tool or Ollama model baseline differs after cleanup, investigate candidate activity and external host drift without assigning the change to cleanup. -For `cleanup: "succeeded"` with a lock-removal error, repair the state-directory filesystem or permissions. -If the error also reports lock-restoration failure, stop the dispatcher and treat the exact `device.lock` state as inconclusive. -Use the ownership, file-type, mode, and exact job-ID checks above to inspect it without following links; do not accept more work until approved recovery and a service restart complete startup cleanup. -Start the dispatcher after the named condition is fixed: - -```bash -sudo systemctl start nemoclaw-jetson-dispatch.service -``` - -Startup runs cleanup and absence verification again before it removes the stale lock. - -Do not dispatch another job when cleanup verification is inconclusive. -Manual recovery must stay within the same cleanup allowlist. -Escalate any suspected protected baseline change for separate host investigation. -Cleanup evidence alone is not evidence that every candidate host change was reversed. - -Retain every `.cleanup.json` file until the temporary path teardown completes. -Preserve the other private job state required for diagnosis and stale-lock recovery. -These private state files can contain candidate output, and the dispatcher does not otherwise prune them. -Apply the host's approved retention policy to other private state after GitHub uploads the artifact and no device lock exists. - -To disable the temporary path, first prevent another controller from reaching the dispatcher. -Delete the repository variable and stop the public tunnel, but keep the dispatcher and its SSH credentials available while any accepted job finishes its bounded execution and cleanup: - -```bash -gh variable delete JETSON_DISPATCH_URL --repo NVIDIA/NemoClaw -sudo systemctl disable --now nemoclaw-jetson-tunnel.service -``` - -Use the last accepted job ID from the controller log. -Wait for the device lock to disappear and require that job's private status to report successful cleanup. -Require the last job's cleanup record. -Use Node.js `readdirSync` with the exact `^[a-f0-9]{64}\.cleanup\.json$` basename pattern. -Validate and aggregate every retained cleanup record that matches this pattern. -Independently verify every aggregated resource identity and the remaining fixed allowlist: - -```bash -set -euo pipefail -LAST_JOB_ID=0000000000000000000000000000000000000000000000000000000000000000 -[[ "$LAST_JOB_ID" =~ ^[a-f0-9]{64}$ ]] -state=/var/lib/nemoclaw-jetson-dispatch/state -sudo -u nemoclaw-jetson-dispatch \ - timeout 3600 bash -c 'while [ -e "$1" ]; do sleep 5; done' wait-lock \ - "$state/device.lock" -sudo -u nemoclaw-jetson-dispatch test ! -e "$state/device.lock" -sudo -u nemoclaw-jetson-dispatch /usr/bin/node -e ' - const fs = require("node:fs"); - const status = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); - if (status.state !== "completed" || status.cleanup !== "succeeded") process.exit(1); -' "$state/$LAST_JOB_ID.json" -cleanup_identities="$(sudo -u nemoclaw-jetson-dispatch /usr/bin/node -e ' - const fs = require("node:fs"); - const path = require("node:path"); - const stateDirectory = process.argv[1]; - const lastJobId = process.argv[2]; - const cleanupName = /^[a-f0-9]{64}\.cleanup\.json$/; - const names = fs.readdirSync(stateDirectory).filter((name) => cleanupName.test(name)).sort(); - if (!names.includes(`${lastJobId}.cleanup.json`)) process.exit(1); - const expectedKeys = ["processIds", "schemaVersion", "volumes"]; - const validVolume = (value) => - typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(value); - const validProcessId = (value) => Number.isSafeInteger(value) && value > 0; - const volumes = new Set(); - const processIds = new Set(); - for (const name of names) { - const file = path.join(stateDirectory, name); - const metadata = fs.lstatSync(file); - if (!metadata.isFile() || (metadata.mode & 0o777) !== 0o600) process.exit(1); - const raw = fs.readFileSync(file, "utf8"); - if (Buffer.byteLength(raw) > 64 * 1024) process.exit(1); - const record = JSON.parse(raw); - const keys = Object.keys(record).sort(); - if ( - JSON.stringify(keys) !== JSON.stringify(expectedKeys) || - record.schemaVersion !== 1 || - !Array.isArray(record.volumes) || - !record.volumes.every(validVolume) || - !Array.isArray(record.processIds) || - !record.processIds.every(validProcessId) - ) process.exit(1); - for (const volume of record.volumes) volumes.add(volume); - for (const processId of record.processIds) processIds.add(processId); - } - for (const volume of [...volumes].sort()) console.log(`volume\t${volume}`); - for (const processId of [...processIds].sort((a, b) => a - b)) { - console.log(`processId\t${processId}`); - } -' "$state" "$LAST_JOB_ID")" -cleanup_volumes=() -cleanup_process_ids=() -while IFS=$'\t' read -r identity_kind identity; do - [ -n "$identity_kind" ] || continue - case "$identity_kind" in - volume) cleanup_volumes+=("$identity") ;; - processId) cleanup_process_ids+=("$identity") ;; - *) exit 1 ;; - esac -done <<<"$cleanup_identities" -for volume in "${cleanup_volumes[@]}"; do - timeout --kill-after=5 120 \ - sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ - -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ - -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ - -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ - -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ - nvidia@192.168.55.1 bash -s -- "$volume" <<'VERIFY_RECORDED_VOLUME' -set -euo pipefail -volume_names="$(docker volume ls --format '{{.Name}}')" || { - echo 'Unable to list Docker volumes' >&2 - exit 1 -} -if printf '%s\n' "$volume_names" | grep -Fqx -- "$1"; then - echo "A recorded job-owned Docker volume remains: $1" >&2 - exit 1 -fi -VERIFY_RECORDED_VOLUME -done -for process_id in "${cleanup_process_ids[@]}"; do - timeout --kill-after=5 120 \ - sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ - -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ - -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ - -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ - -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ - nvidia@192.168.55.1 bash -s -- "$process_id" <<'VERIFY_RECORDED_PROCESS' -set -euo pipefail -if [ -e "/proc/$1" ]; then - echo "A recorded job-owned process ID remains: $1" >&2 - exit 1 -fi -VERIFY_RECORDED_PROCESS -done -timeout --kill-after=5 120 \ - sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ - -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ - -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ - -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ - -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ - nvidia@192.168.55.1 bash -s -- "$LAST_JOB_ID" <<'VERIFY_JETSON_IDLE' -set -euo pipefail -job_id="$1" -[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] -job_home="/var/tmp/nemoclaw-jetson-e2e/$job_id/home" -test ! -e "/var/tmp/nemoclaw-jetson-e2e/$job_id" -test ! -e /tmp/nemoclaw-services-e2e-jetson-nvmap -if [ -e /var/tmp/nemoclaw-jetson-e2e ]; then - test -d /var/tmp/nemoclaw-jetson-e2e - test ! -L /var/tmp/nemoclaw-jetson-e2e - test -z "$(find /var/tmp/nemoclaw-jetson-e2e -mindepth 1 -maxdepth 1 -print -quit)" -fi -sandbox_container_output="$(docker ps -aq \ - --filter label=openshell.ai/managed-by=openshell \ - --filter label=openshell.ai/sandbox-name=e2e-jetson-nvmap)" || { - echo 'Unable to list labeled sandbox containers' >&2 - exit 1 -} -test -z "$sandbox_container_output" -container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { - echo 'Unable to list Docker containers' >&2 - exit 1 -} -if printf '%s\n' "$container_rows" | - awk -F '\t' '$2 == "openshell-cluster-nemoclaw" { found = 1 } END { exit found ? 0 : 1 }'; then - echo 'The job-owned gateway container remains' >&2 - exit 1 -fi -volume_names="$(docker volume ls --format '{{.Name}}')" || { - echo 'Unable to list Docker volumes' >&2 - exit 1 -} -if printf '%s\n' "$volume_names" | grep -Fqx openshell-cluster-nemoclaw; then - echo 'The job-owned gateway volume remains' >&2 - exit 1 -fi -read_proc_uid() { - awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' \ - "$1/status" 2>/dev/null -} -read_proc_environment() { - dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' -} -read_proc_command() { - dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' -} -handle_proc_read_failure() { - local proc_dir="$1" field="$2" process_uid directory_uid - [ -d "$proc_dir" ] || return 0 - if process_uid="$(read_proc_uid "$proc_dir")"; then - [ "$process_uid" = "$(id -u)" ] || return 0 - else - [ -d "$proc_dir" ] || return 0 - if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then - [ -d "$proc_dir" ] || return 0 - echo "Unable to verify the owner of a live process after a failed $field read" >&2 - exit 1 - fi - [ "$directory_uid" = "$(id -u)" ] || return 0 - fi - echo "Unable to inspect $field for a live same-user process" >&2 - exit 1 -} -for proc_dir in /proc/[0-9]*; do - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" owner - continue - fi - [ "$process_uid" = "$(id -u)" ] || continue - if ! environment="$(read_proc_environment "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" environment - continue - fi - printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue - if ! cmdline="$(read_proc_command "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" command - continue - fi - case "$cmdline" in - *ollama-auth-proxy.*|*openshell-gateway*|*openshell-forward*|*openshell\ forward*|*cloudflared*) - echo "A job-owned helper process remains: ${proc_dir##*/}" >&2 - exit 1 - ;; - esac -done -image_rows="$(docker image ls nemoclaw-sandbox-local --format '{{.Repository}}\t{{.Tag}}')" || { - echo 'Unable to list job-owned Docker images' >&2 - exit 1 -} -test -z "$(printf '%s\n' "$image_rows" | - awk '$1 == "nemoclaw-sandbox-local" && index($2, "e2e-jetson-nvmap-") == 1 { print $1 ":" $2 }')" -command -v node npm ollama -timeout --kill-after=5 30 ollama list >/dev/null -for openshell_component in openshell openshell-gateway openshell-sandbox; do - if command -v "$openshell_component" >/dev/null 2>&1; then - echo "A host-level OpenShell binary remains after cleanup: $openshell_component" >&2 - exit 1 - fi - for host_bin in \ - "/usr/local/bin/$openshell_component" \ - "/usr/bin/$openshell_component" \ - "$HOME/.local/bin/$openshell_component"; do - test ! -e "$host_bin" - test ! -L "$host_bin" - done -done -test -c /dev/nvmap -case "$(docker info --format '{{json .Runtimes}}')" in - *nvidia*) ;; - *) exit 1 ;; -esac -VERIFY_JETSON_IDLE -``` - -If the last cleanup record is missing, keep the dispatcher and SSH credentials. -Keep them when any retained cleanup record is malformed or an absence check fails. -Use the recovery procedure above before teardown. -Only after every check passes should you stop the dispatcher: - -```bash -sudo systemctl disable --now nemoclaw-jetson-dispatch.service -``` - -Reauthenticate with `cloudflared tunnel login` as a Cloudflare administrator. -Delete the `jetson-e2e.example.com` DNS record from the Cloudflare zone. -Delete the named tunnel with `cloudflared tunnel delete TUNNEL_UUID`. -Confirm that `cloudflared tunnel list` no longer returns `TUNNEL_UUID`. -Remove the new local account certificate after the administrative deletion. -Remove the local tunnel credential, configuration, and service unit: - -```bash -rm "$HOME/.cloudflared/cert.pem" -sudo rm /etc/cloudflared/TUNNEL_UUID.json \ - /etc/cloudflared/jetson-dispatch.yml \ - /etc/systemd/system/nemoclaw-jetson-tunnel.service -sudo systemctl daemon-reload -test ! -e /etc/cloudflared/TUNNEL_UUID.json -test ! -e /etc/cloudflared/jetson-dispatch.yml -test ! -e /etc/systemd/system/nemoclaw-jetson-tunnel.service -``` - -Confirm that the public endpoint is unreachable and the repository variable is absent: - -```bash -if curl --silent --show-error --output /dev/null --connect-timeout 10 \ - https://jetson-e2e.example.com/v1/jobs; then - echo 'Jetson dispatch endpoint is still reachable' >&2 - exit 1 -fi -test -z "$(gh variable list --repo NVIDIA/NemoClaw --json name \ - --jq '.[] | select(.name == "JETSON_DISPATCH_URL") | .name')" -``` - -Remove the dedicated Jetson public key and the Colossus SSH private key. -Encode the exact authorized-key line so the space-containing value remains one remote-command argument, then verify that exact line is absent before deleting the Colossus key: - -```bash -set -euo pipefail -JETSON_PUBLIC_KEY="$(sudo cat /var/lib/nemoclaw-jetson-dispatch/id_ed25519.pub)" -case "$JETSON_PUBLIC_KEY" in - ssh-ed25519\ *) ;; - *) echo 'Unexpected Jetson public key format' >&2; exit 1 ;; -esac -JETSON_AUTHORIZED_KEY_LINE="from=\"192.168.55.100\",restrict $JETSON_PUBLIC_KEY" -JETSON_AUTHORIZED_KEY_B64="$(printf '%s' "$JETSON_AUTHORIZED_KEY_LINE" | base64 --wrap=0)" -timeout --kill-after=5 120 \ - sudo -u nemoclaw-jetson-dispatch ssh -F /dev/null -T \ - -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ - -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \ - -o UserKnownHostsFile=/var/lib/nemoclaw-jetson-dispatch/known_hosts \ - -i /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ - nvidia@192.168.55.1 bash -s -- "$JETSON_AUTHORIZED_KEY_B64" <<'REMOVE_DISPATCH_KEY' -set -euo pipefail -authorized_key_line="$(printf '%s' "$1" | base64 --decode)" -authorized_keys="$HOME/.ssh/authorized_keys" -test -f "$authorized_keys" -temporary="$(mktemp "$HOME/.ssh/.authorized_keys.XXXXXX")" -trap 'rm -f "$temporary"' EXIT -chmod 600 "$temporary" -while IFS= read -r line || [ -n "$line" ]; do - if [ "$line" != "$authorized_key_line" ]; then - printf '%s\n' "$line" - fi -done <"$authorized_keys" >"$temporary" -mv "$temporary" "$authorized_keys" -trap - EXIT -if grep -Fqx -- "$authorized_key_line" "$authorized_keys"; then - echo 'Dedicated Jetson public key remains authorized' >&2 - exit 1 -fi -REMOVE_DISPATCH_KEY -sudo rm -- \ - /var/lib/nemoclaw-jetson-dispatch/id_ed25519 \ - /var/lib/nemoclaw-jetson-dispatch/id_ed25519.pub -sudo test ! -e /var/lib/nemoclaw-jetson-dispatch/id_ed25519 -sudo test ! -e /var/lib/nemoclaw-jetson-dispatch/id_ed25519.pub -``` - -An SSH transport failure keeps the Colossus SSH key and pinned host-key files. -The Jetson `authorized_keys` result is inconclusive in this case. -Use the serial console to confirm that the exact key line is absent before you continue credential removal. - -Remove the cleanup executable and pinned SSH host-key file, then verify their absence: - -```bash -sudo rm -- \ - /usr/local/libexec/nemoclaw-jetson-cleanup \ - /var/lib/nemoclaw-jetson-dispatch/known_hosts -sudo test ! -e /usr/local/libexec/nemoclaw-jetson-cleanup -sudo test ! -L /usr/local/libexec/nemoclaw-jetson-cleanup -sudo test ! -e /var/lib/nemoclaw-jetson-dispatch/known_hosts -``` - -After the required retention period, remove the private job state and logs. -When no investigation, recovery, or retention requirement remains, remove these remaining deployment files: - -```bash -sudo rm -- \ - /etc/nemoclaw-jetson-dispatch/environment \ - /etc/systemd/system/nemoclaw-jetson-dispatch.service \ - /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy -sudo rm -rf -- \ - /opt/nemoclaw-jetson-dispatch \ - /var/lib/nemoclaw-jetson-status-quarantine -sudo systemctl daemon-reload -sudo test ! -e /etc/nemoclaw-jetson-dispatch/environment -sudo test ! -e /etc/systemd/system/nemoclaw-jetson-dispatch.service -sudo test ! -e /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy -sudo test ! -e /opt/nemoclaw-jetson-dispatch -sudo test ! -L /opt/nemoclaw-jetson-dispatch -sudo test ! -e /var/lib/nemoclaw-jetson-status-quarantine -sudo test ! -L /var/lib/nemoclaw-jetson-status-quarantine -test "$(systemctl show --property=LoadState --value \ - nemoclaw-jetson-dispatch.service)" = not-found -``` diff --git a/test/e2e/docs/jetson-dispatch.md b/test/e2e/docs/jetson-dispatch.md new file mode 100644 index 00000000000..ce53910ebf4 --- /dev/null +++ b/test/e2e/docs/jetson-dispatch.md @@ -0,0 +1,144 @@ + + + +# Jetson Dispatch Controller + +NemoClaw owns the trusted GitHub Actions controller for the +`jetson-nvmap-gpu` live E2E target. An operator-owned service runs the target on +a Jetson device. This page defines the versioned HTTP boundary, GitHub +configuration, and evidence that remain in NemoClaw. It does not define how to +deploy or operate the service. + +## Owned Surface + +NemoClaw owns these files and settings: + +- `.github/workflows/e2e.yaml` selects the fixed target from the trusted + workflow on `main`. +- `tools/e2e/jetson-dispatch-client.mts` obtains a GitHub OpenID Connect (OIDC) + token, sends the exact candidate commit, validates responses, and writes + bounded evidence. +- `tools/e2e/jetson-dispatch-contract.mts` implements HTTP contract version + `1.0.0`. +- `tools/e2e/contracts/v1/jetson-dispatch.json` contains the shared static + compatibility vectors for contract version `1.0.0`. +- The repository variable `JETSON_DISPATCH_URL` selects the operator-provided + service origin. +- `test/e2e/live/jetson-nvmap-gpu.test.ts` defines the live target. + +The service implementation and device lifecycle are operator-owned +infrastructure. NemoClaw has no build-time dependency on that implementation. + +## HTTP Contract 1.0.0 + +Contract version `1.0.0` uses JSON with `schemaVersion: 1`. The controller sends +a request body with these exact fields: + +```json +{ + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "", + "workflowRunId": "", + "workflowRunAttempt": 1 +} +``` + +The request has no command, repository, ref, or free-form target field. The +controller rejects missing and extra fields, any target other than +`jetson-nvmap-gpu`, a noncanonical commit SHA, and invalid workflow-run values. + +The controller uses these endpoints: + +| Method | Path | Purpose | +| --- | --- | --- | +| `POST` | `/v1/jobs` | Submit the exact request and receive its initial status. | +| `GET` | `/v1/jobs/{jobId}` | Poll the queued, running, or completed status. | +| `DELETE` | `/v1/jobs/{jobId}` | Request cancellation after a signal, deadline, or repeated poll failure. | +| `GET` | `/v1/jobs/{jobId}/artifact` | Read the completed status, bounded log, and optional artifact archive. | + +Job creation and status responses wrap the status as `{ "job": }`. +The `jobId` is the lowercase SHA-256 digest of the colon-separated request +tuple `schemaVersion`, `target`, `candidateSha`, `workflowRunId`, and +`workflowRunAttempt`. The controller verifies that relationship before it +accepts a status. + +A status advances through `queued`, `running`, and `completed`. Queued and +running jobs have `cleanup: "pending"`. A completed job has one of these +conclusions: + +- `success` +- `failure` +- `cancelled` +- `timed-out` +- `cleanup-failed` + +A completed job reports cleanup as `succeeded` or `failed`. A failed cleanup +requires `conclusion: "cleanup-failed"`. That conclusion can report cleanup as +`succeeded` when job cleanup finished but a later lifecycle step, such as +device-lock release, failed. A successful result includes the bounded device +identity fields `model`, `jetpackVersion`, +`jetsonLinuxRelease`, and `kernel`. + +The artifact response contains the same completed status and a log of at most +4 MiB. It can contain a canonical base64 artifact archive of at most 1 MiB. A +successful result must contain the archive. The client rejects unknown fields, +invalid timestamps, inconsistent states, a mismatched job ID, oversized +content, and a noncanonical archive. + +The static vectors in +`tools/e2e/contracts/v1/jetson-dispatch.json` are the compatibility boundary +shared with the operator-owned service. They include one request, queued and +completed responses, one artifact, and rejected request examples. Change the +contract version, parser tests, client tests, and vectors together when an +incompatible wire change is required. + +## Trusted GitHub Dispatch + +The Jetson job runs only for an explicit `workflow_dispatch` on the trusted +`main` workflow in `NVIDIA/NemoClaw`. The workflow requires +`allow_jetson_dispatch=true`, limits candidate checkout to the same repository, +and sends the exact `checkout_sha` or current trusted ref commit as +`candidateSha`. GitHub queues later Jetson jobs instead of canceling a running +job. + +The job grants only `contents: read` and `id-token: write`. The controller +requests a short-lived GitHub OIDC token with audience +`nemoclaw-jetson-dispatch` and sends it as a bearer token on every service +request. It does not store the token in the uploaded evidence. + +Configure `JETSON_DISPATCH_URL` as a GitHub repository variable. The client +requires an HTTPS origin without user information, a path, a query, or a +fragment. Do not put a token or other credential in that variable. + +The operator-owned service must already be available and compatible with +contract version `1.0.0` before a maintainer enables the dispatch flag. Keep +the flag disabled for ordinary runs and required pre-tag evidence. + +## Evidence + +The trusted workflow records `dispatch.json` in the +`e2e-dispatch--` artifact before candidate execution. That +receipt binds the candidate repository and SHA, base SHA, trusted workflow SHA, +workflow run ID and attempt, selectors, event, and hardware opt-in decisions. + +The Jetson controller writes private files under the target artifact directory: + +- `jetson-dispatch.json` contains the validated completed status and bounded + log. It excludes the base64 archive payload. +- `jetson-e2e-artifacts.tar.gz` contains the decoded target evidence when the + service returns an archive. + +The workflow uploads that directory as `e2e-jetson-nvmap-gpu`, including on job +failure. A successful proof requires the exact candidate request, a conclusion +of `success`, `cleanup: "succeeded"`, a device identity, and the artifact +archive. + +## Live Target + +`test/e2e/live/jetson-nvmap-gpu.test.ts` runs the supported Jetson hardware +proof against the candidate checkout. It verifies the Jetson hardware gate, +Docker NVIDIA runtime, real noninteractive NemoClaw installation, job-local +installed commands, sandbox access to `/dev/nvmap`, successful CUDA +initialization, and the verified GPU status reported by NemoClaw. The test +records phase evidence through the shared live E2E artifact fixtures. diff --git a/test/e2e/support/colossus-jetson-dispatch-deploy.test.ts b/test/e2e/support/colossus-jetson-dispatch-deploy.test.ts deleted file mode 100644 index 365642f4fbd..00000000000 --- a/test/e2e/support/colossus-jetson-dispatch-deploy.test.ts +++ /dev/null @@ -1,833 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { execFileSync, spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, describe, expect, it } from "vitest"; - -const DEPLOY_SCRIPT = path.join(process.cwd(), "tools/e2e/colossus-jetson-dispatch-deploy.sh"); -const ENVIRONMENT_SOURCE = path.join( - process.cwd(), - "tools/e2e/colossus-jetson-dispatch.environment", -); -const FIXED_ORIGIN = "https://github.com/NVIDIA/NemoClaw.git"; -const UNIT_SOURCE = path.join(process.cwd(), "tools/e2e/nemoclaw-jetson-dispatch.service"); -const temporaryDirectories: string[] = []; - -const HARNESS = String.raw` -set -euo pipefail -source "$DEPLOY_SCRIPT" - -deploy_root="$TEST_ROOT/opt/nemoclaw-jetson-dispatch" -releases_directory="$deploy_root/releases" -current_link="$deploy_root/current" -deploy_lock="$deploy_root/deploy.lock" -cleanup_executable="$TEST_ROOT/usr/local/libexec/nemoclaw-jetson-cleanup" -cleanup_link_target="$current_link/$cleanup_relative" -service_home="$TEST_ROOT/var/lib/nemoclaw-jetson-dispatch" -state_directory="$service_home/state" -device_lock="$state_directory/device.lock" -ssh_identity_file="$service_home/id_ed25519" -ssh_known_hosts_file="$service_home/known_hosts" -node_executable="$NODE_EXECUTABLE" -environment_file="$TEST_ROOT/etc/nemoclaw-jetson-dispatch/environment" -unit_file="$TEST_ROOT/etc/systemd/system/$service_name" - -mkdir -p "$state_directory" "${"$"}{environment_file%/*}" "${"$"}{unit_file%/*}" -printf 'test-private-key\n' >"$ssh_identity_file" -printf 'test-known-host\n' >"$ssh_known_hosts_file" -[ "$SERVICE_LOAD_STATE" != loaded ] || : >"$TEST_ROOT/service-active" -[ "$INITIAL_DEVICE_LOCK" != 1 ] || printf '%064d\n' 0 >"$device_lock" - -effective_uid() { - printf '%s\n' "$FAKE_UID" -} - -id_exec() { - [ "$MISSING_ACCOUNT" != 1 ] || return 1 - printf '1001\n' -} - -require_service_owned_file() { - [ -f "$1" ] && [ ! -L "$1" ] -} - -require_service_owned_directory() { - [ "$INVALID_SERVICE_HOME" != 1 ] || fail "$1 must be a directory, not a symbolic link" - [ -d "$1" ] && [ ! -L "$1" ] -} - -ssh_identity_is_valid() { - [ "$INVALID_SSH_KEY" != 1 ] -} - -known_hosts_has_jetson() { - [ "$INVALID_KNOWN_HOSTS" != 1 ] -} - -node_version_is_supported() { - [ "$UNSUPPORTED_NODE" != 1 ] -} - -install_directory() { - /usr/bin/install -d -m "$2" "$1" -} - -install_file() { - /usr/bin/install -m "$3" "$1" "$2" -} - -require_root_owned_directory() { - [ -d "$1" ] && [ ! -L "$1" ] -} - -require_root_owned_file() { - [ -f "$1" ] && [ ! -L "$1" ] -} - -root_owned_file_matches() { - [ -f "$1" ] && [ ! -L "$1" ] -} - -symlink_owner() { - printf '0:0\n' -} - -acquire_deploy_lock() { - : -} - -move_replace() { - if [ "$2" = "$current_link" ] && [ -n "$FAIL_SELECT_SHA" ]; then - candidate="$(/usr/bin/readlink "$1")" - [ "${"$"}{candidate##*/}" != "$FAIL_SELECT_SHA" ] || return 1 - fi - "$NODE_EXECUTABLE" -e 'require("node:fs").renameSync(process.argv[1], process.argv[2])' "$1" "$2" -} - -move_directory() { - /bin/mv "$1" "$2" -} - -copy_file_preserving_metadata() { - /bin/cp -p "$1" "$2" -} - -files_match() { - /usr/bin/cmp -s "$1" "$2" -} - -git_exec() { - { - printf 'git' - printf '\t%s' "$@" - printf '\n' - } >>"$TEST_ROOT/git.log" - case "${"$"}{3:-}" in - fetch) - /usr/bin/git -c protocol.file.allow=always -C "$2" fetch \ - --depth=1 --no-tags "$FIXTURE_REPOSITORY" "${"$"}{!#}" - ;; - *) /usr/bin/git "$@" ;; - esac -} - -service_load_state() { - if [ -f "$unit_file" ]; then - printf 'loaded\n' - else - printf '%s\n' "$SERVICE_LOAD_STATE" - fi -} - -service_active_state() { - [ "$ACTIVE_STATE_ERROR" != 1 ] || return 1 - if [ -f "$TEST_ROOT/service-active" ]; then - printf 'active\n' - else - printf 'inactive\n' - fi -} - -current_sha() { - /usr/bin/readlink "$current_link" 2>/dev/null | /usr/bin/awk -F/ '{print $NF}' -} - -tunnel_is_active() { - tunnel_check_count="$(cat "$TEST_ROOT/tunnel-check-count" 2>/dev/null || printf '0')" - [ "$TUNNEL_ACTIVE" = 1 ] || - { [ "$TUNNEL_ACTIVATE_AFTER_CHECK" -gt 0 ] && - [ "$tunnel_check_count" -gt "$TUNNEL_ACTIVATE_AFTER_CHECK" ]; } -} - -systemctl_exec() { - printf 'systemctl\t%s\n' "$*" >>"$TEST_ROOT/service.log" - case "$1" in - show) - if [ "${"$"}{!#}" = "$tunnel_service_name" ]; then - case "$2" in - --property=LoadState) - tunnel_check_count="$(cat "$TEST_ROOT/tunnel-check-count" 2>/dev/null || printf '0')" - tunnel_check_count="$((tunnel_check_count + 1))" - printf '%s\n' "$tunnel_check_count" >"$TEST_ROOT/tunnel-check-count" - if tunnel_is_active; then printf 'loaded\n'; else printf 'not-found\n'; fi - ;; - --property=ActiveState) - if tunnel_is_active; then printf 'active\n'; else printf 'inactive\n'; fi - ;; - --property=UnitFileState) - if tunnel_is_active; then printf 'enabled\n'; else printf 'disabled\n'; fi - ;; - *) return 1 ;; - esac - else - return 1 - fi - ;; - daemon-reload) : ;; - stop) - stop_count="$(cat "$TEST_ROOT/stop-count" 2>/dev/null || printf '0')" - stop_count="$((stop_count + 1))" - printf '%s\n' "$stop_count" >"$TEST_ROOT/stop-count" - if [ "$FAIL_ROLLBACK_STOP" = 1 ] && [ "$stop_count" -gt 1 ]; then - return 1 - fi - /bin/rm -f "$TEST_ROOT/service-active" - if [ "$LEAVE_DEVICE_LOCK" = 1 ]; then - printf '%064d\n' 0 >"$device_lock" - else - /bin/rm -f "$device_lock" - fi - ;; - enable) - selected="$(current_sha)" - printf 'enable-sha\t%s\n' "$selected" >>"$TEST_ROOT/service.log" - [ "$selected" != "$FAIL_START_SHA" ] || return 1 - : >"$TEST_ROOT/service-active" - ;; - disable) - [ "$FAIL_INITIAL_ROLLBACK_DISABLE" != 1 ] || return 1 - /bin/rm -f "$TEST_ROOT/service-active" - ;; - start) - selected="$(current_sha)" - printf 'start-sha\t%s\n' "$selected" >>"$TEST_ROOT/service.log" - [ "$selected" != "$FAIL_START_SHA" ] || return 1 - : >"$TEST_ROOT/service-active" - ;; - is-active) [ -f "$TEST_ROOT/service-active" ] ;; - *) return 1 ;; - esac -} - -ss_exec() { - if [ "$(current_sha)" = "$FAIL_VERIFY_SHA" ]; then - printf 'LISTEN 0 511 0.0.0.0:8787 0.0.0.0:*\n' - else - printf 'LISTEN 0 511 127.0.0.1:8787 0.0.0.0:*\n' - fi -} - -curl_exec() { - { - printf 'curl' - printf '\t%s' "$@" - printf '\n' - } >>"$TEST_ROOT/curl.log" - if [ "$(current_sha)" = "$FAIL_VERIFY_SHA" ]; then - printf '503' - else - printf '401' - fi -} - -sleep_exec() { - : -} - -main "$@" -`; - -interface Fixture { - directory: string; - repository: string; - firstSha: string; - secondSha: string; -} - -interface DeployOptions { - activeStateError?: boolean; - fakeUid?: string; - failInitialRollbackDisable?: boolean; - failRollbackStop?: boolean; - failSelectSha?: string; - serviceLoadState?: "loaded" | "not-found"; - failStartSha?: string; - failVerifySha?: string; - initialDeviceLock?: boolean; - invalidKnownHosts?: boolean; - invalidServiceHome?: boolean; - invalidSshKey?: boolean; - leaveDeviceLock?: boolean; - missingAccount?: boolean; - tunnelActive?: boolean; - tunnelActivatesAfterCheck?: number; - unsupportedNode?: boolean; -} - -afterEach(() => { - for (const directory of temporaryDirectories.splice(0)) { - fs.rmSync(directory, { force: true, recursive: true }); - } -}); - -function git(repository: string, ...args: string[]): string { - return execFileSync("/usr/bin/git", ["-C", repository, ...args], { - encoding: "utf8", - }).trim(); -} - -function createFixture(): Fixture { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-colossus-deploy-")); - temporaryDirectories.push(directory); - const repository = path.join(directory, "origin"); - fs.mkdirSync(path.join(repository, "tools/e2e"), { recursive: true }); - execFileSync("/usr/bin/git", ["init", "--quiet", repository]); - git(repository, "config", "user.name", "NemoClaw Test"); - git(repository, "config", "user.email", "test@example.com"); - const cleanup = path.join(repository, "tools/e2e/jetson-dispatch-cleanup.sh"); - const environment = path.join(repository, "tools/e2e/colossus-jetson-dispatch.environment"); - const unit = path.join(repository, "tools/e2e/nemoclaw-jetson-dispatch.service"); - fs.writeFileSync(cleanup, "#!/usr/bin/env bash\necho first-release\n", { mode: 0o755 }); - fs.writeFileSync(environment, "JETSON_DISPATCH_PORT=8787\n"); - fs.writeFileSync(unit, "[Service]\nEnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment\n"); - git(repository, "add", "tools/e2e"); - git(repository, "commit", "--quiet", "-m", "first release"); - const firstSha = git(repository, "rev-parse", "HEAD"); - fs.writeFileSync(cleanup, "#!/usr/bin/env bash\necho second-release\n", { mode: 0o755 }); - git(repository, "add", cleanup); - git(repository, "commit", "--quiet", "-m", "second release"); - return { directory, repository, firstSha, secondSha: git(repository, "rev-parse", "HEAD") }; -} - -function runDeploy( - fixture: Fixture, - args: string[], - options: DeployOptions = {}, -): ReturnType { - return spawnSync("/bin/bash", ["-c", HARNESS, "deploy-harness", ...args], { - cwd: process.cwd(), - encoding: "utf8", - env: { - ...process.env, - ACTIVE_STATE_ERROR: options.activeStateError ? "1" : "0", - DEPLOY_SCRIPT, - FAIL_INITIAL_ROLLBACK_DISABLE: options.failInitialRollbackDisable ? "1" : "0", - FAIL_ROLLBACK_STOP: options.failRollbackStop ? "1" : "0", - FAIL_SELECT_SHA: options.failSelectSha ?? "", - FAIL_START_SHA: options.failStartSha ?? "", - FAIL_VERIFY_SHA: options.failVerifySha ?? "", - FAKE_UID: options.fakeUid ?? "0", - FIXTURE_REPOSITORY: fixture.repository, - INITIAL_DEVICE_LOCK: options.initialDeviceLock ? "1" : "0", - INVALID_KNOWN_HOSTS: options.invalidKnownHosts ? "1" : "0", - INVALID_SERVICE_HOME: options.invalidServiceHome ? "1" : "0", - INVALID_SSH_KEY: options.invalidSshKey ? "1" : "0", - LEAVE_DEVICE_LOCK: options.leaveDeviceLock ? "1" : "0", - MISSING_ACCOUNT: options.missingAccount ? "1" : "0", - NODE_EXECUTABLE: process.execPath, - SERVICE_LOAD_STATE: options.serviceLoadState ?? "not-found", - TEST_ROOT: fixture.directory, - TUNNEL_ACTIVE: options.tunnelActive ? "1" : "0", - TUNNEL_ACTIVATE_AFTER_CHECK: String(options.tunnelActivatesAfterCheck ?? 0), - UNSUPPORTED_NODE: options.unsupportedNode ? "1" : "0", - }, - }); -} - -function deploymentPaths(fixture: Fixture) { - const root = path.join(fixture.directory, "opt/nemoclaw-jetson-dispatch"); - return { - cleanup: path.join(fixture.directory, "usr/local/libexec/nemoclaw-jetson-cleanup"), - current: path.join(root, "current"), - environment: path.join(fixture.directory, "etc/nemoclaw-jetson-dispatch/environment"), - releases: path.join(root, "releases"), - serviceLog: path.join(fixture.directory, "service.log"), - unit: path.join(fixture.directory, "etc/systemd/system/nemoclaw-jetson-dispatch.service"), - }; -} - -function readIfPresent(file: string): string { - return fs.existsSync(file) ? fs.readFileSync(file, "utf8") : ""; -} - -describe("Colossus Jetson dispatcher deployment", () => { - it("ships the fixed dispatcher environment and restricted systemd unit (#8142)", () => { - const environment = fs.readFileSync(ENVIRONMENT_SOURCE, "utf8"); - const unit = fs.readFileSync(UNIT_SOURCE, "utf8"); - - expect(environment).toContain("JETSON_DISPATCH_GITHUB_REPOSITORY_ID=1182547092"); - expect(environment).toContain("JETSON_DISPATCH_SSH_DESTINATION=nvidia@192.168.55.1"); - expect(environment).toContain( - "JETSON_DISPATCH_CLEANUP_EXECUTABLE=/usr/local/libexec/nemoclaw-jetson-cleanup", - ); - expect(environment).not.toContain("=REPOSITORY_ID"); - expect(unit).toContain("EnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment"); - expect(unit).toContain("WorkingDirectory=/opt/nemoclaw-jetson-dispatch/current"); - expect(unit).toContain("NoNewPrivileges=true"); - expect(unit).not.toContain("nemoclaw-jetson-tunnel.service"); - }); - - it("installs and verifies an exact release when the service is absent (#8142)", () => { - const fixture = createFixture(); - const result = runDeploy(fixture, ["--commit", fixture.firstSha]); - const paths = deploymentPaths(fixture); - - expect(result.status, String(result.stderr)).toBe(0); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readlinkSync(paths.cleanup)).toBe( - path.join(paths.current, "tools/e2e/jetson-dispatch-cleanup.sh"), - ); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - expect(fs.readdirSync(paths.releases)).toEqual([fixture.firstSha]); - expect(fs.readFileSync(paths.environment, "utf8")).toBe("JETSON_DISPATCH_PORT=8787\n"); - expect(fs.readFileSync(paths.unit, "utf8")).toContain( - "EnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment", - ); - expect(readIfPresent(paths.serviceLog)).toMatch( - /systemctl\tdaemon-reload[\s\S]*systemctl\tenable --now nemoclaw-jetson-dispatch\.service/u, - ); - expect(result.stdout).toMatch( - /\[1\/5\].*[\s\S]*\[2\/5\].*[\s\S]*\[3\/5\].*[\s\S]*\[4\/5\].*[\s\S]*\[5\/5\]/u, - ); - expect(result.stdout).toContain("public ingress remains disabled"); - expect(readIfPresent(paths.serviceLog)).not.toContain( - "enable --now nemoclaw-jetson-tunnel.service", - ); - expect(fs.readFileSync(path.join(fixture.directory, "curl.log"), "utf8")).toContain( - "curl\t--disable\t--noproxy\t*\t--silent", - ); - expect(fs.readFileSync(path.join(fixture.directory, "git.log"), "utf8")).toContain( - `remote\tadd\torigin\t${FIXED_ORIGIN}`, - ); - }); - - it.each([ - ["dispatcher account", { missingAccount: true }, "account is required"], - ["service home", { invalidServiceHome: true }, "must be a directory"], - ["SSH identity", { invalidSshKey: true }, "is not a readable OpenSSH private key"], - ["pinned host key", { invalidKnownHosts: true }, "does not contain the pinned Jetson"], - ["Node.js version", { unsupportedNode: true }, "must be Node.js 22.19.0 or later"], - ] satisfies Array< - [string, DeployOptions, string] - >)("rejects initial deployment when the prepared %s is invalid (#8142)", (_name, options, error) => { - const fixture = createFixture(); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], options); - const paths = deploymentPaths(fixture); - - expect(result.status).toBe(1); - expect(result.stderr).toContain(error); - expect(fs.existsSync(paths.current)).toBe(false); - expect(fs.existsSync(paths.environment)).toBe(false); - expect(fs.existsSync(paths.unit)).toBe(false); - }); - - it("rejects an unmanaged cleanup path before initial deployment (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - fs.mkdirSync(path.dirname(paths.cleanup), { recursive: true }); - fs.writeFileSync(paths.cleanup, "unmanaged\n"); - - const result = runDeploy(fixture, ["--commit", fixture.firstSha]); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("exists without one managed current release"); - expect(fs.readFileSync(paths.cleanup, "utf8")).toBe("unmanaged\n"); - expect(fs.existsSync(paths.environment)).toBe(false); - expect(fs.existsSync(paths.unit)).toBe(false); - }); - - it("rejects an unmanaged current link before initial deployment (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - fs.mkdirSync(path.dirname(paths.current), { recursive: true }); - fs.symlinkSync(fixture.repository, paths.current); - - const result = runDeploy(fixture, ["--commit", fixture.firstSha]); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("does not select one managed release"); - expect(fs.readlinkSync(paths.current)).toBe(fixture.repository); - expect(fs.existsSync(paths.cleanup)).toBe(false); - expect(fs.existsSync(paths.environment)).toBe(false); - expect(fs.existsSync(paths.unit)).toBe(false); - }); - - it("rejects initial deployment while public ingress is active (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { - tunnelActive: true, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("must be disabled and inactive"); - expect(fs.existsSync(paths.current)).toBe(false); - expect(fs.existsSync(paths.environment)).toBe(false); - expect(fs.existsSync(paths.unit)).toBe(false); - }); - - it("rolls back initial deployment when public ingress becomes active (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { - tunnelActivatesAfterCheck: 1, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("release, unit, and environment were rolled back"); - expect(fs.existsSync(paths.current)).toBe(false); - expect(fs.existsSync(paths.cleanup)).toBe(false); - expect(fs.existsSync(paths.environment)).toBe(false); - expect(fs.existsSync(paths.unit)).toBe(false); - }); - - it("rolls back the initial release, unit, and environment when verification fails (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { - failVerifySha: fixture.firstSha, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("release, unit, and environment were rolled back"); - expect(fs.existsSync(paths.current)).toBe(false); - expect(fs.existsSync(paths.cleanup)).toBe(false); - expect(fs.existsSync(paths.environment)).toBe(false); - expect(fs.existsSync(paths.unit)).toBe(false); - expect(readIfPresent(paths.serviceLog)).toMatch( - /systemctl\tenable --now nemoclaw-jetson-dispatch\.service[\s\S]*systemctl\tdisable --now nemoclaw-jetson-dispatch\.service[\s\S]*systemctl\tdaemon-reload/u, - ); - }); - - it("reports when initial rollback cannot disable the service (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { - failInitialRollbackDisable: true, - failVerifySha: fixture.firstSha, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("rollback did not restore the prepared host"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - expect(fs.existsSync(paths.environment)).toBe(true); - expect(fs.existsSync(paths.unit)).toBe(true); - }); - - it("reports inconclusive containment when ingress activates and initial disable fails (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { - failInitialRollbackDisable: true, - tunnelActivatesAfterCheck: 2, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("rollback did not restore the prepared host"); - expect(fs.existsSync(path.join(fixture.directory, "service-active"))).toBe(true); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.existsSync(paths.environment)).toBe(true); - expect(fs.existsSync(paths.unit)).toBe(true); - }); - - it("stops, switches, starts, and verifies an installed loopback service (#8142)", () => { - const fixture = createFixture(); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - serviceLoadState: "loaded", - }); - const serviceLog = fs.readFileSync(deploymentPaths(fixture).serviceLog, "utf8"); - const curlLog = fs.readFileSync(path.join(fixture.directory, "curl.log"), "utf8"); - - expect(result.status).toBe(0); - expect(serviceLog).toMatch( - /systemctl\tstop nemoclaw-jetson-dispatch\.service[\s\S]*start-sha\t[0-9a-f]{40}/u, - ); - expect(result.stdout).toContain("verified its loopback service"); - expect(curlLog).toContain("curl\t--disable\t--noproxy\t*\t--silent"); - }); - - it("rejects a later dispatcher deployment while public ingress is active (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - const serviceLogBeforeDeployment = fs.readFileSync(paths.serviceLog, "utf8"); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - serviceLoadState: "loaded", - tunnelActive: true, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("must be disabled and inactive"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - const deploymentServiceLog = fs - .readFileSync(paths.serviceLog, "utf8") - .slice(serviceLogBeforeDeployment.length); - expect(deploymentServiceLog).not.toContain("systemctl\tstop"); - expect(deploymentServiceLog).not.toContain("start-sha"); - }); - - it.each([ - ["after the dispatcher starts", 3], - ["after loopback verification", 4], - ])("leaves the dispatcher stopped when public ingress activates %s (#8142)", (_phase, check) => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - fs.rmSync(path.join(fixture.directory, "tunnel-check-count"), { force: true }); - const serviceLogBeforeDeployment = fs.readFileSync(paths.serviceLog, "utf8"); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - serviceLoadState: "loaded", - tunnelActivatesAfterCheck: check, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("rollback did not restore a verified service"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.existsSync(path.join(fixture.directory, "service-active"))).toBe(false); - const deploymentServiceLog = fs - .readFileSync(paths.serviceLog, "utf8") - .slice(serviceLogBeforeDeployment.length); - expect(deploymentServiceLog).toContain(`start-sha\t${fixture.secondSha}`); - expect(deploymentServiceLog).not.toContain(`start-sha\t${fixture.firstSha}`); - }); - - it("reports inconclusive dispatcher state when ingress containment cannot stop it (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - fs.rmSync(path.join(fixture.directory, "tunnel-check-count"), { force: true }); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - failRollbackStop: true, - serviceLoadState: "loaded", - tunnelActivatesAfterCheck: 3, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("rollback did not restore a verified service"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.secondSha)); - expect(fs.existsSync(path.join(fixture.directory, "service-active"))).toBe(true); - }); - - it.each([ - ["a branch", ["--commit", "main"]], - ["a tag", ["--commit", "v0.1.0"]], - ["a URL", ["--commit", FIXED_ORIGIN]], - ["an abbreviated SHA", ["--commit", "a".repeat(12)]], - ["an uppercase SHA", ["--commit", "A".repeat(40)]], - ["an extra argument", ["--commit", "a".repeat(40), "extra"]], - ["a missing flag", ["a".repeat(40)]], - ])("rejects %s before deployment (#8142)", (_name, args) => { - const result = runDeploy(createFixture(), args); - - expect(result.status).toBe(2); - expect(result.stderr).toContain("--commit "); - }); - - it("rejects deployment by a non-root caller (#8142)", () => { - const fixture = createFixture(); - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { fakeUid: "1000" }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("this command requires root"); - }); - - it("rejects a release whose configured origin is not the fixed repository (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - git( - path.join(paths.releases, fixture.firstSha), - "remote", - "set-url", - "origin", - "https://example.test/other.git", - ); - - const result = runDeploy(fixture, ["--commit", fixture.firstSha]); - - expect(result.status).toBe(1); - expect(result.stderr).toContain(`release origin is not ${FIXED_ORIGIN}`); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - }); - - it("rejects a modified release checkout (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - fs.appendFileSync( - path.join(paths.releases, fixture.firstSha, "tools/e2e/jetson-dispatch-cleanup.sh"), - "modified\n", - ); - - const result = runDeploy(fixture, ["--commit", fixture.firstSha]); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("release checkout is modified"); - }); - - it("rejects a current link that does not select one managed release (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - fs.unlinkSync(paths.current); - fs.symlinkSync(fixture.repository, paths.current); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha]); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("does not select one managed release"); - expect(fs.readlinkSync(paths.current)).toBe(fixture.repository); - }); - - it("rolls back code and cleanup when service startup fails (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - failStartSha: fixture.secondSha, - serviceLoadState: "loaded", - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("previous deployment state was restored"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - }); - - it("rejects an inconclusive service state after stopping (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - activeStateError: true, - serviceLoadState: "loaded", - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "could not inspect nemoclaw-jetson-dispatch.service after stop", - ); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - }); - - it("does not switch releases when service cleanup leaves the device lock (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - leaveDeviceLock: true, - serviceLoadState: "loaded", - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("recover cleanup before deployment"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - }); - - it("rejects bootstrap when a stale device lock remains (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - - const result = runDeploy(fixture, ["--commit", fixture.firstSha], { - initialDeviceLock: true, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("recover cleanup before deployment"); - expect(fs.existsSync(paths.current)).toBe(false); - expect(fs.existsSync(paths.cleanup)).toBe(false); - expect(fs.existsSync(paths.releases)).toBe(true); - expect(fs.readdirSync(paths.releases)).toEqual([]); - }); - - it("redeploys the selected commit without creating another release (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - expect(fs.readdirSync(paths.releases)).toEqual([fixture.firstSha]); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - }); - - it("rolls back code and cleanup when loopback verification fails (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - failVerifySha: fixture.secondSha, - serviceLoadState: "loaded", - }); - const serviceLog = fs.readFileSync(paths.serviceLog, "utf8"); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("previous deployment state was restored"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - expect(serviceLog).toContain(`start-sha\t${fixture.secondSha}`); - expect(serviceLog).toContain(`start-sha\t${fixture.firstSha}`); - }); - - it("keeps code and cleanup on the prior release when selection fails (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - failSelectSha: fixture.secondSha, - serviceLoadState: "loaded", - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("previous deployment state was restored"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.firstSha)); - expect(fs.readlinkSync(paths.cleanup)).toBe( - path.join(paths.current, "tools/e2e/jetson-dispatch-cleanup.sh"), - ); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("first-release"); - }); - - it("keeps the new code and cleanup paired when rollback cannot stop the service (#8142)", () => { - const fixture = createFixture(); - const paths = deploymentPaths(fixture); - expect(runDeploy(fixture, ["--commit", fixture.firstSha]).status).toBe(0); - - const result = runDeploy(fixture, ["--commit", fixture.secondSha], { - failRollbackStop: true, - failVerifySha: fixture.secondSha, - serviceLoadState: "loaded", - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("rollback did not restore a verified service"); - expect(fs.readlinkSync(paths.current)).toBe(path.join(paths.releases, fixture.secondSha)); - expect(fs.readFileSync(paths.cleanup, "utf8")).toContain("second-release"); - }); -}); diff --git a/test/e2e/support/jetson-dispatch-client.test.ts b/test/e2e/support/jetson-dispatch-client.test.ts new file mode 100644 index 00000000000..692d72c2182 --- /dev/null +++ b/test/e2e/support/jetson-dispatch-client.test.ts @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createGitHubOidcTokenProvider, + dispatcherBaseUrl, + dispatcherRequest, + pollJetsonDispatch, +} from "../../../tools/e2e/jetson-dispatch-client.mts"; +import { + JETSON_DISPATCH_AUDIENCE, + JETSON_DISPATCH_CONTRACT_VERSION, + type JetsonDispatchArtifact, + type JetsonDispatchRequest, + type JetsonDispatchStatus, + parseJetsonDispatchArtifact, + parseJetsonDispatchRequest, + parseJetsonDispatchStatus, + parseJetsonDispatchStatusResponse, +} from "../../../tools/e2e/jetson-dispatch-contract.mts"; + +interface CompatibilityVectors { + artifact: unknown; + completedStatus: unknown; + contractVersion: string; + invalidRequests: Array<{ name: string; value: unknown }>; + request: unknown; + queuedResponse: unknown; +} + +const vectors = JSON.parse( + fs.readFileSync(path.join(process.cwd(), "tools/e2e/contracts/v1/jetson-dispatch.json"), "utf8"), +) as CompatibilityVectors; +const request = parseJetsonDispatchRequest(vectors.request); +const queuedStatus = parseJetsonDispatchStatusResponse(vectors.queuedResponse); +const completedStatus = parseJetsonDispatchStatus(vectors.completedStatus); +const invalidRequestErrors: Record = { + "extra request field": "dispatch request fields do not match Jetson dispatch contract 1.0.0", + "noncanonical candidate SHA": "candidateSha must be a lowercase 40-character commit SHA", + "wrong target": "dispatch target must be jetson-nvmap-gpu", +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("Jetson dispatch static HTTP contract", () => { + it("accepts every version 1.0.0 compatibility vector (#8142)", () => { + expect(vectors.contractVersion).toBe(JETSON_DISPATCH_CONTRACT_VERSION); + expect(request).toEqual(vectors.request); + expect({ job: queuedStatus }).toEqual(vectors.queuedResponse); + expect(completedStatus).toEqual(vectors.completedStatus); + expect(parseJetsonDispatchArtifact(vectors.artifact, completedStatus.jobId)).toEqual( + vectors.artifact, + ); + }); + + it.each(vectors.invalidRequests)("rejects $name (#8142)", ({ name, value }) => { + expect(() => parseJetsonDispatchRequest(value)).toThrow(invalidRequestErrors[name]); + }); + + it("rejects a valid status for a different submitted request (#8142)", () => { + expect(() => + parseJetsonDispatchStatusResponse(vectors.queuedResponse, { + ...request, + workflowRunAttempt: request.workflowRunAttempt + 1, + }), + ).toThrow("Jetson dispatcher response does not match the submitted request"); + }); + + it.each([ + { + name: "an extra status field", + value: { ...(vectors.queuedResponse as { job: object }).job, command: "untrusted" }, + expectedError: + "queued Jetson dispatch status fields do not match Jetson dispatch contract 1.0.0", + }, + { + name: "a mismatched job ID", + value: { ...(vectors.queuedResponse as { job: object }).job, jobId: "f".repeat(64) }, + expectedError: "Jetson dispatch status does not match its request and job ID", + }, + { + name: "completed cleanup on a queued job", + value: { ...(vectors.queuedResponse as { job: object }).job, cleanup: "succeeded" }, + expectedError: "queued Jetson dispatch cleanup must be pending", + }, + { + name: "a successful result without device identity", + value: { ...(vectors.completedStatus as Record), device: undefined }, + expectedError: "successful Jetson dispatch must include device identity", + }, + { + name: "a cleanup conclusion mismatch", + value: { + ...(vectors.completedStatus as Record), + cleanup: "failed", + conclusion: "failure", + }, + expectedError: "completed Jetson dispatch result is invalid", + }, + { + name: "a successful result with failed cleanup", + value: { + ...(vectors.completedStatus as Record), + cleanup: "failed", + }, + expectedError: "completed Jetson dispatch result is invalid", + }, + { + name: "a non-string conclusion", + value: { + ...(vectors.completedStatus as Record), + conclusion: ["success"], + }, + expectedError: "completed Jetson dispatch result is invalid", + }, + { + name: "a non-string cleanup state", + value: { + ...(vectors.completedStatus as Record), + cleanup: ["succeeded"], + }, + expectedError: "completed Jetson dispatch result is invalid", + }, + { + name: "an out-of-order completion timestamp", + value: { + ...(vectors.completedStatus as Record), + completedAt: "2026-08-11T23:59:59.000Z", + }, + expectedError: "completed Jetson dispatch timestamps are invalid", + }, + ])("rejects $name (#8142)", ({ expectedError, value }) => { + expect(() => parseJetsonDispatchStatus(value)).toThrow(expectedError); + }); + + it("accepts cleanup success when later device-lock release fails (#8142)", () => { + const status = { + ...(vectors.completedStatus as Record), + conclusion: "cleanup-failed", + error: "Jetson lock removal failed: fixture failure", + }; + const artifact = { + ...(vectors.artifact as Record), + status, + }; + + expect(parseJetsonDispatchStatus(status)).toEqual(status); + expect(parseJetsonDispatchArtifact(artifact, completedStatus.jobId)).toEqual(artifact); + }); + + it("accepts the backend device-identity bound and rejects larger values (#8142)", () => { + const baseDevice = (vectors.completedStatus as { device: Record }).device; + const bounded = { + ...(vectors.completedStatus as Record), + device: { ...baseDevice, model: "x".repeat(500) }, + }; + const oversized = { + ...bounded, + device: { ...baseDevice, model: "x".repeat(501) }, + }; + + expect(parseJetsonDispatchStatus(bounded).device?.model).toHaveLength(500); + expect(() => parseJetsonDispatchStatus(oversized)).toThrow("Jetson device model is invalid"); + }); + + it.each([ + { + name: "an extra artifact field", + value: { ...(vectors.artifact as Record), secret: "unexpected" }, + expectedError: "Jetson dispatch artifact fields do not match Jetson dispatch contract 1.0.0", + }, + { + name: "a mismatched artifact job ID", + value: vectors.artifact, + expectedJobId: "f".repeat(64), + expectedError: "Jetson dispatch artifact does not match its completed job", + }, + { + name: "a noncanonical artifact archive", + value: { + ...(vectors.artifact as Record), + artifactArchiveBase64: "not base64", + }, + expectedError: "Jetson artifact archive must be bounded canonical base64", + }, + { + name: "a successful result without its archive", + value: { + status: (vectors.artifact as { status: unknown }).status, + log: "fixture log\n", + }, + expectedError: "successful Jetson dispatch must include its artifact archive", + }, + { + name: "a non-string successful conclusion without an archive", + value: { + status: { + ...(vectors.completedStatus as Record), + conclusion: ["success"], + }, + log: "fixture log\n", + }, + expectedError: "completed Jetson dispatch result is invalid", + }, + ])("rejects $name (#8142)", ({ expectedError, expectedJobId, value }) => { + expect(() => + parseJetsonDispatchArtifact(value, expectedJobId ?? completedStatus.jobId), + ).toThrow(expectedError); + }); +}); + +describe("Jetson dispatch GitHub controller", () => { + it("requests a GitHub OIDC token for the fixed dispatcher audience (#8142)", async () => { + const fetchImpl = vi.fn(async (_input: Parameters[0], _init?: RequestInit) => + Response.json({ value: "oidc-token" }), + ); + const token = createGitHubOidcTokenProvider({ fetchImpl }); + + await expect( + token({ + ACTIONS_ID_TOKEN_REQUEST_TOKEN: "request-token", + ACTIONS_ID_TOKEN_REQUEST_URL: "https://token.actions.test/id-token?ignored=value", + }), + ).resolves.toBe("oidc-token"); + + expect(fetchImpl).toHaveBeenCalledOnce(); + const [input, init] = fetchImpl.mock.calls[0]!; + expect(new URL(String(input)).searchParams.get("audience")).toBe(JETSON_DISPATCH_AUDIENCE); + expect(init?.headers).toEqual({ Authorization: "Bearer request-token" }); + }); + + it("rejects missing or non-HTTPS GitHub OIDC configuration (#8142)", async () => { + const token = createGitHubOidcTokenProvider({ fetchImpl: vi.fn() }); + + await expect(token({})).rejects.toThrow("GitHub OIDC environment is unavailable"); + await expect( + token({ + ACTIONS_ID_TOKEN_REQUEST_TOKEN: "request-token", + ACTIONS_ID_TOKEN_REQUEST_URL: "http://token.actions.test/id-token", + }), + ).rejects.toThrow("GitHub OIDC request URL must use HTTPS"); + }); + + it("reuses a GitHub OIDC token only during its bounded cache window (#8142)", async () => { + let nowMs = 1_000; + const fetchImpl = vi.fn(async () => Response.json({ value: `token-${nowMs}` })); + const token = createGitHubOidcTokenProvider({ fetchImpl, now: () => nowMs }); + const env = { + ACTIONS_ID_TOKEN_REQUEST_TOKEN: "request-token", + ACTIONS_ID_TOKEN_REQUEST_URL: "https://token.actions.test/id-token", + }; + + await expect(token(env)).resolves.toBe("token-1000"); + nowMs += 60_000; + await expect(token(env)).resolves.toBe("token-1000"); + nowMs += 4 * 60_000; + await expect(token(env)).resolves.toBe("token-301000"); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("sends the exact request with a bearer token and validates the response (#8142)", async () => { + const tokenProvider = vi.fn(async () => "oidc-token"); + const fetchImpl = vi.fn(async () => Response.json(vectors.queuedResponse, { status: 202 })); + + const response = await dispatcherRequest({ + baseUrl: new URL("https://dispatch.test/"), + method: "POST", + path: "v1/jobs", + body: request, + maxBytes: 64 * 1024, + fetchImpl, + tokenProvider, + }); + + expect(parseJetsonDispatchStatusResponse(response)).toEqual(queuedStatus); + expect(tokenProvider).toHaveBeenCalledOnce(); + expect(fetchImpl).toHaveBeenCalledWith( + new URL("https://dispatch.test/v1/jobs"), + expect.objectContaining({ + method: "POST", + headers: { + Accept: "application/json", + Authorization: "Bearer oidc-token", + "Content-Type": "application/json", + }, + body: JSON.stringify(request), + }), + ); + }); + + it.each([ + undefined, + "http://dispatch.test", + "https://user@dispatch.test", + "https://dispatch.test/v1/jobs", + "https://dispatch.test/?query=value", + ])("rejects an unsafe dispatcher URL value %# (#8142)", (value) => { + expect(() => dispatcherBaseUrl(value)).toThrow(); + }); + + it("rejects a dispatcher response that exceeds its byte bound (#8142)", async () => { + const fetchImpl = vi.fn( + async () => + new Response(JSON.stringify(vectors.queuedResponse), { + headers: { "Content-Length": "65537" }, + status: 202, + }), + ); + + await expect( + dispatcherRequest({ + baseUrl: new URL("https://dispatch.test/"), + method: "POST", + path: "v1/jobs", + body: request, + maxBytes: 64 * 1024, + fetchImpl, + tokenProvider: async () => "oidc-token", + }), + ).rejects.toThrow("Jetson dispatcher response is too large"); + }); + + it("rejects an oversized streamed response without Content-Length (#8142)", async () => { + const chunk = new TextEncoder().encode("x".repeat(4096)); + const fetchImpl = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(chunk); + controller.enqueue(chunk); + controller.enqueue(chunk); + controller.close(); + }, + }), + { status: 202 }, + ), + ); + + await expect( + dispatcherRequest({ + baseUrl: new URL("https://dispatch.test/"), + method: "GET", + path: "v1/jobs/test", + maxBytes: 8 * 1024, + fetchImpl, + tokenProvider: async () => "oidc-token", + }), + ).rejects.toThrow("Jetson dispatcher response is too large"); + }); + + it("retries status transport failures and validates the completed status (#8142)", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const requestImpl = vi + .fn() + .mockRejectedValueOnce(new Error("transport reset")) + .mockResolvedValueOnce({ job: completedStatus }); + + await expect( + pollJetsonDispatch({ + baseUrl: new URL("https://dispatch.test/"), + deadlineMs: 10_000, + initialStatus: queuedStatus, + jobId: queuedStatus.jobId, + now: () => 0, + request: requestImpl, + wait: async () => {}, + }), + ).resolves.toEqual(completedStatus); + }); + + it("requests cancellation when the controller deadline expires (#8142)", async () => { + const requestImpl = vi.fn(async () => ({ job: queuedStatus })); + + await expect( + pollJetsonDispatch({ + baseUrl: new URL("https://dispatch.test/"), + deadlineMs: 10_000, + initialStatus: queuedStatus, + jobId: queuedStatus.jobId, + now: () => 10_000, + request: requestImpl, + wait: async () => {}, + }), + ).rejects.toThrow("Jetson dispatcher did not complete before the controller deadline"); + expect(requestImpl).toHaveBeenCalledWith( + expect.objectContaining({ method: "DELETE", path: `v1/jobs/${queuedStatus.jobId}` }), + ); + }); + + it("keeps the published response types compatible with the controller (#8142)", () => { + const typedRequest: JetsonDispatchRequest = request; + const typedStatus: JetsonDispatchStatus = completedStatus; + const typedArtifact: JetsonDispatchArtifact = parseJetsonDispatchArtifact( + vectors.artifact, + completedStatus.jobId, + ); + + expect([ + typedRequest.schemaVersion, + typedStatus.schemaVersion, + typedArtifact.status.schemaVersion, + ]).toEqual([1, 1, 1]); + }); +}); diff --git a/test/e2e/support/jetson-dispatch-contract.test.ts b/test/e2e/support/jetson-dispatch-contract.test.ts deleted file mode 100644 index 287452a0e44..00000000000 --- a/test/e2e/support/jetson-dispatch-contract.test.ts +++ /dev/null @@ -1,1022 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createSign, generateKeyPairSync, type KeyObject } from "node:crypto"; -import fs from "node:fs"; -import type { AddressInfo } from "node:net"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; -import { - createGitHubOidcTokenProvider, - pollJetsonDispatch, -} from "../../../tools/e2e/jetson-dispatch-client.mts"; -import { - createGitHubJwksResolver, - JETSON_DISPATCH_AUDIENCE, - JETSON_DISPATCH_REPOSITORY, - JETSON_DISPATCH_TARGET, - JETSON_DISPATCH_WORKFLOW_REF, - parseJetsonDispatchRequest, - verifyGitHubOidcToken, -} from "../../../tools/e2e/jetson-dispatch-contract.mts"; -import { - JetsonDispatchBusyError, - JetsonDispatchCoordinator, - type JetsonDispatchStatus, - type JetsonDispatchWorker, - MAX_JETSON_ARTIFACT_ARCHIVE_BYTES, - MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, - MAX_JETSON_DISPATCH_LOG_BYTES, -} from "../../../tools/e2e/jetson-dispatch-lifecycle.mts"; -import { createJetsonDispatchServer } from "../../../tools/e2e/jetson-dispatch-service.mts"; -import { testTimeoutOptions } from "../../helpers/timeouts.ts"; - -const NOW_SECONDS = 1_800_000_000; -const REPOSITORY_ID = "987654321"; -const CANDIDATE_SHA = "a".repeat(40); -const WORKFLOW_SHA = "b".repeat(40); -const ARTIFACT_ARCHIVE_BASE64 = Buffer.from("bounded archive").toString("base64"); -const REQUEST = parseJetsonDispatchRequest({ - schemaVersion: 1, - target: JETSON_DISPATCH_TARGET, - candidateSha: CANDIDATE_SHA, - workflowRunId: "123456789", - workflowRunAttempt: 1, -}); - -const temporaryDirectories: string[] = []; - -afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - for (const directory of temporaryDirectories.splice(0)) { - fs.rmSync(directory, { force: true, recursive: true }); - } -}); - -function temporaryDirectory(): string { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-jetson-dispatch-")); - temporaryDirectories.push(directory); - return directory; -} - -function failTest(message: string): never { - throw new Error(message); -} - -function claims(overrides: Record = {}): Record { - return { - iss: "https://token.actions.githubusercontent.com", - aud: JETSON_DISPATCH_AUDIENCE, - sub: "repo:NVIDIA/NemoClaw:ref:refs/heads/main", - repository: JETSON_DISPATCH_REPOSITORY, - repository_id: REPOSITORY_ID, - workflow_ref: JETSON_DISPATCH_WORKFLOW_REF, - workflow_sha: WORKFLOW_SHA, - sha: WORKFLOW_SHA, - ref: "refs/heads/main", - event_name: "workflow_dispatch", - runner_environment: "github-hosted", - run_id: REQUEST.workflowRunId, - run_attempt: String(REQUEST.workflowRunAttempt), - jti: "token-id", - iat: NOW_SECONDS - 60, - nbf: NOW_SECONDS - 60, - exp: NOW_SECONDS + 9 * 60, - ...overrides, - }; -} - -function signToken(privateKey: KeyObject, payload: Record): string { - const header = Buffer.from( - JSON.stringify({ alg: "RS256", kid: "test-key", typ: "JWT" }), - ).toString("base64url"); - const body = Buffer.from(JSON.stringify(payload)).toString("base64url"); - const signer = createSign("RSA-SHA256"); - signer.update(`${header}.${body}`, "ascii"); - signer.end(); - return `${header}.${body}.${signer.sign(privateKey).toString("base64url")}`; -} - -function signingKeys(): { privateKey: KeyObject; publicKey: KeyObject } { - return generateKeyPairSync("rsa", { modulusLength: 2048 }); -} - -function identity() { - return { - repository: JETSON_DISPATCH_REPOSITORY, - repositoryId: REPOSITORY_ID, - runId: REQUEST.workflowRunId, - runAttempt: REQUEST.workflowRunAttempt, - workflowSha: WORKFLOW_SHA, - tokenId: "token-id", - } as const; -} - -async function waitForCompletion( - coordinator: JetsonDispatchCoordinator, - jobId: string, -): Promise { - await vi.waitFor(() => expect(coordinator.status(jobId).state).toBe("completed"), { - interval: 1, - timeout: 1_000, - }); - return coordinator.status(jobId); -} - -function worker( - options: { cleanup?: JetsonDispatchWorker["cleanup"]; run?: JetsonDispatchWorker["run"] } = {}, -): JetsonDispatchWorker { - return { - run: - options.run ?? - (async () => ({ - artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, - device: { - model: "NVIDIA Jetson AGX Thor Developer Kit", - jetpackVersion: "7.2.2", - jetsonLinuxRelease: "R38", - kernel: "6.8.12-tegra", - }, - log: "passed\n", - })), - cleanup: options.cleanup ?? (async () => {}), - }; -} - -function coordinator(stateDirectory: string, dispatchWorker: JetsonDispatchWorker) { - return new JetsonDispatchCoordinator({ - stateDirectory, - worker: dispatchWorker, - executionTimeoutMs: 60_000, - cleanupTimeoutMs: 10_000, - }); -} - -describe("Jetson dispatch request and OIDC boundary", () => { - it("accepts only the fixed target and exact workflow run identity (#8142)", () => { - expect(REQUEST).toEqual({ - schemaVersion: 1, - target: "jetson-nvmap-gpu", - candidateSha: CANDIDATE_SHA, - workflowRunId: "123456789", - workflowRunAttempt: 1, - }); - }); - - it.each([ - "command", - "host", - "identityFile", - "repository", - "url", - ])("rejects request-controlled %s fields (#8142)", (field) => { - expect(() => parseJetsonDispatchRequest({ ...REQUEST, [field]: "unsafe" })).toThrow( - "dispatch request fields must match the fixed Jetson contract", - ); - }); - - it("verifies the signature and all trusted GitHub controller claims (#8142)", async () => { - const keys = signingKeys(); - const verified = await verifyGitHubOidcToken({ - token: signToken(keys.privateKey, claims()), - request: REQUEST, - policy: { repositoryId: REPOSITORY_ID }, - resolveSigningKey: async () => keys.publicKey, - nowMs: NOW_SECONDS * 1_000, - }); - - expect(verified).toEqual(identity()); - }); - - it("reuses each GitHub OIDC token for a bounded polling interval (#8142)", async () => { - let nowMs = 1_000; - const fetchImpl = vi.fn(async () => Response.json({ value: `token-${nowMs}` })); - const token = createGitHubOidcTokenProvider({ fetchImpl, now: () => nowMs }); - const env = { - ACTIONS_ID_TOKEN_REQUEST_TOKEN: "request-token", - ACTIONS_ID_TOKEN_REQUEST_URL: "https://token.actions.test/id-token", - }; - - await expect(token(env)).resolves.toBe("token-1000"); - nowMs += 60_000; - await expect(token(env)).resolves.toBe("token-1000"); - nowMs += 4 * 60_000; - await expect(token(env)).resolves.toBe("token-301000"); - - expect(fetchImpl).toHaveBeenCalledTimes(2); - }); - - it.each([ - ["issuer", { iss: "https://example.test" }], - ["audience", { aud: "another-service" }], - ["repository", { repository: "someone/NemoClaw" }], - ["repository ID", { repository_id: "123" }], - ["workflow", { workflow_ref: "NVIDIA/NemoClaw/.github/workflows/other.yaml@refs/heads/main" }], - ["ref", { ref: "refs/heads/feature" }], - ["event", { event_name: "pull_request" }], - ["runner", { runner_environment: "self-hosted" }], - ["run ID", { run_id: "999" }], - ["run attempt", { run_attempt: "2" }], - ["workflow SHA", { sha: "c".repeat(40) }], - ])("rejects a mismatched %s claim (#8142)", async (_name, override) => { - const keys = signingKeys(); - await expect( - verifyGitHubOidcToken({ - token: signToken(keys.privateKey, claims(override)), - request: REQUEST, - policy: { repositoryId: REPOSITORY_ID }, - resolveSigningKey: async () => keys.publicKey, - nowMs: NOW_SECONDS * 1_000, - }), - ).rejects.toThrow("OIDC claims do not match the trusted Jetson controller"); - }); - - it("rejects expired and incorrectly signed controller tokens (#8142)", async () => { - const trusted = signingKeys(); - const untrusted = signingKeys(); - await expect( - verifyGitHubOidcToken({ - token: signToken(trusted.privateKey, claims({ exp: NOW_SECONDS - 60 })), - request: REQUEST, - policy: { repositoryId: REPOSITORY_ID }, - resolveSigningKey: async () => trusted.publicKey, - nowMs: NOW_SECONDS * 1_000, - }), - ).rejects.toThrow("OIDC token is outside its allowed validity window"); - await expect( - verifyGitHubOidcToken({ - token: signToken(untrusted.privateKey, claims()), - request: REQUEST, - policy: { repositoryId: REPOSITORY_ID }, - resolveSigningKey: async () => trusted.publicKey, - nowMs: NOW_SECONDS * 1_000, - }), - ).rejects.toThrow("OIDC signature verification failed"); - }); - - it("coalesces key-set loads and bounds unknown-key refreshes (#8142)", async () => { - const keys = signingKeys(); - const jwk = keys.publicKey.export({ format: "jwk" }); - const fetchImpl = vi.fn(async () => - Response.json({ keys: [{ ...jwk, alg: "RS256", kid: "known", use: "sig" }] }), - ); - const resolve = createGitHubJwksResolver(fetchImpl as typeof fetch); - - await Promise.all([ - expect(resolve("missing-1")).rejects.toThrow("OIDC signing key is unknown"), - expect(resolve("missing-2")).rejects.toThrow("OIDC signing key is unknown"), - ]); - await expect(resolve("missing-3")).rejects.toThrow("OIDC signing key is unknown"); - await expect(resolve("known")).resolves.toMatchObject({ type: "public" }); - - expect(fetchImpl).toHaveBeenCalledOnce(); - }); -}); - -describe("Jetson single-device lifecycle", () => { - it("keeps the maximum log and archive response within the controller bound (#8142)", () => { - const artifact = { - artifactArchiveBase64: Buffer.alloc(MAX_JETSON_ARTIFACT_ARCHIVE_BYTES).toString("base64"), - log: "\0".repeat(MAX_JETSON_DISPATCH_LOG_BYTES), - status: { - schemaVersion: 1, - jobId: "d".repeat(64), - request: REQUEST, - state: "completed", - conclusion: "success", - createdAt: new Date(0).toISOString(), - completedAt: new Date(0).toISOString(), - cleanup: "succeeded", - }, - }; - - expect(Buffer.byteLength(`${JSON.stringify(artifact)}\n`)).toBeLessThanOrEqual( - MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, - ); - }); - - it("records success only after cleanup succeeds (#8142)", async () => { - const cleanup = vi.fn(async () => {}); - const dispatch = coordinator(temporaryDirectory(), worker({ cleanup })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ - state: "completed", - conclusion: "success", - cleanup: "succeeded", - device: { model: "NVIDIA Jetson AGX Thor Developer Kit" }, - }); - expect(cleanup).toHaveBeenCalledOnce(); - expect(dispatch.artifact(accepted.jobId)).toMatchObject({ - artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, - log: "passed\n", - }); - }); - - it("records test failure after cleanup succeeds (#8142)", async () => { - const cleanup = vi.fn(async () => {}); - const dispatch = coordinator( - temporaryDirectory(), - worker({ cleanup, run: async () => Promise.reject(new Error("test failed")) }), - ); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ - cleanup: "succeeded", - conclusion: "failure", - error: "test failed", - }); - expect(cleanup).toHaveBeenCalledOnce(); - }); - - it("returns completed evidence when its result log is missing (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const dispatch = coordinator(stateDirectory, worker()); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - await waitForCompletion(dispatch, accepted.jobId); - fs.rmSync(path.join(stateDirectory, `${accepted.jobId}.log`)); - - expect(dispatch.artifact(accepted.jobId)).toMatchObject({ - artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, - log: "", - status: { conclusion: "success", state: "completed" }, - }); - }); - - it("rejects a second job while the Jetson lock is held (#8142)", async () => { - let finish: (() => void) | undefined; - const run = vi.fn( - () => - new Promise((_resolve, reject) => { - finish = () => reject(new Error("finished by test")); - }), - ); - const dispatch = coordinator(temporaryDirectory(), worker({ run })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const other = { ...REQUEST, candidateSha: "c".repeat(40) }; - expect(() => dispatch.dispatch(other, identity())).toThrow(JetsonDispatchBusyError); - finish!(); - await waitForCompletion(dispatch, accepted.jobId); - }); - - it("persists the lock and queued status before worker execution (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const events: string[] = []; - const fsyncSync = fs.fsyncSync.bind(fs); - vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { - events.push(fs.fstatSync(descriptor).isDirectory() ? "directory-fsync" : "file-fsync"); - fsyncSync(descriptor); - }); - const unlinkSync = fs.unlinkSync.bind(fs); - vi.spyOn(fs, "unlinkSync").mockImplementation((file) => { - events.push(file === path.join(stateDirectory, "device.lock") ? "lock-unlink" : "unlink"); - unlinkSync(file); - }); - const run = vi.fn(async (_request, options) => { - events.push("worker-run"); - expect(fs.readFileSync(path.join(stateDirectory, "device.lock"), "utf8")).toBe( - `${options.jobId}\n`, - ); - expect( - JSON.parse(fs.readFileSync(path.join(stateDirectory, `${options.jobId}.json`), "utf8")), - ).toMatchObject({ jobId: options.jobId, state: "queued" }); - return worker().run(_request, options); - }); - const dispatch = coordinator(stateDirectory, worker({ run })); - await dispatch.initialize(); - - const accepted = dispatch.dispatch(REQUEST, identity()); - await waitForCompletion(dispatch, accepted.jobId); - - expect(events.indexOf("directory-fsync")).toBeLessThan(events.indexOf("worker-run")); - expect(events.indexOf("worker-run")).toBeLessThan(events.indexOf("lock-unlink")); - expect(events.indexOf("lock-unlink")).toBeLessThan(events.lastIndexOf("directory-fsync")); - }); - - it("does not start the worker when state directory persistence fails (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const fsyncSync = fs.fsyncSync.bind(fs); - let rejectDirectoryFsync = true; - vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { - const isDirectory = fs.fstatSync(descriptor).isDirectory(); - const shouldReject = rejectDirectoryFsync && isDirectory; - rejectDirectoryFsync = rejectDirectoryFsync && !isDirectory; - return shouldReject ? failTest("state directory persistence failed") : fsyncSync(descriptor); - }); - const run = vi.fn(worker().run); - const cleanup = vi.fn(async () => {}); - const dispatch = coordinator(stateDirectory, worker({ cleanup, run })); - await dispatch.initialize(); - - expect(() => dispatch.dispatch(REQUEST, identity())).toThrow( - "state directory persistence failed", - ); - - expect(run).not.toHaveBeenCalled(); - expect(cleanup).not.toHaveBeenCalled(); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - - const restarted = coordinator(stateDirectory, worker({ cleanup, run })); - await restarted.initialize(); - - expect(cleanup).toHaveBeenCalledOnce(); - expect(run).not.toHaveBeenCalled(); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); - }); - - it("cancels the worker and still cleans the device with a fresh signal (#8142)", async () => { - let runSignal: AbortSignal | undefined; - let cleanupSignal: AbortSignal | undefined; - const cleanup = vi.fn(async (options: { jobId: string; signal: AbortSignal }) => { - cleanupSignal = options.signal; - }); - const run = vi.fn( - (_request, options) => - new Promise((_resolve, reject) => { - runSignal = options.signal; - options.signal.addEventListener("abort", () => reject(new Error("cancelled")), { - once: true, - }); - }), - ); - const dispatch = coordinator(temporaryDirectory(), worker({ cleanup, run })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - dispatch.cancel(accepted.jobId); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ cleanup: "succeeded", conclusion: "cancelled" }); - expect(cleanup).toHaveBeenCalledOnce(); - expect(cleanupSignal).not.toBe(runSignal); - expect(cleanupSignal?.aborted).toBe(false); - }); - - it("times out the worker and still cleans the device (#8142)", async () => { - vi.useFakeTimers(); - const cleanup = vi.fn(async () => {}); - const run = vi.fn( - (_request, options) => - new Promise((_resolve, reject) => { - options.signal.addEventListener("abort", () => reject(new Error("timed out")), { - once: true, - }); - }), - ); - const dispatch = coordinator(temporaryDirectory(), worker({ cleanup, run })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - await vi.advanceTimersByTimeAsync(60_000); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ cleanup: "succeeded", conclusion: "timed-out" }); - expect(cleanup).toHaveBeenCalledOnce(); - }); - - it("keeps cleanup evidence accurate when lock removal fails (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const cleanup = vi.fn(async () => {}); - const unlinkSync = fs.unlinkSync.bind(fs); - vi.spyOn(fs, "unlinkSync") - .mockImplementationOnce(() => { - throw new Error("lock filesystem unavailable"); - }) - .mockImplementation(unlinkSync); - const dispatch = coordinator(stateDirectory, worker({ cleanup })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ - conclusion: "cleanup-failed", - error: "Jetson lock removal failed: lock filesystem unavailable", - cleanup: "succeeded", - }); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - - const restarted = coordinator(stateDirectory, worker({ cleanup })); - await restarted.initialize(); - - expect(cleanup).toHaveBeenCalledTimes(2); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); - }); - - it("restores the device lock when directory fsync fails after removal (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const cleanup = vi.fn(async () => {}); - const fsyncSync = fs.fsyncSync.bind(fs); - let directorySyncCount = 0; - vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { - const isDirectory = fs.fstatSync(descriptor).isDirectory(); - directorySyncCount += Number(isDirectory); - return isDirectory && directorySyncCount === 2 - ? failTest("lock directory persistence failed") - : fsyncSync(descriptor); - }); - const dispatch = coordinator(stateDirectory, worker({ cleanup })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ - conclusion: "cleanup-failed", - error: "Jetson lock removal failed: lock directory persistence failed", - cleanup: "succeeded", - }); - expect(directorySyncCount).toBe(3); - expect(fs.readFileSync(path.join(stateDirectory, "device.lock"), "utf8")).toBe( - `${accepted.jobId}\n`, - ); - - const restarted = coordinator(stateDirectory, worker({ cleanup })); - await restarted.initialize(); - - expect(cleanup).toHaveBeenCalledTimes(2); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); - }); - - it("blocks later dispatch when lock restoration cannot recreate the lock (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const lockPath = path.join(stateDirectory, "device.lock"); - const fsyncSync = fs.fsyncSync.bind(fs); - let directorySyncCount = 0; - vi.spyOn(fs, "fsyncSync").mockImplementation((descriptor) => { - const isDirectory = fs.fstatSync(descriptor).isDirectory(); - directorySyncCount += Number(isDirectory); - return isDirectory && directorySyncCount === 2 - ? failTest("lock directory persistence failed") - : fsyncSync(descriptor); - }); - const run = vi.fn(worker().run); - const dispatch = coordinator(stateDirectory, worker({ run })); - await dispatch.initialize(); - const openSync = fs.openSync.bind(fs); - let lockOpenCount = 0; - vi.spyOn(fs, "openSync").mockImplementation((file, flags, mode) => { - const isLock = file === lockPath; - lockOpenCount += Number(isLock); - return isLock && lockOpenCount === 2 - ? failTest("lock restoration unavailable") - : openSync(file, flags, mode); - }); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed.error).toContain("lock restoration failed: lock restoration unavailable"); - expect(fs.existsSync(lockPath)).toBe(false); - expect(() => - dispatch.dispatch({ ...REQUEST, candidateSha: "c".repeat(40) }, identity()), - ).toThrow("Jetson device lock requires recovery"); - expect(run).toHaveBeenCalledOnce(); - await expect(dispatch.shutdown()).rejects.toThrow( - "Jetson device lock state requires operator recovery", - ); - }); - - it("keeps the lock until restart persists a terminal result (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const expectedResult = await worker().run(REQUEST, { - jobId: "e".repeat(64), - signal: new AbortController().signal, - }); - let finishRun!: () => void; - const run = vi.fn(async () => { - await new Promise((resolve) => { - finishRun = resolve; - }); - return expectedResult; - }); - const first = coordinator(stateDirectory, worker({ run })); - await first.initialize(); - const accepted = first.dispatch(REQUEST, identity()); - await vi.waitFor(() => expect(run).toHaveBeenCalledOnce()); - const statusPath = path.join(stateDirectory, `${accepted.jobId}.json`); - const runningStatus = fs.readFileSync(statusPath, "utf8"); - fs.rmSync(statusPath); - fs.mkdirSync(statusPath); - - finishRun(); - const failedPersistence = await waitForCompletion(first, accepted.jobId); - - expect(failedPersistence).toMatchObject({ - cleanup: "succeeded", - conclusion: "failure", - error: expect.stringContaining("Final status persistence failed"), - }); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - - const recoveryCleanup = vi.fn(async () => {}); - await expect( - coordinator(stateDirectory, worker({ cleanup: recoveryCleanup })).initialize(), - ).rejects.toThrow(`Jetson dispatch status ${accepted.jobId}.json is invalid`); - expect(recoveryCleanup).toHaveBeenCalledOnce(); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - - fs.rmSync(statusPath, { recursive: true }); - fs.writeFileSync(statusPath, runningStatus, { mode: 0o600 }); - const replayRun = vi.fn(worker().run); - const restarted = coordinator( - stateDirectory, - worker({ cleanup: recoveryCleanup, run: replayRun }), - ); - await restarted.initialize(); - - expect(restarted.status(accepted.jobId)).toMatchObject({ - cleanup: "succeeded", - conclusion: "failure", - error: "Jetson dispatcher restarted before terminal status was persisted", - state: "completed", - }); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); - expect(recoveryCleanup).toHaveBeenCalledTimes(2); - expect(restarted.dispatch(REQUEST, identity()).state).toBe("completed"); - expect(replayRun).not.toHaveBeenCalled(); - }); - - it("restores completed evidence without replaying the job after restart (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const first = coordinator(stateDirectory, worker()); - await first.initialize(); - const firstAccepted = first.dispatch(REQUEST, identity()); - const firstCompleted = await waitForCompletion(first, firstAccepted.jobId); - expect(first.artifact(firstAccepted.jobId).artifactArchiveBase64).toBe(ARTIFACT_ARCHIVE_BASE64); - - const replayRun = vi.fn(worker().run); - const restarted = coordinator(stateDirectory, worker({ run: replayRun })); - await restarted.initialize(); - const restoredStatus = restarted.status(firstAccepted.jobId); - const restoredArtifact = restarted.artifact(firstAccepted.jobId); - const replayAccepted = restarted.dispatch(REQUEST, identity()); - - expect(replayAccepted.jobId).toBe(firstAccepted.jobId); - expect(replayAccepted).toEqual(firstCompleted); - expect(restoredStatus).toEqual(firstCompleted); - expect(restoredArtifact).toMatchObject({ - artifactArchiveBase64: ARTIFACT_ARCHIVE_BASE64, - log: "passed\n", - status: firstCompleted, - }); - expect(replayRun).not.toHaveBeenCalled(); - }); - - it("rejects initialization when a completed status record is invalid (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const first = coordinator(stateDirectory, worker()); - await first.initialize(); - const accepted = first.dispatch(REQUEST, identity()); - await waitForCompletion(first, accepted.jobId); - const statusPath = path.join(stateDirectory, `${accepted.jobId}.json`); - const persisted = JSON.parse(fs.readFileSync(statusPath, "utf8")); - const invalidRecords = [ - { - contents: `${JSON.stringify({ ...persisted, jobId: "f".repeat(64) })}\n`, - error: "persisted Jetson dispatch status does not match its job ID", - }, - { contents: "{", error: "persisted Jetson dispatch status is not valid JSON" }, - { contents: " ".repeat(16 * 1024 + 1), error: "exceeds 16384 bytes" }, - ]; - - for (const record of invalidRecords) { - fs.writeFileSync(statusPath, record.contents); - await expect(coordinator(stateDirectory, worker()).initialize()).rejects.toThrow( - new RegExp(`Jetson dispatch status ${accepted.jobId}\\.json is invalid: .*${record.error}`), - ); - } - }); - - it( - "restores private completed status after in-memory eviction (#8142)", - testTimeoutOptions(10_000), - async () => { - const stateDirectory = temporaryDirectory(); - const dispatch = coordinator(stateDirectory, worker()); - await dispatch.initialize(); - let firstJobId = ""; - for (let index = 0; index < 129; index += 1) { - const accepted = dispatch.dispatch( - { ...REQUEST, candidateSha: index.toString(16).padStart(40, "0") }, - identity(), - ); - firstJobId ||= accepted.jobId; - await waitForCompletion(dispatch, accepted.jobId); - } - - expect(dispatch.status(firstJobId)).toMatchObject({ state: "completed" }); - expect(fs.existsSync(path.join(stateDirectory, `${firstJobId}.json`))).toBe(true); - fs.writeFileSync(path.join(stateDirectory, `${firstJobId}.json`), "{"); - await expect(coordinator(stateDirectory, worker()).initialize()).rejects.toThrow( - `Jetson dispatch status ${firstJobId}.json is invalid`, - ); - }, - ); - - it("keeps the device locked when cleanup loses a pre-existing Ollama model (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const dispatch = coordinator( - stateDirectory, - worker({ - cleanup: async () => - Promise.reject( - new Error("Jetson cleanup did not preserve every pre-existing Ollama model"), - ), - }), - ); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ cleanup: "failed", conclusion: "cleanup-failed" }); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - expect(() => - dispatch.dispatch({ ...REQUEST, candidateSha: "c".repeat(40) }, identity()), - ).toThrow("Jetson device lock requires recovery"); - await expect(dispatch.shutdown()).rejects.toThrow( - "Jetson device lock still requires cleanup recovery", - ); - }); - - it("reports both candidate and cleanup failures while retaining the lock (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const dispatch = coordinator( - stateDirectory, - worker({ - cleanup: async () => Promise.reject(new Error("cleanup verification failed")), - run: async () => Promise.reject(new Error("artifact collection failed")), - }), - ); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed).toMatchObject({ - cleanup: "failed", - conclusion: "cleanup-failed", - error: "artifact collection failed; Jetson cleanup failed: cleanup verification failed", - }); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - }); - - it("keeps a cleanup failure visible after a 500-character execution error (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const dispatch = coordinator( - stateDirectory, - worker({ - cleanup: async () => Promise.reject(new Error("cleanup verification failed")), - run: async () => Promise.reject(new Error("x".repeat(500))), - }), - ); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed.error).toContain("Jetson cleanup failed: cleanup verification failed"); - expect(completed.error).toHaveLength(500); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - }); - - it("keeps a final persistence failure visible after a 500-character execution error (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - let rejectRun!: (error: Error) => void; - const run = vi.fn( - () => - new Promise((_resolve, reject) => { - rejectRun = reject; - }), - ); - const dispatch = coordinator(stateDirectory, worker({ run })); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - await vi.waitFor(() => expect(run).toHaveBeenCalledOnce()); - const statusPath = path.join(stateDirectory, `${accepted.jobId}.json`); - fs.rmSync(statusPath); - fs.mkdirSync(statusPath); - rejectRun(new Error("x".repeat(500))); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed.error).toContain("Final status persistence failed:"); - expect(completed.error).toHaveLength(500); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - }); - - it("keeps a lock-removal failure visible after a 500-character execution error (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { - throw new Error("lock filesystem unavailable"); - }); - const dispatch = coordinator( - stateDirectory, - worker({ run: async () => Promise.reject(new Error("x".repeat(500))) }), - ); - await dispatch.initialize(); - const accepted = dispatch.dispatch(REQUEST, identity()); - const completed = await waitForCompletion(dispatch, accepted.jobId); - - expect(completed.error).toContain("Jetson lock removal failed: lock filesystem unavailable"); - expect(completed.error).toHaveLength(500); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - }); - - it("cleans a stale lock before accepting work after restart (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - const staleJobId = "d".repeat(64); - fs.writeFileSync(path.join(stateDirectory, "device.lock"), `${staleJobId}\n`, { mode: 0o600 }); - const cleanup = vi.fn(async () => {}); - const dispatch = coordinator(stateDirectory, worker({ cleanup })); - await dispatch.initialize(); - - expect(cleanup).toHaveBeenCalledWith({ jobId: staleJobId, signal: expect.any(AbortSignal) }); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(false); - }); - - it("rejects a stale lock that cannot identify one dispatcher job (#8142)", async () => { - const stateDirectory = temporaryDirectory(); - fs.writeFileSync(path.join(stateDirectory, "device.lock"), "../../arbitrary-path\n", { - mode: 0o600, - }); - const cleanup = vi.fn(async () => {}); - const dispatch = coordinator(stateDirectory, worker({ cleanup })); - - await expect(dispatch.initialize()).rejects.toThrow( - "Jetson device lock contains an invalid job ID", - ); - expect(cleanup).not.toHaveBeenCalled(); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - }); - - it.each([ - `${"d".repeat(64)}\nextra\n`, - `${"d".repeat(64)}\n\n`, - ])("rejects trailing data in a stale device lock (#8142)", async (lockContents) => { - const stateDirectory = temporaryDirectory(); - fs.writeFileSync(path.join(stateDirectory, "device.lock"), lockContents, { mode: 0o600 }); - const cleanup = vi.fn(async () => {}); - const dispatch = coordinator(stateDirectory, worker({ cleanup })); - - await expect(dispatch.initialize()).rejects.toThrow( - "Jetson device lock contains an invalid job ID", - ); - expect(cleanup).not.toHaveBeenCalled(); - expect(fs.existsSync(path.join(stateDirectory, "device.lock"))).toBe(true); - }); -}); - -describe("Jetson dispatch HTTP boundary", () => { - it("reports the controller deadline when cancellation also fails (#8142)", async () => { - const jobId = "d".repeat(64); - const request = vi.fn().mockRejectedValue(new Error("tunnel unavailable")); - - await expect( - pollJetsonDispatch({ - baseUrl: new URL("https://dispatch.test/"), - deadlineMs: 10_000, - initialStatus: { - schemaVersion: 1, - jobId, - request: REQUEST, - state: "queued", - createdAt: new Date(0).toISOString(), - cleanup: "pending", - }, - jobId, - now: () => 10_000, - request, - wait: async () => {}, - }), - ).rejects.toThrow("Jetson dispatcher did not complete before the controller deadline"); - expect(request).toHaveBeenCalledWith( - expect.objectContaining({ method: "DELETE", path: `v1/jobs/${jobId}` }), - ); - }); - - it("retries transient status failures and cancels after the bounded limit (#8142)", async () => { - vi.spyOn(console, "warn").mockImplementation(() => {}); - const jobId = "d".repeat(64); - const initialStatus: JetsonDispatchStatus = { - schemaVersion: 1, - jobId, - request: REQUEST, - state: "queued", - createdAt: new Date(0).toISOString(), - cleanup: "pending", - }; - const completed = { - job: { - ...initialStatus, - state: "completed", - conclusion: "success", - cleanup: "succeeded", - }, - }; - const recoveredRequest = vi - .fn() - .mockRejectedValueOnce(new Error("tunnel reset")) - .mockRejectedValueOnce(new Error("tunnel reset")) - .mockResolvedValueOnce(completed); - - await expect( - pollJetsonDispatch({ - baseUrl: new URL("https://dispatch.test/"), - deadlineMs: 10_000, - initialStatus, - jobId, - now: () => 0, - request: recoveredRequest, - wait: async () => {}, - }), - ).resolves.toMatchObject({ state: "completed" }); - - const failedRequest = vi.fn().mockRejectedValue(new Error("tunnel unavailable")); - await expect( - pollJetsonDispatch({ - baseUrl: new URL("https://dispatch.test/"), - deadlineMs: 10_000, - initialStatus, - jobId, - now: () => 0, - request: failedRequest, - wait: async () => {}, - }), - ).rejects.toThrow("tunnel unavailable"); - expect(failedRequest).toHaveBeenLastCalledWith( - expect.objectContaining({ method: "DELETE", path: `v1/jobs/${jobId}` }), - ); - }); - - it("denies anonymous requests and returns authenticated result evidence (#8142)", async () => { - const keys = signingKeys(); - const now = Math.floor(Date.now() / 1_000); - const token = signToken( - keys.privateKey, - claims({ exp: now + 9 * 60, iat: now - 60, nbf: now - 60 }), - ); - const dispatch = coordinator(temporaryDirectory(), worker()); - await dispatch.initialize(); - const server = createJetsonDispatchServer({ - coordinator: dispatch, - repositoryId: REPOSITORY_ID, - resolveSigningKey: async () => keys.publicKey, - }); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", resolve); - }); - const { port } = server.address() as AddressInfo; - const url = `http://127.0.0.1:${port}/v1/jobs`; - try { - const anonymous = await fetch(url, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(REQUEST), - }); - expect(anonymous.status).toBe(401); - - const injected = await fetch(url, { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify({ ...REQUEST, command: "uname -a" }), - }); - expect(injected.status).toBe(400); - - const accepted = await fetch(url, { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify(REQUEST), - }); - expect(accepted.status).toBe(202); - const acceptedBody = (await accepted.json()) as { job: { jobId: string } }; - await waitForCompletion(dispatch, acceptedBody.job.jobId); - - const anonymousExisting = await fetch(`${url}/${acceptedBody.job.jobId}`); - const anonymousUnknown = await fetch(`${url}/${"f".repeat(64)}`); - const authenticatedUnknown = await fetch(`${url}/${"f".repeat(64)}`, { - headers: { Authorization: `Bearer ${token}` }, - }); - expect([ - anonymousExisting.status, - anonymousUnknown.status, - authenticatedUnknown.status, - ]).toEqual([401, 401, 401]); - - const artifact = await fetch(`${url}/${acceptedBody.job.jobId}/artifact`, { - headers: { Authorization: `Bearer ${token}` }, - }); - expect(artifact.status).toBe(200); - await expect(artifact.json()).resolves.toMatchObject({ - log: "passed\n", - status: { cleanup: "succeeded", conclusion: "success" }, - }); - } finally { - await new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ); - } - }); -}); diff --git a/test/e2e/support/jetson-dispatch-worker.test.ts b/test/e2e/support/jetson-dispatch-worker.test.ts deleted file mode 100644 index b352a9818ab..00000000000 --- a/test/e2e/support/jetson-dispatch-worker.test.ts +++ /dev/null @@ -1,1411 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { - loadSshJetsonWorkerConfig, - ProcessFailure, - SshJetsonDispatchWorker, -} from "../../../tools/e2e/jetson-dispatch-worker.mts"; - -const dispatcherRunbook = fs.readFileSync( - path.join(process.cwd(), "test/e2e/docs/jetson-colossus-dispatch.md"), - "utf8", -); -const cleanupProgram = fs.readFileSync( - path.join(process.cwd(), "tools/e2e/jetson-dispatch-cleanup.sh"), - "utf8", -); -const dispatcherServiceUnit = fs.readFileSync( - path.join(process.cwd(), "tools/e2e/nemoclaw-jetson-dispatch.service"), - "utf8", -); -const workerProgram = fs.readFileSync( - path.join(process.cwd(), "tools/e2e/jetson-dispatch-worker.mts"), - "utf8", -); -const jetsonLiveTest = fs.readFileSync( - path.join(process.cwd(), "test/e2e/live/jetson-nvmap-gpu.test.ts"), - "utf8", -); -const temporaryDirectories: string[] = []; -const BASELINE = { - nodePath: "/usr/bin/node", - nodeVersion: "v22.19.0", - npmPath: "/usr/bin/npm", - npmVersion: "10.9.3", - ollamaPath: "/usr/local/bin/ollama", - ollamaModels: [{ name: "existing:latest", id: "a".repeat(12) }], - openshellState: "absent", -}; - -function baselineOutput(overrides: Partial = {}): string { - const baseline = { ...BASELINE, ...overrides }; - const ollamaModelsBase64 = Buffer.from( - baseline.ollamaModels.map((model) => `${model.name}\t${model.id}`).join("\n"), - ).toString("base64"); - return `${Object.entries({ - nodePath: baseline.nodePath, - nodeVersion: baseline.nodeVersion, - npmPath: baseline.npmPath, - npmVersion: baseline.npmVersion, - ollamaPath: baseline.ollamaPath, - ollamaModelsBase64, - openshellState: baseline.openshellState, - }) - .map(([key, value]) => `${key}\t${value}`) - .join("\n")}\n`; -} - -const BASELINE_OUTPUT = baselineOutput(); - -function cleanupOutput({ - volumes = [], - processIds = [], -}: { - volumes?: string[]; - processIds?: number[]; -} = {}): string { - return [ - "nemoclaw-cleanup-evidence-v1-begin", - ...volumes.map((volume) => `volume\t${volume}`), - ...processIds.map((pid) => `processId\t${pid}`), - "nemoclaw-cleanup-evidence-v1-end", - "", - ].join("\n"); -} - -function persistCleanupEvidence( - files: ReturnType, - jobId: string, - evidence: { schemaVersion: 1; volumes: string[]; processIds: number[] }, -): string { - const file = path.join(files.stateDirectory, `${jobId}.cleanup.json`); - fs.writeFileSync(file, `${JSON.stringify(evidence)}\n`, { mode: 0o600 }); - return file; -} - -function cleanupProcessFixture( - options: { - command?: string; - deny?: "cmdline" | "environ"; - disappear?: "cmdline" | "environ"; - home?: string; - uid?: number; - } = {}, -) { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-jetson-proc-")); - temporaryDirectories.push(directory); - const procRoot = path.join(directory, "proc"); - const procDirectory = path.join(procRoot, "1234"); - const binDirectory = path.join(directory, "bin"); - fs.mkdirSync(procDirectory, { recursive: true }); - fs.mkdirSync(binDirectory); - fs.writeFileSync( - path.join(procDirectory, "status"), - `Name:\ttest\nUid:\t${options.uid ?? process.getuid!()}\t0\t0\t0\n`, - ); - fs.writeFileSync( - path.join(procDirectory, "environ"), - Buffer.from(`HOME=${options.home ?? "/job/home"}\0PATH=/usr/bin\0`), - ); - fs.writeFileSync( - path.join(procDirectory, "cmdline"), - Buffer.from(`${options.command ?? "node ollama-auth-proxy.mjs"}\0`), - ); - fs.writeFileSync( - path.join(binDirectory, "dd"), - `#!/usr/bin/env bash -set -eu -input= -for argument in "$@"; do - case "$argument" in if=*) input="${"${argument#if=}"}" ;; esac -done -if [ "${"${DENY_PROC_FILE:-}"}" = "$input" ]; then exit 1; fi -if [ "${"${DISAPPEAR_PROC_FILE:-}"}" = "$input" ]; then rm -rf -- "${"${input%/*}"}"; exit 1; fi -/bin/cat "$input" -`, - { mode: 0o700 }, - ); - const functionsStart = cleanupProgram.indexOf("read_proc_uid() {"); - const functionsEnd = cleanupProgram.indexOf("\nauth_proxy_pid=", functionsStart); - assert.ok(functionsStart >= 0 && functionsEnd >= 0, "cleanup process functions are absent"); - const functions = cleanupProgram - .slice(functionsStart, functionsEnd) - .replace('proc_dir="/proc/$pid"', 'proc_dir="$PROC_ROOT/$pid"'); - return { - binDirectory, - environment: { - ...process.env, - DENY_PROC_FILE: options.deny ? path.join(procDirectory, options.deny) : "", - DISAPPEAR_PROC_FILE: options.disappear ? path.join(procDirectory, options.disappear) : "", - PATH: `${binDirectory}:${process.env.PATH ?? ""}`, - PROC_ROOT: procRoot, - }, - functions, - procRoot, - }; -} - -afterEach(() => { - for (const directory of temporaryDirectories.splice(0)) { - fs.rmSync(directory, { force: true, recursive: true }); - } -}); - -function deploymentFiles() { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-jetson-worker-")); - temporaryDirectories.push(directory); - const identityFile = path.join(directory, "id_ed25519"); - const knownHostsFile = path.join(directory, "known_hosts"); - const cleanupExecutable = path.join(directory, "cleanup-jetson"); - fs.writeFileSync(identityFile, "test-key", { mode: 0o600 }); - fs.writeFileSync(knownHostsFile, "test-host-key", { mode: 0o600 }); - fs.writeFileSync(cleanupExecutable, "#!/bin/sh\nexit 0\n", { mode: 0o700 }); - return { cleanupExecutable, identityFile, knownHostsFile, stateDirectory: directory }; -} - -function environment(files = deploymentFiles()): NodeJS.ProcessEnv { - return { - JETSON_DISPATCH_SSH_DESTINATION: "nvidia@192.168.55.1", - JETSON_DISPATCH_SSH_IDENTITY_FILE: files.identityFile, - JETSON_DISPATCH_SSH_KNOWN_HOSTS_FILE: files.knownHostsFile, - JETSON_DISPATCH_CLEANUP_EXECUTABLE: files.cleanupExecutable, - JETSON_DISPATCH_TEST_TIMEOUT_SECONDS: "2700", - }; -} - -function loadConfig(files = deploymentFiles()) { - return loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, environment(files)); -} - -describe("Colossus SSH worker deployment boundary", () => { - it("loads the dispatcher service with direct Node.js type stripping (#8142)", () => { - const service = pathToFileURL( - path.join(process.cwd(), "tools/e2e/jetson-dispatch-service.mts"), - ).href; - const result = spawnSync( - "node", - [ - "--experimental-strip-types", - "--no-warnings", - "--input-type=module", - "--eval", - `await import(${JSON.stringify(service)})`, - ], - { encoding: "utf8" }, - ); - - expect(result.status, result.stderr).toBe(0); - expect(result.stderr).not.toContain("does not provide an export named JetsonDeviceIdentity"); - }); - - it("uses one fixed command for initial service deployment (#8142)", () => { - const deployment = - "sudo nemoclaw-colossus-jetson-dispatch-deploy --commit 0000000000000000000000000000000000000000"; - const configure = dispatcherRunbook.indexOf("## Configure the Dispatcher Service"); - expect(dispatcherRunbook).toContain( - "tools/e2e/colossus-jetson-dispatch-deploy.sh \\\n /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy", - ); - expect(dispatcherRunbook).toContain(deployment); - expect(dispatcherRunbook).toContain("tools/e2e/colossus-jetson-dispatch.environment"); - expect(dispatcherRunbook).toContain("tools/e2e/nemoclaw-jetson-dispatch.service"); - expect(dispatcherServiceUnit).toContain( - "WorkingDirectory=/opt/nemoclaw-jetson-dispatch/current", - ); - expect(dispatcherRunbook.indexOf(deployment)).toBeGreaterThan(configure); - expect(dispatcherRunbook.indexOf(deployment)).toBeLessThan( - dispatcherRunbook.indexOf("## Deploy a Later Reviewed Commit"), - ); - expect(dispatcherRunbook).not.toContain("sudo git -C /opt/nemoclaw-jetson-dispatch init"); - expect(dispatcherRunbook).not.toContain("git clone --branch main"); - }); - - it("distinguishes pinned SSH identity from the persisted cleanup baseline (#8142)", () => { - expect(dispatcherRunbook).toContain( - "The pinned `known_hosts` file validates the SSH host identity separately from the persisted cleanup baseline.", - ); - expect(dispatcherRunbook).not.toContain( - "Treat the verified SSH host identity as part of the protected baseline.", - ); - }); - - it("keeps public ingress stopped until a later release passes local verification (#8142)", () => { - const section = dispatcherRunbook.slice( - dispatcherRunbook.indexOf("## Deploy a Later Reviewed Commit"), - dispatcherRunbook.indexOf("## Publish the Dispatcher With Cloudflare Tunnel"), - ); - const stopTunnel = "sudo systemctl stop nemoclaw-jetson-tunnel.service"; - const deploy = "sudo /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy"; - const startTunnel = "sudo systemctl start nemoclaw-jetson-tunnel.service"; - const publicProof = 'test "$PUBLIC_HTTP_CODE" = 401'; - - expect(section).toContain("set -euo pipefail"); - expect(section.indexOf(stopTunnel)).toBeLessThan(section.indexOf(deploy)); - expect(section.indexOf(deploy)).toBeLessThan(section.indexOf(startTunnel)); - expect(section.indexOf(startTunnel)).toBeLessThan(section.indexOf(publicProof)); - expect(section.indexOf(publicProof)).toBeLessThan(section.indexOf("tunnel_verified=1")); - expect(section).toContain("trap stop_unverified_tunnel EXIT"); - expect(section).toContain("curl --disable --noproxy '*'"); - }); - - it.each([ - { - expectedContainmentDiagnostics: [], - expectedLog: ["stop", "deploy", "start", "stop"], - expectedState: "inactive\n", - failContainmentStop: false, - name: "stops public ingress when the public proof fails", - }, - { - expectedContainmentDiagnostics: [ - "PUBLIC INGRESS CONTAINMENT FAILED: tunnel stop status=5; ActiveState=active; inspection status=0", - ], - expectedLog: ["stop", "deploy", "start", "stop-failed"], - expectedState: "active\n", - failContainmentStop: true, - name: "reports when public-ingress containment fails", - }, - ])("$name (#8142)", ({ - expectedContainmentDiagnostics, - expectedLog, - expectedState, - failContainmentStop, - }) => { - const section = dispatcherRunbook.slice( - dispatcherRunbook.indexOf("## Deploy a Later Reviewed Commit"), - dispatcherRunbook.indexOf("## Publish the Dispatcher With Cloudflare Tunnel"), - ); - const block = section.match(/```bash\n([\s\S]*?)\n```/u)?.[1]; - expect(block).toBeDefined(); - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-tunnel-deploy-test-")); - temporaryDirectories.push(directory); - const log = path.join(directory, "service.log"); - const state = path.join(directory, "tunnel.state"); - const stopCount = path.join(directory, "stop.count"); - fs.writeFileSync(state, "active\n"); - fs.writeFileSync( - path.join(directory, "sudo"), - `#!/bin/sh -set -eu -case "$1:$2" in - systemctl:stop) - count="$(cat "$STOP_COUNT" 2>/dev/null || printf '0')" - count="$((count + 1))" - printf '%s\\n' "$count" >"$STOP_COUNT" - if [ "$FAIL_CONTAINMENT_STOP" = 1 ] && [ "$count" -gt 1 ]; then - printf 'stop-failed\\n' >>"$SERVICE_LOG" - exit 5 - fi - printf 'stop\\n' >>"$SERVICE_LOG" - printf 'inactive\\n' >"$TUNNEL_STATE" - ;; - systemctl:start) printf 'start\\n' >>"$SERVICE_LOG"; printf 'active\\n' >"$TUNNEL_STATE" ;; - systemctl:show) cat "$TUNNEL_STATE" ;; - /usr/local/sbin/nemoclaw-colossus-jetson-dispatch-deploy:--commit) printf 'deploy\\n' >>"$SERVICE_LOG" ;; - *) exit 1 ;; -esac -`, - { mode: 0o755 }, - ); - fs.writeFileSync(path.join(directory, "curl"), "#!/bin/sh\nprintf '503'\n", { mode: 0o755 }); - - const result = spawnSync("/bin/bash", ["-c", block!], { - encoding: "utf8", - env: { - ...process.env, - FAIL_CONTAINMENT_STOP: failContainmentStop ? "1" : "0", - PATH: `${directory}:${process.env.PATH ?? ""}`, - SERVICE_LOG: log, - STOP_COUNT: stopCount, - TUNNEL_STATE: state, - }, - }); - - expect(result.status).not.toBe(0); - expect(fs.readFileSync(log, "utf8").trim().split("\n")).toEqual(expectedLog); - expect(fs.readFileSync(state, "utf8")).toBe(expectedState); - expect( - result.stderr - .split("\n") - .filter((line) => line.startsWith("PUBLIC INGRESS CONTAINMENT FAILED:")), - ).toEqual(expectedContainmentDiagnostics); - }); - - it("loads fixed SSH, host-key, timeout, and cleanup configuration (#8142)", () => { - const files = deploymentFiles(); - expect(loadConfig(files)).toEqual({ - cleanupExecutable: files.cleanupExecutable, - destination: "nvidia@192.168.55.1", - identityFile: files.identityFile, - knownHostsFile: files.knownHostsFile, - stateDirectory: files.stateDirectory, - testTimeoutSeconds: 2700, - }); - }); - - it("collects a bounded artifact archive before the remote workspace is removed (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); - fs.writeFileSync(baselinePath, `${JSON.stringify({ ...BASELINE, nodeVersion: "v22.0.0" })}\n`, { - mode: 0o600, - }); - const artifactArchiveBase64 = Buffer.from("remote archive").toString("base64"); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) - .mockResolvedValueOnce({ - stdout: - "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", - stderr: "", - }) - .mockResolvedValueOnce({ stdout: "test passed\n", stderr: "" }) - .mockResolvedValueOnce({ stdout: artifactArchiveBase64, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect( - worker.run( - { - schemaVersion: 1, - target: "jetson-nvmap-gpu", - candidateSha: "a".repeat(40), - workflowRunId: "123", - workflowRunAttempt: 1, - }, - { jobId, signal: new AbortController().signal }, - ), - ).resolves.toMatchObject({ - artifactArchiveBase64, - log: expect.stringContaining("test passed"), - }); - expect(processRunner.mock.calls[0]![0].input).toContain( - "major < 22 || (major === 22 && minor < 19)", - ); - expect(processRunner.mock.calls[0]![0].input).toContain("npm_major >= 10"); - expect(processRunner.mock.calls[2]![0].input).toContain( - "major < 22 || (major === 22 && minor < 19)", - ); - expect(processRunner.mock.calls[2]![0].input).toContain("npm_major >= 10"); - expect(processRunner.mock.calls[2]![0].input).toContain('"$workspace/runtime"'); - expect(processRunner.mock.calls[2]![0].input).toContain( - 'export XDG_RUNTIME_DIR="$workspace/runtime"', - ); - expect(processRunner.mock.calls[2]![0].input).toContain("unset DBUS_SESSION_BUS_ADDRESS"); - expect(JSON.parse(fs.readFileSync(baselinePath, "utf8"))).toEqual(BASELINE); - expect(processRunner.mock.calls[2]![0].input).not.toContain("trap cleanup EXIT"); - expect(processRunner.mock.calls[3]![0].input).not.toContain("trap cleanup EXIT"); - expect(processRunner.mock.calls[3]![0].input).toContain("base64 --wrap=0"); - }); - - it("preserves test-failure logs when artifact collection fails (#8142)", async () => { - const files = deploymentFiles(); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) - .mockResolvedValueOnce({ - stdout: - "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", - stderr: "", - }) - .mockRejectedValueOnce( - new ProcessFailure("Jetson test failed", { - stdout: "failed test output\n", - stderr: "failed test error\n", - }), - ) - .mockRejectedValueOnce(new Error("artifact collection unavailable")); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect( - worker.run( - { - schemaVersion: 1, - target: "jetson-nvmap-gpu", - candidateSha: "a".repeat(40), - workflowRunId: "123", - workflowRunAttempt: 1, - }, - { jobId: "b".repeat(64), signal: new AbortController().signal }, - ), - ).rejects.toMatchObject({ - message: "Jetson test failed; artifact collection failed: artifact collection unavailable", - log: expect.stringContaining("failed test output"), - }); - }); - - it("preserves successful test logs when artifact collection fails (#8142)", async () => { - const files = deploymentFiles(); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) - .mockResolvedValueOnce({ - stdout: - "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", - stderr: "", - }) - .mockResolvedValueOnce({ stdout: "successful test output\n", stderr: "" }) - .mockRejectedValueOnce(new Error("artifact collection unavailable")); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect( - worker.run( - { - schemaVersion: 1, - target: "jetson-nvmap-gpu", - candidateSha: "a".repeat(40), - workflowRunId: "123", - workflowRunAttempt: 1, - }, - { jobId: "b".repeat(64), signal: new AbortController().signal }, - ), - ).rejects.toMatchObject({ - message: "artifact collection unavailable", - log: expect.stringContaining("successful test output"), - }); - }); - - it.each([ - "nvidia@192.168.55.2", - "nvidia@192.168.55.1 -o StrictHostKeyChecking=no", - "nvidia@host;uname", - "root@", - "-oProxyCommand=unsafe", - ])("rejects an SSH destination that is not the fixed Jetson user and host: %s (#8142)", (destination) => { - const files = deploymentFiles(); - expect(() => - loadSshJetsonWorkerConfig( - { stateDirectory: files.stateDirectory }, - { - ...environment(files), - JETSON_DISPATCH_SSH_DESTINATION: destination, - }, - ), - ).toThrow("JETSON_DISPATCH_SSH_DESTINATION must be nvidia@192.168.55.1"); - }); - - it("rejects a writable SSH identity file (#8142)", () => { - const files = deploymentFiles(); - fs.chmodSync(files.identityFile, 0o666); - expect(() => loadConfig(files)).toThrow("must not be group- or world-writable"); - }); - - it("rejects a group-readable SSH identity file (#8142)", () => { - const files = deploymentFiles(); - fs.chmodSync(files.identityFile, 0o640); - expect(() => loadConfig(files)).toThrow("must be readable only by its owner"); - }); - - it("rejects a symbolic-link cleanup executable (#8142)", () => { - const files = deploymentFiles(); - const link = path.join(path.dirname(files.cleanupExecutable), "cleanup-link"); - fs.symlinkSync(files.cleanupExecutable, link); - expect(() => - loadSshJetsonWorkerConfig( - { stateDirectory: files.stateDirectory }, - { - ...environment(files), - JETSON_DISPATCH_CLEANUP_EXECUTABLE: link, - }, - ), - ).toThrow("must be a regular file"); - }); - - it("accepts only the exact root-owned current-release cleanup link (#8142)", () => { - const files = deploymentFiles(); - const cleanupExecutable = "/usr/local/libexec/nemoclaw-jetson-cleanup"; - const cleanupTarget = - "/opt/nemoclaw-jetson-dispatch/current/tools/e2e/jetson-dispatch-cleanup.sh"; - const localLink = path.join(files.stateDirectory, "managed-cleanup-link"); - fs.symlinkSync(files.cleanupExecutable, localLink); - const realLstatSync = fs.lstatSync.bind(fs); - const realReadlinkSync = fs.readlinkSync.bind(fs); - const localLinkStat = realLstatSync(localLink); - const rootLinkStat = Object.assign( - Object.create(Object.getPrototypeOf(localLinkStat)), - localLinkStat, - { uid: 0 }, - ) as fs.Stats; - const nonRootLinkStat = Object.assign( - Object.create(Object.getPrototypeOf(localLinkStat)), - localLinkStat, - { uid: 1000 }, - ) as fs.Stats; - const secureTargetStat = realLstatSync(files.cleanupExecutable); - const unsafeTargetStat = Object.assign( - Object.create(Object.getPrototypeOf(secureTargetStat)), - secureTargetStat, - { mode: secureTargetStat.mode | 0o022 }, - ) as fs.Stats; - let selectedTarget = "/tmp/unmanaged-cleanup"; - let managedLinkStat = rootLinkStat; - let managedTargetStat = secureTargetStat; - const lstatOverrides = new Map([ - [cleanupExecutable, () => managedLinkStat], - [cleanupTarget, () => managedTargetStat], - ]); - const lstat = vi.spyOn(fs, "lstatSync").mockImplementation((candidate, options) => { - return ( - lstatOverrides.get(String(candidate))?.() ?? realLstatSync(candidate, options as never) - ); - }); - const readlink = vi - .spyOn(fs, "readlinkSync") - .mockImplementation(((candidate, options) => - String(candidate) === cleanupExecutable - ? selectedTarget - : realReadlinkSync(candidate, options as never)) as typeof fs.readlinkSync); - - try { - const env = { - ...environment(files), - JETSON_DISPATCH_CLEANUP_EXECUTABLE: cleanupExecutable, - }; - expect(() => - loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), - ).toThrow("must select the managed current-release cleanup program"); - - selectedTarget = cleanupTarget; - managedLinkStat = nonRootLinkStat; - expect(() => - loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), - ).toThrow("symbolic link must be owned by root"); - - managedLinkStat = rootLinkStat; - managedTargetStat = unsafeTargetStat; - expect(() => - loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), - ).toThrow("must not be group- or world-writable"); - - managedTargetStat = secureTargetStat; - expect( - loadSshJetsonWorkerConfig({ stateDirectory: files.stateDirectory }, env), - ).toMatchObject({ cleanupExecutable }); - } finally { - readlink.mockRestore(); - lstat.mockRestore(); - } - }); - - it("invokes the fixed cleanup executable without request-controlled arguments (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); - fs.writeFileSync(baselinePath, `${JSON.stringify(BASELINE)}\n`, { mode: 0o600 }); - persistCleanupEvidence(files, jobId, { - schemaVersion: 1, - volumes: ["sandbox-volume"], - processIds: [1234], - }); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ - stdout: cleanupOutput({ volumes: ["sandbox-volume"], processIds: [1234] }), - stderr: "", - }) - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect( - worker.cleanup({ jobId, signal: new AbortController().signal }), - ).resolves.toBeUndefined(); - - expect(processRunner.mock.calls[0]![0]).toMatchObject({ - args: [], - executable: files.cleanupExecutable, - }); - expect(processRunner.mock.calls[0]![0].input).toBeUndefined(); - expect(processRunner.mock.calls[1]![0]).toMatchObject({ - executable: "ssh", - input: expect.stringContaining('workspace="/var/tmp/nemoclaw-jetson-e2e/$job_id"'), - }); - expect(processRunner.mock.calls[1]![0].input).toContain("docker info --format"); - expect(processRunner.mock.calls[1]![0].input).toContain("test -c /dev/nvmap"); - expect(processRunner.mock.calls[1]![0].input).toContain("ollama list"); - expect(processRunner.mock.calls[1]![0].input).toContain( - 'for volume in "${recorded_volumes[@]}"', - ); - expect(processRunner.mock.calls[1]![0].input).toContain( - 'for pid in "${recorded_process_ids[@]}"', - ); - expect(processRunner.mock.calls[1]![0].input).toContain("docker container ls --all --no-trunc"); - expect(processRunner.mock.calls[1]![0].input).toContain( - "docker volume ls --format '{{.Name}}'", - ); - expect(processRunner.mock.calls[1]![0].input).not.toContain("docker container inspect"); - expect(processRunner.mock.calls[1]![0].input).not.toContain("docker volume inspect"); - expect(processRunner.mock.calls[1]![0].args).toEqual( - expect.arrayContaining(["nvidia@192.168.55.1", "--", jobId]), - ); - const encodedEvidence = processRunner.mock.calls[1]![0].args.at(-1); - expect(JSON.parse(Buffer.from(encodedEvidence!, "base64").toString("utf8"))).toEqual({ - schemaVersion: 1, - volumes: ["sandbox-volume"], - processIds: [1234], - }); - expect(fs.existsSync(baselinePath)).toBe(true); - expect( - JSON.parse(fs.readFileSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`), "utf8")), - ).toEqual({ schemaVersion: 1, volumes: ["sandbox-volume"], processIds: [1234] }); - expect(fs.statSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`)).mode & 0o777).toBe( - 0o600, - ); - }); - - it("verifies cleanup after a pre-candidate failure without a baseline record (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect( - worker.cleanup({ jobId, signal: new AbortController().signal }), - ).resolves.toBeUndefined(); - - expect(processRunner).toHaveBeenCalledTimes(2); - expect(fs.existsSync(path.join(files.stateDirectory, `${jobId}.baseline.json`))).toBe(false); - }); - - it.each([ - ["Node.js", { nodeVersion: "v22.20.0" }], - ["npm", { npmVersion: "11.0.0" }], - ["Ollama executable", { ollamaPath: "/tmp/ollama" }], - ])("rejects cleanup when the protected %s baseline changes (#8142)", async (_name, change) => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); - fs.writeFileSync(baselinePath, `${JSON.stringify(BASELINE)}\n`, { mode: 0o600 }); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: baselineOutput(change), stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - "Jetson protected tool baseline differs after cleanup", - ); - expect(fs.existsSync(baselinePath)).toBe(true); - }); - - it.each([ - { - name: "an initially empty model set followed by the onboarding model", - initial: [], - final: [{ name: "qwen3.5:9b", id: "b".repeat(12) }], - }, - { - name: "a pre-existing model plus a newly installed model", - initial: [{ name: "existing:latest", id: "a".repeat(12) }], - final: [ - { name: "existing:latest", id: "a".repeat(12) }, - { name: "qwen3.5:9b", id: "b".repeat(12) }, - ], - }, - ])("allows $name after cleanup (#8142)", async ({ initial, final }) => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - fs.writeFileSync( - path.join(files.stateDirectory, `${jobId}.baseline.json`), - `${JSON.stringify({ ...BASELINE, ollamaModels: initial })}\n`, - { mode: 0o600 }, - ); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: final }), stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect( - worker.cleanup({ jobId, signal: new AbortController().signal }), - ).resolves.toBeUndefined(); - }); - - it("keeps a maximum accepted Ollama baseline readable after persistence (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const models = Array.from({ length: 64 }, (_value, index) => ({ - name: `model-${index.toString().padStart(2, "0")}-${"x".repeat(22)}`, - id: index.toString(16).padStart(12, "0"), - })); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: models }), stderr: "" }) - .mockResolvedValueOnce({ - stdout: - "model\tNVIDIA Jetson AGX Thor Developer Kit\njetpackVersion\t7.2.2\njetsonLinuxRelease\tR38\nkernel\t6.8.12-tegra\n", - stderr: "", - }) - .mockResolvedValueOnce({ stdout: "test passed\n", stderr: "" }) - .mockResolvedValueOnce({ stdout: Buffer.from("archive").toString("base64"), stderr: "" }) - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: models }), stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await worker.run( - { - schemaVersion: 1, - target: "jetson-nvmap-gpu", - candidateSha: "a".repeat(40), - workflowRunId: "123", - workflowRunAttempt: 1, - }, - { jobId, signal: new AbortController().signal }, - ); - const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); - expect(fs.statSync(baselinePath).size).toBeGreaterThan(4 * 1024); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - - await expect( - worker.cleanup({ jobId, signal: new AbortController().signal }), - ).resolves.toBeUndefined(); - }); - - it.each([ - { - name: "a pre-existing model is removed", - final: [], - }, - { - name: "a pre-existing model ID changes", - final: [{ name: "existing:latest", id: "c".repeat(12) }], - }, - ])("fails cleanup when $name (#8142)", async ({ final }) => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const baselinePath = path.join(files.stateDirectory, `${jobId}.baseline.json`); - fs.writeFileSync(baselinePath, `${JSON.stringify(BASELINE)}\n`, { mode: 0o600 }); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: baselineOutput({ ollamaModels: final }), stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - "Jetson cleanup did not preserve every pre-existing Ollama model", - ); - expect(fs.existsSync(baselinePath)).toBe(true); - }); - - it.each([ - { - name: "malformed model data", - encoded: Buffer.from("qwen3.5:9b\tnot-an-id").toString("base64"), - }, - { - name: "duplicate model names", - encoded: Buffer.from(`qwen3.5:9b\t${"a".repeat(12)}\nqwen3.5:9b\t${"b".repeat(12)}`).toString( - "base64", - ), - }, - { - name: "oversized model data", - encoded: Buffer.from( - Array.from( - { length: 65 }, - (_value, index) => `model-${index}:latest\t${index.toString(16).padStart(12, "0")}`, - ).join("\n"), - ).toString("base64"), - }, - ])("rejects $name in the protected baseline (#8142)", async ({ encoded }) => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const malformedOutput = baselineOutput().replace( - /^ollamaModelsBase64\t.*$/mu, - `ollamaModelsBase64\t${encoded}`, - ); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: malformedOutput, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - /Jetson protected-baseline Ollama model data/u, - ); - }); - - it("rejects cleanup unless OpenShell returns to the absent baseline (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - fs.writeFileSync( - path.join(files.stateDirectory, `${jobId}.baseline.json`), - `${JSON.stringify(BASELINE)}\n`, - { mode: 0o600 }, - ); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const processRunner = vi - .fn() - .mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }) - .mockResolvedValueOnce({ stdout: baselineOutput({ openshellState: "present" }), stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - "Jetson protected-baseline OpenShell state is malformed", - ); - }); - - it("retains every validated cleanup identity across stale-lock recovery (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - fs.writeFileSync( - path.join(files.stateDirectory, `${jobId}.baseline.json`), - `${JSON.stringify(BASELINE)}\n`, - { mode: 0o600 }, - ); - const processRunner = vi - .fn() - .mockImplementationOnce(async () => { - persistCleanupEvidence(files, jobId, { - schemaVersion: 1, - volumes: ["sandbox-volume"], - processIds: [1234], - }); - return { - stdout: cleanupOutput({ volumes: ["sandbox-volume"], processIds: [1234] }), - stderr: "", - }; - }) - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }) - .mockImplementationOnce(async () => { - persistCleanupEvidence(files, jobId, { - schemaVersion: 1, - volumes: ["gateway-volume", "sandbox-volume"], - processIds: [1234, 5678], - }); - return { - stdout: cleanupOutput({ - volumes: ["gateway-volume", "sandbox-volume"], - processIds: [1234, 5678], - }), - stderr: "", - }; - }) - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await worker.cleanup({ jobId, signal: new AbortController().signal }); - await worker.cleanup({ jobId, signal: new AbortController().signal }); - - expect( - JSON.parse(fs.readFileSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`), "utf8")), - ).toEqual({ - schemaVersion: 1, - volumes: ["gateway-volume", "sandbox-volume"], - processIds: [1234, 5678], - }); - }); - - it("retains a failed volume identity for startup cleanup recovery (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const cleanupEvidencePath = path.join(files.stateDirectory, `${jobId}.cleanup.json`); - fs.writeFileSync( - path.join(files.stateDirectory, `${jobId}.baseline.json`), - `${JSON.stringify(BASELINE)}\n`, - { mode: 0o600 }, - ); - const processRunner = vi - .fn() - .mockImplementationOnce(async () => { - persistCleanupEvidence(files, jobId, { - schemaVersion: 1, - volumes: ["sandbox-volume"], - processIds: [], - }); - throw new ProcessFailure("cleanup interrupted after container removal", { - stdout: "container removed\n", - stderr: "service interrupted\n", - }); - }) - .mockResolvedValueOnce({ - stdout: cleanupOutput({ volumes: ["sandbox-volume"] }), - stderr: "", - }) - .mockResolvedValueOnce({ stdout: BASELINE_OUTPUT, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - "cleanup interrupted after container removal", - ); - expect(JSON.parse(fs.readFileSync(cleanupEvidencePath, "utf8"))).toEqual({ - schemaVersion: 1, - volumes: ["sandbox-volume"], - processIds: [], - }); - - const restartedWorker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - await expect( - restartedWorker.cleanup({ jobId, signal: new AbortController().signal }), - ).resolves.toBeUndefined(); - const encodedEvidence = processRunner.mock.calls[2]![0].args.at(-1); - expect(JSON.parse(Buffer.from(encodedEvidence!, "base64").toString("utf8"))).toEqual({ - schemaVersion: 1, - volumes: ["sandbox-volume"], - processIds: [], - }); - }); - - it.each([ - "", - "nemoclaw-cleanup-evidence-v1-begin\nvolume\t../unsafe\nnemoclaw-cleanup-evidence-v1-end\n", - "nemoclaw-cleanup-evidence-v1-begin\nprocessId\t0\nnemoclaw-cleanup-evidence-v1-end\n", - ])("rejects missing or unsafe cleanup identity evidence (#8142)", async (stdout) => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - const processRunner = vi.fn().mockResolvedValueOnce({ stdout, stderr: "" }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - /Jetson cleanup/u, - ); - expect(processRunner).toHaveBeenCalledTimes(1); - expect(fs.existsSync(path.join(files.stateDirectory, `${jobId}.cleanup.json`))).toBe(false); - }); - - it("rejects cleanup output that differs from the durable identity record (#8142)", async () => { - const files = deploymentFiles(); - const jobId = "b".repeat(64); - persistCleanupEvidence(files, jobId, { schemaVersion: 1, volumes: [], processIds: [] }); - const processRunner = vi.fn().mockResolvedValueOnce({ - stdout: cleanupOutput({ volumes: ["unpersisted-volume"] }), - stderr: "", - }); - const worker = new SshJetsonDispatchWorker(loadConfig(files), processRunner); - - await expect(worker.cleanup({ jobId, signal: new AbortController().signal })).rejects.toThrow( - "Jetson cleanup output differs from its durable resource evidence", - ); - expect(processRunner).toHaveBeenCalledTimes(1); - }); - - it("reports a missing durable cleanup record after valid helper output (#8142)", async () => { - const files = deploymentFiles(); - const worker = new SshJetsonDispatchWorker( - loadConfig(files), - vi.fn().mockResolvedValueOnce({ stdout: cleanupOutput(), stderr: "" }), - ); - - await expect( - worker.cleanup({ jobId: "b".repeat(64), signal: new AbortController().signal }), - ).rejects.toThrow("Jetson cleanup evidence is missing"); - }); - - it("ignores permission denial while inspecting an unrelated process (#8142)", () => { - const fixture = cleanupProcessFixture({ - deny: "environ", - uid: process.getuid!() + 1, - }); - const result = spawnSync( - "/bin/bash", - [ - "-c", - `set -euo pipefail -${fixture.functions} -job_home=/job/home -for proc_dir in "$PROC_ROOT"/*; do - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" owner - continue - fi - [ "$process_uid" = "$(id -u)" ] || continue - if ! environment="$(read_proc_environment "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" environment - continue - fi - printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue - if ! cmdline="$(read_proc_command "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" command - continue - fi -done -printf 'inspection-complete\n' -`, - ], - { encoding: "utf8", env: fixture.environment }, - ); - - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toBe("inspection-complete\n"); - }); - - it("rejects an unreadable live same-user process during discovery (#8142)", () => { - const fixture = cleanupProcessFixture({ deny: "environ" }); - const result = spawnSync( - "/bin/bash", - [ - "-c", - `set -euo pipefail -${fixture.functions} -job_home=/job/home -for proc_dir in "$PROC_ROOT"/*; do - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" owner - continue - fi - [ "$process_uid" = "$(id -u)" ] || continue - if ! environment="$(read_proc_environment "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" environment - continue - fi -done -`, - ], - { encoding: "utf8", env: fixture.environment }, - ); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("Unable to inspect environment for a live same-user process"); - }); - - it("ignores a recorded process that disappears during inspection (#8142)", () => { - const fixture = cleanupProcessFixture({ disappear: "cmdline" }); - const result = spawnSync( - "/bin/bash", - [ - "-c", - `set -euo pipefail -${fixture.functions} -kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } -job_home=/job/home -validate_recorded_pid 1234 ollama-auth-proxy. -`, - ], - { encoding: "utf8", env: fixture.environment }, - ); - - expect(result.status, result.stderr).toBe(0); - }); - - it("completes the destructive cleanup phase when a process disappears during inspection (#8142)", () => { - const fixture = cleanupProcessFixture({ disappear: "environ" }); - fs.writeFileSync(path.join(fixture.binDirectory, "docker"), "#!/bin/sh\nexit 0\n", { - mode: 0o700, - }); - const marker = "<<'JETSON_CLEANUP'\n"; - const phaseStart = cleanupProgram.indexOf(marker); - const phaseEnd = cleanupProgram.indexOf("\nJETSON_CLEANUP", phaseStart + marker.length); - assert.ok(phaseStart >= 0 && phaseEnd >= 0, "destructive cleanup phase is absent"); - const script = cleanupProgram - .slice(phaseStart + marker.length, phaseEnd) - .replace("set -euo pipefail", "set -eo pipefail") - .replace( - "workspace_root=/var/tmp/nemoclaw-jetson-e2e", - 'workspace_root="$TEST_WORKSPACE_ROOT"', - ) - .replace( - 'service_directory="/tmp/nemoclaw-services-$sandbox_name"', - 'service_directory="$TEST_SERVICE_DIRECTORY"', - ) - .replace("for proc_dir in /proc/[0-9]*; do", 'for proc_dir in "$PROC_ROOT"/*; do') - .replace( - 'remaining_job_images="$(list_test_owned_images)"', - "printf 'destructive-phase-complete\\n'\nexit 0\nremaining_job_images=\"$(list_test_owned_images)\"", - ); - const evidence = Buffer.from( - JSON.stringify({ schemaVersion: 1, volumes: [], processIds: [] }), - ).toString("base64"); - const result = spawnSync("/bin/bash", ["-s", "--", "b".repeat(64), evidence], { - encoding: "utf8", - env: { - ...fixture.environment, - TEST_SERVICE_DIRECTORY: path.join(fixture.procRoot, "service"), - TEST_WORKSPACE_ROOT: path.join(fixture.procRoot, "workspace"), - }, - input: script, - }); - - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toBe("destructive-phase-complete\n"); - }); - - it("rejects a live recorded PID whose owner cannot be verified (#8142)", () => { - const fixture = cleanupProcessFixture(); - fs.rmSync(path.join(fixture.procRoot, "1234", "status")); - const result = spawnSync( - "/bin/bash", - [ - "-c", - `set -euo pipefail -${fixture.functions} -kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } -job_home=/job/home -validate_recorded_pid 1234 ollama-auth-proxy. -`, - ], - { encoding: "utf8", env: fixture.environment }, - ); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("owner cannot be verified"); - }); - - it.each([ - { - name: "owner", - options: { uid: process.getuid!() + 1 }, - error: "owned by another user", - }, - { - name: "HOME", - options: { home: "/unrelated/home" }, - error: "outside the job home", - }, - { - name: "command", - options: { command: "node unrelated-service.mjs" }, - error: "unexpected command", - }, - ])("rejects a recorded PID with an unexpected $name (#8142)", ({ error, options }) => { - const fixture = cleanupProcessFixture(options); - const result = spawnSync( - "/bin/bash", - [ - "-c", - `set -euo pipefail -${fixture.functions} -kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } -job_home=/job/home -validate_recorded_pid 1234 ollama-auth-proxy. -`, - ], - { encoding: "utf8", env: fixture.environment }, - ); - - expect(result.status).toBe(1); - expect(result.stderr).toContain(error); - }); - - it("accepts a recorded PID with the expected owner, HOME, and command (#8142)", () => { - const fixture = cleanupProcessFixture(); - const result = spawnSync( - "/bin/bash", - [ - "-c", - `set -euo pipefail -${fixture.functions} -kill() { [ "$1" = -0 ] && [ -d "$PROC_ROOT/$2" ]; } -job_home=/job/home -validate_recorded_pid 1234 ollama-auth-proxy. -`, - ], - { encoding: "utf8", env: fixture.environment }, - ); - - expect(result.status, result.stderr).toBe(0); - }); - - it("keeps cleanup targets fixed and preserves the Jetson baseline (#8142)", () => { - const fileFsync = cleanupProgram.indexOf("fs.fsyncSync(descriptor)"); - const atomicRename = cleanupProgram.indexOf("fs.renameSync(temporaryFile, cleanupFile)"); - const directoryFsync = cleanupProgram.indexOf("fs.fsyncSync(directoryDescriptor)"); - const destructivePhase = cleanupProgram.indexOf("<<'JETSON_CLEANUP'"); - const stopFunctionStart = cleanupProgram.lastIndexOf("stop_recorded_pid()"); - const stopFunction = cleanupProgram.slice( - stopFunctionStart, - cleanupProgram.indexOf('for pid in "${recorded_process_ids[@]}"', stopFunctionStart), - ); - expect(cleanupProgram).toContain("sandbox_name=e2e-jetson-nvmap"); - expect(cleanupProgram).toContain("gateway_container=openshell-cluster-nemoclaw"); - expect(cleanupProgram).toContain("destination=nvidia@192.168.55.1"); - expect(cleanupProgram).toContain('if [ "$#" -ne 0 ]'); - expect(cleanupProgram).toContain('mapfile -t lock_lines <"$lock_file"'); - expect(cleanupProgram).toContain('[ "${#lock_lines[@]}" -ne 1 ]'); - expect(cleanupProgram).toContain('require_plain_directory_if_present "$service_directory"'); - expect(cleanupProgram).toContain('rm -rf -- "$workspace"'); - expect(cleanupProgram).toContain("ollama-auth-proxy.pid"); - expect(cleanupProgram).toContain("read_proc_uid()"); - expect(cleanupProgram).toContain('dd if="$1/environ"'); - expect(cleanupProgram).toContain('dd if="$1/cmdline"'); - expect( - cleanupProgram.match(/handle_proc_read_failure "\$proc_dir" environment/gu), - ).toHaveLength(2); - expect(cleanupProgram.match(/handle_proc_read_failure "\$proc_dir" command/gu)).toHaveLength(2); - expect(workerProgram).toContain('handle_proc_read_failure "$proc_dir" environment'); - expect(workerProgram).toContain('handle_proc_read_failure "$proc_dir" command'); - expect(cleanupProgram).toContain('grep -Fqx "HOME=$job_home"'); - expect(cleanupProgram).toContain("*openshell-forward*"); - expect(cleanupProgram).toContain("*openshell\\ forward*"); - expect(cleanupProgram).toContain('kill "$pid"'); - expect(stopFunction.indexOf('process_uid="$(read_proc_uid')).toBeLessThan( - stopFunction.indexOf('kill "$pid"'), - ); - expect(stopFunction.indexOf('cmdline="$(read_proc_command')).toBeLessThan( - stopFunction.indexOf('kill "$pid"'), - ); - expect(stopFunction.indexOf('grep -Fqx "HOME=$job_home"')).toBeLessThan( - stopFunction.indexOf('kill "$pid"'), - ); - expect(cleanupProgram).toContain("label=openshell.ai/sandbox-name=$sandbox_name"); - expect(cleanupProgram).toContain("image_repository=nemoclaw-sandbox-local"); - expect(cleanupProgram).toContain('docker image rm "$image"'); - expect(cleanupProgram.indexOf('docker image rm "$image"')).toBeLessThan( - cleanupProgram.indexOf('rm -rf -- "$workspace"'), - ); - expect(cleanupProgram).not.toContain("job_openshell"); - expect(cleanupProgram).not.toContain('"$job_home/.local/bin/openshell"'); - expect(cleanupProgram).toContain("docker container ls --all --no-trunc"); - expect(cleanupProgram).toContain("docker volume ls --format '{{.Name}}'"); - expect(cleanupProgram).not.toMatch(/if docker (?:container|volume) inspect/u); - expect(cleanupProgram).not.toContain("mapfile -t sandbox_containers < <("); - expect(cleanupProgram).toContain("A host-level OpenShell binary remains after cleanup"); - expect(cleanupProgram).toContain("nemoclaw-cleanup-evidence-v1-begin"); - expect(cleanupProgram).toContain("nemoclaw-cleanup-evidence-v1-end"); - expect(cleanupProgram.indexOf("nemoclaw-cleanup-evidence-v1-begin")).toBeLessThan( - cleanupProgram.indexOf('docker container rm --force "$container"'), - ); - expect(cleanupProgram.indexOf("nemoclaw-cleanup-evidence-v1-end")).toBeLessThan( - cleanupProgram.indexOf('docker volume rm "$volume"'), - ); - expect(cleanupProgram).toContain("fs.constants.O_EXCL"); - expect(fileFsync).toBeGreaterThan(cleanupProgram.indexOf("<<'JETSON_DISCOVERY'")); - expect(atomicRename).toBeGreaterThan(fileFsync); - expect(directoryFsync).toBeGreaterThan(atomicRename); - expect(destructivePhase).toBeGreaterThan(directoryFsync); - expect(cleanupProgram).not.toMatch(/pkill|pgrep|docker (?:system|container|volume) prune/u); - expect(cleanupProgram).not.toContain('rm -rf -- "$workspace_root"'); - expect(cleanupProgram).not.toContain("ollama serve"); - expect(cleanupProgram).not.toMatch(/ollama (?:rm|delete)/u); - expect(cleanupProgram).not.toMatch(/apt(?:-get)?|boardctl|reboot|shutdown|nvidia-l4t/u); - expect(cleanupProgram).not.toMatch(/npm uninstall|uninstall\.sh|rm .*ollama/u); - }); - - it("keeps OpenShell installation owned by NemoClaw onboarding inside the job workspace (#8142)", () => { - expect(workerProgram).not.toContain("NEMOCLAW_DEFER_OPENSHELL_INSTALL"); - expect(jetsonLiveTest).not.toContain("NEMOCLAW_DEFER_OPENSHELL_INSTALL"); - expect(workerProgram).not.toContain("scripts/install-openshell.sh"); - expect(cleanupProgram).not.toContain("scripts/install-openshell.sh"); - expect(jetsonLiveTest).not.toContain("scripts/install-openshell.sh"); - expect(jetsonLiveTest).toContain('host.command("bash", ["install.sh", "--non-interactive"]'); - expect(workerProgram).toContain('export XDG_BIN_HOME="$workspace/home/.local/bin"'); - expect(workerProgram).toContain('export PATH="$XDG_BIN_HOME:$workspace/npm-prefix/bin:$PATH"'); - expect(workerProgram).toContain( - "for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox", - ); - expect(jetsonLiveTest).toContain( - "for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox", - ); - expect(cleanupProgram.indexOf('stop_recorded_pid "$pid"')).toBeLessThan( - cleanupProgram.indexOf('docker container rm --force "$container"'), - ); - expect(cleanupProgram.indexOf('docker container rm --force "$container"')).toBeLessThan( - cleanupProgram.indexOf('rm -rf -- "$workspace"'), - ); - }); - - it("requires every protected Jetson prerequisite before dispatch (#8142)", () => { - expect(dispatcherRunbook).toContain("OpenShell must be absent"); - expect(dispatcherRunbook).toContain( - "if (major < 22 || (major === 22 && minor < 19)) process.exit(1)", - ); - expect(dispatcherRunbook).toContain('test "${npm_version%%.*}" -ge 10'); - expect(dispatcherRunbook).toContain("ollama list"); - }); - - it("uses a private temporary file for the pinned Jetson host key (#8142)", () => { - const hostKeySection = dispatcherRunbook.slice( - dispatcherRunbook.indexOf("Capture the Jetson host key"), - dispatcherRunbook.indexOf("## Define the Cleanup Program"), - ); - const temporary = hostKeySection.indexOf( - 'known_hosts_tmp="$(mktemp /tmp/nemoclaw-jetson-known-hosts.XXXXXX)"', - ); - const cleanupTrap = hostKeySection.indexOf("trap 'rm -f -- \"$known_hosts_tmp\"' EXIT"); - const keyScan = hostKeySection.indexOf("ssh-keyscan -T 10"); - const install = hostKeySection.indexOf("sudo install"); - - expect(hostKeySection).toContain("umask 077"); - expect(temporary).toBeGreaterThan(-1); - expect(cleanupTrap).toBeGreaterThan(temporary); - expect(keyScan).toBeGreaterThan(cleanupTrap); - expect(install).toBeGreaterThan(keyScan); - expect(hostKeySection).not.toContain("/tmp/jetson_known_hosts"); - }); - - it("bounds every Jetson teardown SSH probe and the Ollama model query (#8142)", () => { - const teardown = dispatcherRunbook.slice( - dispatcherRunbook.indexOf("## Recover or Disable the Deployment"), - ); - const sshProbes = teardown.match(/sudo -u nemoclaw-jetson-dispatch ssh/gmu) ?? []; - const boundedSshProbes = - teardown.match( - /timeout --kill-after=5 120 \\\n\s+sudo -u nemoclaw-jetson-dispatch ssh -F \/dev\/null -T \\\n\s+-o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \\\n\s+-o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=3/gmu, - ) ?? []; - - expect(sshProbes).toHaveLength(4); - expect(boundedSshProbes).toHaveLength(sshProbes.length); - expect(teardown).toContain("timeout --kill-after=5 30 ollama list >/dev/null"); - }); - - it("stops credential teardown when the Jetson key-removal probe fails (#8142)", () => { - const credentialTeardown = dispatcherRunbook.slice( - dispatcherRunbook.indexOf("Remove the dedicated Jetson public key"), - dispatcherRunbook.indexOf("Remove the cleanup executable and pinned SSH host-key file"), - ); - const strictMode = credentialTeardown.indexOf("set -euo pipefail"); - const keyRemovalProbe = credentialTeardown.indexOf("timeout --kill-after=5 120"); - const privateKeyRemoval = credentialTeardown.indexOf( - "sudo rm -- \\\n /var/lib/nemoclaw-jetson-dispatch/id_ed25519", - ); - - expect(strictMode).toBeGreaterThan(-1); - expect(keyRemovalProbe).toBeGreaterThan(strictMode); - expect(privateKeyRemoval).toBeGreaterThan(keyRemovalProbe); - expect(credentialTeardown.slice(keyRemovalProbe, privateKeyRemoval)).not.toContain("|| true"); - expect(dispatcherRunbook).toContain( - "Use the serial console to confirm that the exact key line is absent", - ); - }); - - it("keeps recovery credentials until idle teardown verification succeeds (#8142)", () => { - const stopTunnel = dispatcherRunbook.indexOf( - "sudo systemctl disable --now nemoclaw-jetson-tunnel.service", - ); - const waitForLock = dispatcherRunbook.indexOf( - "timeout 3600 bash -c 'while [ -e \"$1\" ]; do sleep 5; done'", - ); - const verifyCleanup = dispatcherRunbook.indexOf( - 'if (status.state !== "completed" || status.cleanup !== "succeeded") process.exit(1)', - ); - const aggregateCleanupRecords = dispatcherRunbook.indexOf( - "const cleanupName = /^[a-f0-9]{64}\\.cleanup\\.json$/", - ); - const verifyRecordedVolume = dispatcherRunbook.indexOf("docker volume ls --format '{{.Name}}'"); - const verifyRecordedProcess = dispatcherRunbook.indexOf('if [ -e "/proc/$1" ]'); - const stopDispatcher = dispatcherRunbook.indexOf( - "sudo systemctl disable --now nemoclaw-jetson-dispatch.service", - ); - const removeSshKey = dispatcherRunbook.indexOf( - "Remove the dedicated Jetson public key and the Colossus SSH private key.", - ); - - expect(dispatcherServiceUnit).toContain("TimeoutStopSec=360"); - expect(stopTunnel).toBeGreaterThan(-1); - expect(waitForLock).toBeGreaterThan(stopTunnel); - expect(verifyCleanup).toBeGreaterThan(waitForLock); - expect(dispatcherRunbook).toContain( - 'const expectedKeys = ["processIds", "schemaVersion", "volumes"]', - ); - expect(dispatcherRunbook).toContain("record.schemaVersion !== 1"); - expect(aggregateCleanupRecords).toBeGreaterThan(verifyCleanup); - expect(verifyRecordedVolume).toBeGreaterThan(aggregateCleanupRecords); - expect(verifyRecordedProcess).toBeGreaterThan(verifyRecordedVolume); - expect(stopDispatcher).toBeGreaterThan(verifyRecordedProcess); - expect(removeSshKey).toBeGreaterThan(stopDispatcher); - }); -}); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 2e3a8a9476c..e319e559017 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -17,7 +17,7 @@ function validateWorkflowMutation( } describe("Jetson nvmap GPU E2E workflow boundary", () => { - it("rejects a permissive Colossus dispatch opt-in (#8142)", () => { + it("rejects a permissive Jetson dispatch opt-in (#8142)", () => { const inputErrors = validateWorkflowMutation((workflow) => { const triggers = (workflow.on ?? workflow[true as unknown as string]) as { workflow_dispatch?: { @@ -33,7 +33,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect.arrayContaining([ "workflow_dispatch allow_jetson_dispatch input must be boolean", "workflow_dispatch allow_jetson_dispatch input must default to false", - "workflow_dispatch allow_jetson_dispatch input must require the Colossus dispatcher, tunnel, cleanup helper, repository URL variable, and deployment checks", + "workflow_dispatch allow_jetson_dispatch input must require the operator-owned backend, repository URL variable, and controller documentation", ]), ); }); @@ -51,7 +51,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { ); }); - it("queues every Jetson dispatch on one fixed device without cancellation (#8142)", () => { + it("queues every operator-backend dispatch without cancellation (#8142)", () => { const workflow = readWorkflow(); const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { concurrency?: Record; @@ -62,7 +62,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { "cancel-in-progress": true, }; expect(validateJetsonDispatchBoundary(workflow)).toContain( - "jetson-nvmap-gpu concurrency must queue every dispatch on one fixed device without cancellation", + "jetson-nvmap-gpu concurrency must queue every operator-backend dispatch without cancellation", ); }); @@ -78,7 +78,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { )!; checkout.with!.ref = "${{ inputs.checkout_sha }}"; const dispatch = job.steps!.find( - (step) => step.name === "Dispatch exact commit to Jetson through Colossus", + (step) => step.name === "Dispatch exact commit to Jetson through operator backend", )!; dispatch.run = "bash candidate-script.sh"; }); @@ -118,7 +118,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { E2E_ARTIFACT_DIR: "${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu", }; const dispatch = job.steps!.find( - (step) => step.name === "Dispatch exact commit to Jetson through Colossus", + (step) => step.name === "Dispatch exact commit to Jetson through operator backend", )!; dispatch.env!.E2E_ARTIFACT_DIR = "${{ github.workspace }}/e2e-artifacts"; }); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 592ab5f4d4c..b5b338f59d7 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -93,20 +93,8 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ testsToRun: runTests("test/setup-jetson.test.ts"), }, { - pattern: /(?:^|\/)tools\/e2e\/jetson-dispatch-cleanup\.sh$/, - testsToRun: runTests("test/e2e/support/jetson-dispatch-worker.test.ts"), - }, - { - pattern: /(?:^|\/)tools\/e2e\/colossus-jetson-dispatch-deploy\.sh$/, - testsToRun: runTests("test/e2e/support/colossus-jetson-dispatch-deploy.test.ts"), - }, - { - pattern: - /(?:^|\/)tools\/e2e\/(?:colossus-jetson-dispatch\.environment|nemoclaw-jetson-dispatch\.service)$/, - testsToRun: runTests( - "test/e2e/support/colossus-jetson-dispatch-deploy.test.ts", - "test/e2e/support/jetson-dispatch-worker.test.ts", - ), + pattern: /(?:^|\/)tools\/e2e\/contracts\/v1\/jetson-dispatch\.json$/, + testsToRun: runTests("test/e2e/support/jetson-dispatch-client.test.ts"), }, { pattern: diff --git a/test/maintainer-e2e-skill.test.ts b/test/maintainer-e2e-skill.test.ts index a4ae9584015..2a8a60c596a 100644 --- a/test/maintainer-e2e-skill.test.ts +++ b/test/maintainer-e2e-skill.test.ts @@ -222,7 +222,7 @@ describe("nemoclaw-maintainer-e2e evidence validation", () => { "dispatch.jobs", ], [ - "a full dispatch that opts into Colossus Jetson execution", + "a full dispatch that opts into operator-owned Jetson execution", (evidence: ReturnType) => { evidence.dispatch.allowJetsonDispatch = true; }, diff --git a/test/nemoclaw-start-gateway-health.test.ts b/test/nemoclaw-start-gateway-health.test.ts index 13ac5899a73..bfdca90abda 100644 --- a/test/nemoclaw-start-gateway-health.test.ts +++ b/test/nemoclaw-start-gateway-health.test.ts @@ -32,6 +32,10 @@ function rootGatewayLifecycleFunctions(src: string, gatewayLog: string): string return [ pidIdentityFunctions(src), extractShellFunction(src, "arm_openclaw_gateway_supervisor_cleanup"), + extractShellFunction(src, "launch_openclaw_gateway_process").replaceAll( + "/tmp/gateway.log", + gatewayLog, + ), extractShellFunction(src, "launch_openclaw_gateway").replaceAll("/tmp/gateway.log", gatewayLog), extractShellFunction(src, "openclaw_supervised_aux_pid_is_live"), extractShellFunction(src, "stop_openclaw_supervised_gateway"), @@ -583,6 +587,7 @@ describe("gateway launch wiring (#4710)", () => { 'kill() { printf "unexpected-kill:%s\\n" "$*" >>"$EVENT_LOG"; }', 'wait() { printf "unexpected-wait:%s\\n" "$*" >>"$EVENT_LOG"; }', extractShellFunction(src, "arm_openclaw_gateway_supervisor_cleanup"), + extractShellFunction(src, "launch_openclaw_gateway_process"), launch, "launch_openclaw_gateway", ].join("\n"), diff --git a/test/nemoclaw-start-gateway-marker.test.ts b/test/nemoclaw-start-gateway-marker.test.ts index e60d253905d..0de801d8629 100644 --- a/test/nemoclaw-start-gateway-marker.test.ts +++ b/test/nemoclaw-start-gateway-marker.test.ts @@ -331,6 +331,7 @@ describe("nemoclaw-start in-container gateway healthcheck marker (#4503, #4710)" markerPath, ), extractShellFunctionFromSource(src, "arm_openclaw_gateway_supervisor_cleanup"), + extractShellFunctionFromSource(src, "launch_openclaw_gateway_process"), extractShellFunctionFromSource(src, launchFunction), "cleanup_openclaw_on_signal() { exit 143; }", "STEP_DOWN_PREFIX_GATEWAY=(env)", diff --git a/test/openclaw-2026-7-startup-compat.test.ts b/test/openclaw-2026-7-startup-compat.test.ts index 7310d07400c..c06dec6570d 100644 --- a/test/openclaw-2026-7-startup-compat.test.ts +++ b/test/openclaw-2026-7-startup-compat.test.ts @@ -171,6 +171,10 @@ describe("OpenClaw 2026.7 startup compatibility", () => { "mark_in_container_gateway() { :; }", "capture_openclaw_pid_start_identity() { printf -v \"$2\" '%s' test-identity; }", "record_gateway_pid() { :; }", + extractShellFunction(source, "launch_openclaw_gateway_process").replaceAll( + "/tmp/gateway.log", + gatewayLog, + ), launch, "launch_openclaw_gateway", 'wait "$GATEWAY_PID"', diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index f6a3da0ab85..6a09d9d808b 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -61,8 +61,7 @@ const OPAQUE_INPUTS = [ "agents/hermes/mcp-config-transaction.py", "test/e2e/lib/ci-compatible-inference.sh", "scripts/setup-jetson.sh", - "tools/e2e/jetson-dispatch-cleanup.sh", - "tools/e2e/colossus-jetson-dispatch-deploy.sh", + "tools/e2e/contracts/v1/jetson-dispatch.json", ".github/workflows/base-image.yaml", "scripts/export-managed-base-image-contract.sh", "scripts/checks/validate-managed-base-index.sh", @@ -132,11 +131,8 @@ describe("Vitest opaque-input watch triggers", () => { "test/e2e/support/hosted-inference.test.ts", ]); expect(triggeredBy("scripts/setup-jetson.sh")).toEqual(["test/setup-jetson.test.ts"]); - expect(triggeredBy("tools/e2e/jetson-dispatch-cleanup.sh")).toEqual([ - "test/e2e/support/jetson-dispatch-worker.test.ts", - ]); - expect(triggeredBy("tools/e2e/colossus-jetson-dispatch-deploy.sh")).toEqual([ - "test/e2e/support/colossus-jetson-dispatch-deploy.test.ts", + expect(triggeredBy("tools/e2e/contracts/v1/jetson-dispatch.json")).toEqual([ + "test/e2e/support/jetson-dispatch-client.test.ts", ]); expect(triggeredBy(".github/workflows/base-image.yaml")).toEqual([ "test/managed-base-image-contract.test.ts", diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 4680dc6501a..e4c2b1f9ede 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -34,9 +34,7 @@ export const PR_E2E_MANUAL_CONTROLLER_JOB_IDS = [ "inference-routing", "managed-image-protected-runtime", ] as const; -const PR_E2E_MANUAL_CONTROLLER_JOB_ID_SET = new Set( - PR_E2E_MANUAL_CONTROLLER_JOB_IDS, -); +const PR_E2E_MANUAL_CONTROLLER_JOB_ID_SET = new Set(PR_E2E_MANUAL_CONTROLLER_JOB_IDS); const DEEPAGENTS_HEADLESS_INFERENCE_CHECK = "test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh"; const DEEPAGENTS_CODE_RUNTIME_ROOT = "agents/langchain-deepagents-code/"; @@ -244,7 +242,7 @@ export function isPrE2eTypedTargetId(value: string): boolean { } export function isPrE2ePlanningJob(value: string): boolean { - // Automatic PR planning cannot attest the external Colossus dispatcher and cleanup path. + // Automatic PR planning cannot attest the operator-owned Jetson backend and hardware path. // Remove this exclusion after that hardware gate produces trusted planning evidence. return !PR_E2E_PLANNING_OMITTED_JOB_IDS.has(value); } diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index 3ba5eed8ca1..5d146aec86e 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -39,7 +39,7 @@ const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 = "3578a053cede863f7aa4814d8399b4ca21ea0b77cee712e6d549c684818f11dd"; const CLI_ARTIFACT_WORKFLOW_CONTRACT_SHA256 = - "6b9110770f2a6f5370910054d40eee6935161eb8fc5ee4ed3afec808cf536e89"; + "715e590815e3b0e3e1511d19dc548c0a08c28d3cb9255e872c438eeed9028a7f"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { diff --git a/tools/e2e/colossus-jetson-dispatch-deploy.sh b/tools/e2e/colossus-jetson-dispatch-deploy.sh deleted file mode 100755 index 9adae79c93a..00000000000 --- a/tools/e2e/colossus-jetson-dispatch-deploy.sh +++ /dev/null @@ -1,711 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -export PATH -umask 022 - -fixed_origin=https://github.com/NVIDIA/NemoClaw.git -deploy_root=/opt/nemoclaw-jetson-dispatch -releases_directory="$deploy_root/releases" -current_link="$deploy_root/current" -deploy_lock="$deploy_root/deploy.lock" -cleanup_relative=tools/e2e/jetson-dispatch-cleanup.sh -environment_relative=tools/e2e/colossus-jetson-dispatch.environment -unit_relative=tools/e2e/nemoclaw-jetson-dispatch.service -cleanup_executable=/usr/local/libexec/nemoclaw-jetson-cleanup -cleanup_link_target="$current_link/$cleanup_relative" -service_user=nemoclaw-jetson-dispatch -service_group=nemoclaw-jetson-dispatch -service_home=/var/lib/nemoclaw-jetson-dispatch -state_directory="$service_home/state" -device_lock="$state_directory/device.lock" -ssh_identity_file="$service_home/id_ed25519" -ssh_known_hosts_file="$service_home/known_hosts" -node_executable=/usr/bin/node -service_name=nemoclaw-jetson-dispatch.service -environment_file=/etc/nemoclaw-jetson-dispatch/environment -unit_file="/etc/systemd/system/$service_name" -tunnel_service_name=nemoclaw-jetson-tunnel.service -service_port=8787 - -staging_directory="" -temporary_link="" -cleanup_stage="" -managed_file_stage="" -bootstrap_environment_installed=0 -bootstrap_unit_installed=0 -bootstrap_daemon_reloaded=0 -bootstrap_enable_attempted=0 - -usage() { - echo "Usage: nemoclaw-colossus-jetson-dispatch-deploy --commit " >&2 -} - -fail() { - echo "Colossus Jetson dispatcher deployment failed: $*" >&2 - exit 1 -} - -effective_uid() { - /usr/bin/id -u -} - -git_exec() { - /usr/bin/env \ - -u GIT_ALTERNATE_OBJECT_DIRECTORIES \ - -u GIT_COMMON_DIR \ - -u GIT_CONFIG_PARAMETERS \ - -u GIT_DIR \ - -u GIT_EXEC_PATH \ - -u GIT_OBJECT_DIRECTORY \ - -u GIT_PROXY_COMMAND \ - -u GIT_SSH \ - -u GIT_SSH_COMMAND \ - -u GIT_SSL_NO_VERIFY \ - -u GIT_TEMPLATE_DIR \ - -u GIT_WORK_TREE \ - GIT_CONFIG_COUNT=0 \ - GIT_CONFIG_GLOBAL=/dev/null \ - GIT_CONFIG_NOSYSTEM=1 \ - HOME=/nonexistent \ - /usr/bin/git \ - -c core.hooksPath=/dev/null \ - -c credential.helper= \ - -c http.sslVerify=true \ - -c protocol.ext.allow=never \ - -c protocol.file.allow=never \ - "$@" -} - -systemctl_exec() { - /usr/bin/systemctl "$@" -} - -ss_exec() { - /usr/bin/ss "$@" -} - -curl_exec() { - /usr/bin/curl "$@" -} - -sleep_exec() { - /usr/bin/sleep "$@" -} - -id_exec() { - /usr/bin/id "$@" -} - -ssh_keygen_exec() { - /usr/bin/ssh-keygen "$@" -} - -install_file() { - /usr/bin/install -o root -g root -m "$3" "$1" "$2" -} - -files_match() { - /usr/bin/cmp -s "$1" "$2" -} - -symlink_owner() { - /usr/bin/stat -c '%u:%g' "$1" -} - -install_directory() { - local path="$1" mode="$2" - /usr/bin/install -d -o root -g root -m "$mode" "$path" -} - -move_replace() { - /usr/bin/mv -Tf -- "$1" "$2" -} - -move_directory() { - /usr/bin/mv -- "$1" "$2" -} - -require_root_owned_directory() { - local path="$1" expected_mode="$2" metadata - if [ ! -d "$path" ] || [ -L "$path" ]; then - fail "$path must be a directory, not a symbolic link" - fi - metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" - if [ "$metadata" != "0:0:$expected_mode" ]; then - fail "$path must be owned by root:root with mode $expected_mode" - fi -} - -require_root_owned_file() { - local path="$1" expected_mode="$2" metadata - if [ ! -f "$path" ] || [ -L "$path" ]; then - fail "$path must be a regular file, not a symbolic link" - fi - metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" - if [ "$metadata" != "0:0:$expected_mode" ]; then - fail "$path must be owned by root:root with mode $expected_mode" - fi -} - -root_owned_file_matches() { - local path="$1" expected_mode="$2" metadata - [ -f "$path" ] && [ ! -L "$path" ] || return 1 - metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || return 1 - [ "$metadata" = "0:0:$expected_mode" ] -} - -require_service_owned_file() { - local path="$1" expected_mode="$2" uid gid metadata - uid="$(id_exec -u "$service_user")" || fail "could not inspect the $service_user account" - gid="$(id_exec -g "$service_user")" || fail "could not inspect the $service_group group" - if [ ! -f "$path" ] || [ -L "$path" ]; then - fail "$path must be a regular file, not a symbolic link" - fi - metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" - [ "$metadata" = "$uid:$gid:$expected_mode" ] \ - || fail "$path must be owned by $service_user:$service_group with mode $expected_mode" -} - -require_service_owned_directory() { - local path="$1" expected_mode="$2" uid gid metadata - uid="$(id_exec -u "$service_user")" || fail "could not inspect the $service_user account" - gid="$(id_exec -g "$service_user")" || fail "could not inspect the $service_group group" - if [ ! -d "$path" ] || [ -L "$path" ]; then - fail "$path must be a directory, not a symbolic link" - fi - metadata="$(/usr/bin/stat -c '%u:%g:%a' "$path")" || fail "could not inspect $path" - [ "$metadata" = "$uid:$gid:$expected_mode" ] \ - || fail "$path must be owned by $service_user:$service_group with mode $expected_mode" -} - -ssh_identity_is_valid() { - ssh_keygen_exec -y -P '' -f "$ssh_identity_file" >/dev/null 2>&1 -} - -known_hosts_has_jetson() { - ssh_keygen_exec -F 192.168.55.1 -f "$ssh_known_hosts_file" >/dev/null 2>&1 -} - -node_version_is_supported() { - "$node_executable" -e ' - const [major, minor] = process.versions.node.split(".").map(Number); - if (major < 22 || (major === 22 && minor < 19)) process.exit(1); - ' -} - -validate_dispatcher_prerequisites() { - local uid gid - uid="$(id_exec -u "$service_user")" || fail "the $service_user account is required" - gid="$(id_exec -g "$service_user")" || fail "the $service_group group is required" - [[ "$uid" =~ ^[1-9][0-9]*$ ]] || fail "$service_user must not use UID 0" - [[ "$gid" =~ ^[1-9][0-9]*$ ]] || fail "$service_group must not use GID 0" - - require_service_owned_directory "$service_home" 700 - require_service_owned_file "$ssh_identity_file" 600 - ssh_identity_is_valid \ - || fail "$ssh_identity_file is not a readable OpenSSH private key" - require_service_owned_file "$ssh_known_hosts_file" 600 - known_hosts_has_jetson \ - || fail "$ssh_known_hosts_file does not contain the pinned Jetson host key" - - [ -x "$node_executable" ] || fail "$node_executable is required" - node_version_is_supported || fail "$node_executable must be Node.js 22.19.0 or later" -} - -cleanup_temporary_paths() { - if [ -n "$staging_directory" ] && [[ "$staging_directory" == "$releases_directory"/.* ]]; then - rm -rf -- "$staging_directory" - fi - if [ -n "$temporary_link" ] && [[ "$temporary_link" == "$deploy_root"/.current.* ]]; then - rm -f -- "$temporary_link" - fi - if [ -n "$cleanup_stage" ] && [[ "$cleanup_stage" == "${cleanup_executable%/*}"/.nemoclaw-jetson-cleanup.* ]]; then - rm -f -- "$cleanup_stage" - fi - if [ -n "$managed_file_stage" ]; then - case "$managed_file_stage" in - "${environment_file%/*}"/.environment.* | "${unit_file%/*}"/.nemoclaw-jetson-dispatch.service.*) - rm -f -- "$managed_file_stage" - ;; - esac - fi -} - -trap cleanup_temporary_paths EXIT - -require_root() { - [ "$(effective_uid)" = 0 ] || fail "this command requires root" -} - -parse_commit() { - if [ "$#" -ne 2 ] || [ "$1" != "--commit" ] || [[ ! "$2" =~ ^[a-f0-9]{40}$ ]]; then - usage - exit 2 - fi - printf '%s\n' "$2" -} - -print_stage() { - printf '[%s/5] %s\n' "$1" "$2" -} - -ensure_layout() { - install_directory "$deploy_root" 0755 - install_directory "$releases_directory" 0755 - install_directory "${cleanup_executable%/*}" 0755 - install_directory "${environment_file%/*}" 0755 - require_root_owned_directory "$deploy_root" 755 - require_root_owned_directory "$releases_directory" 755 - require_root_owned_directory "${cleanup_executable%/*}" 755 - require_root_owned_directory "${environment_file%/*}" 755 - require_root_owned_directory "${unit_file%/*}" 755 -} - -acquire_deploy_lock() { - if [ -e "$deploy_lock" ] || [ -L "$deploy_lock" ]; then - require_root_owned_file "$deploy_lock" 600 - else - (umask 077 && : >"$deploy_lock") - chown root:root "$deploy_lock" || fail "could not set deployment-lock ownership" - chmod 0600 "$deploy_lock" || fail "could not set deployment-lock permissions" - fi - exec 9<>"$deploy_lock" - /usr/bin/flock -n 9 || fail "another deployment is running" -} - -verify_release() { - local release="$1" sha="$2" observed - require_root_owned_directory "$release" 755 - if ! observed="$(git_exec -C "$release" config --get remote.origin.url)"; then - fail "could not read the release origin" - fi - [ "$observed" = "$fixed_origin" ] || fail "release origin is not $fixed_origin" - if ! observed="$(git_exec -C "$release" rev-parse --verify HEAD)"; then - fail "could not read the release commit" - fi - [ "$observed" = "$sha" ] || fail "release commit does not match $sha" - if ! observed="$(git_exec -C "$release" status --porcelain=v1 --untracked-files=all)"; then - fail "could not inspect the release checkout" - fi - [ -z "$observed" ] || fail "release checkout is modified" - require_root_owned_file "$release/$cleanup_relative" 755 - require_root_owned_file "$release/$environment_relative" 644 - require_root_owned_file "$release/$unit_relative" 644 -} - -prepare_release() { - local sha="$1" - local release="$releases_directory/$sha" - if [ -e "$release" ] || [ -L "$release" ]; then - verify_release "$release" "$sha" - printf '%s\n' "$release" - return - fi - - staging_directory="$(/usr/bin/mktemp -d "$releases_directory/.${sha}.XXXXXX")" - chmod 0755 "$staging_directory" || fail "could not set release-directory permissions" - git_exec -C "$staging_directory" init --quiet || fail "could not initialize the release checkout" - if ! git_exec -C "$staging_directory" remote add origin "$fixed_origin"; then - fail "could not set the fixed release origin" - fi - if ! git_exec -C "$staging_directory" fetch --depth=1 --no-tags origin "$sha"; then - fail "could not fetch commit $sha from the fixed origin" - fi - if ! git_exec -C "$staging_directory" checkout --detach --quiet FETCH_HEAD; then - fail "could not check out commit $sha" - fi - verify_release "$staging_directory" "$sha" - move_directory "$staging_directory" "$release" || fail "could not install release $sha" - staging_directory="" - verify_release "$release" "$sha" - printf '%s\n' "$release" -} - -service_load_state() { - systemctl_exec show --property=LoadState --value "$service_name" -} - -service_active_state() { - systemctl_exec show --property=ActiveState --value "$service_name" -} - -device_lock_is_absent() { - [ ! -e "$device_lock" ] && [ ! -L "$device_lock" ] -} - -require_device_lock_absent() { - device_lock_is_absent || fail "cleanup left $device_lock; recover cleanup before deployment" -} - -stop_installed_service() { - local active_state - systemctl_exec stop "$service_name" || fail "could not stop $service_name" - if ! active_state="$(service_active_state)"; then - fail "could not inspect $service_name after stop" - fi - [ "$active_state" = inactive ] || fail "$service_name has unexpected state after stop: $active_state" - require_device_lock_absent -} - -selected_release() { - local target sha - if [ ! -e "$current_link" ] && [ ! -L "$current_link" ]; then - return 1 - fi - [ -L "$current_link" ] || fail "$current_link must be a symbolic link" - target="$(/usr/bin/readlink "$current_link")" || fail "could not read $current_link" - sha="${target##*/}" - if [[ ! "$sha" =~ ^[a-f0-9]{40}$ ]] || [ "$target" != "$releases_directory/$sha" ]; then - fail "$current_link does not select one managed release" - fi - verify_release "$target" "$sha" - printf '%s\n' "$target" -} - -cleanup_link_matches() { - local metadata target - [ -L "$cleanup_executable" ] || return 1 - target="$(/usr/bin/readlink "$cleanup_executable")" || return 1 - [ "$target" = "$cleanup_link_target" ] || return 1 - metadata="$(symlink_owner "$cleanup_executable")" || return 1 - [ "$metadata" = 0:0 ] -} - -validate_prior_cleanup_selection() { - local previous_release="$1" - if [ -n "$previous_release" ]; then - cleanup_link_matches || fail "$cleanup_executable does not select the managed current release" - return - fi - if [ ! -e "$cleanup_executable" ] && [ ! -L "$cleanup_executable" ]; then - return - fi - cleanup_link_matches || fail "$cleanup_executable exists without one managed current release" -} - -require_public_ingress_disabled() { - public_ingress_is_disabled || fail "$public_ingress_error" -} - -public_ingress_is_disabled() { - local load_state active_state unit_file_state - public_ingress_error="" - if ! load_state="$(systemctl_exec show --property=LoadState --value "$tunnel_service_name")"; then - public_ingress_error="could not inspect $tunnel_service_name" - return 1 - fi - if [ "$load_state" = not-found ]; then - return - fi - if [ "$load_state" != loaded ]; then - public_ingress_error="$tunnel_service_name has unsupported load state: $load_state" - return 1 - fi - if ! active_state="$(systemctl_exec show --property=ActiveState --value "$tunnel_service_name")"; then - public_ingress_error="could not inspect $tunnel_service_name activity" - return 1 - fi - if ! unit_file_state="$(systemctl_exec show --property=UnitFileState --value "$tunnel_service_name")"; then - public_ingress_error="could not inspect $tunnel_service_name enablement" - return 1 - fi - if [ "$active_state" != inactive ] || [ "$unit_file_state" != disabled ]; then - public_ingress_error="$tunnel_service_name must be disabled and inactive" - return 1 - fi -} - -require_bootstrap_destinations_absent() { - if [ -e "$environment_file" ] || [ -L "$environment_file" ]; then - fail "$environment_file exists while $service_name is not loaded" - fi - if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then - fail "$unit_file exists while $service_name is not loaded" - fi -} - -atomic_install_cleanup_link() { - cleanup_link_matches && return 0 - if [ -e "$cleanup_executable" ] || [ -L "$cleanup_executable" ]; then - return 1 - fi - cleanup_stage="$(/usr/bin/mktemp "${cleanup_executable%/*}/.nemoclaw-jetson-cleanup.XXXXXX")" - rm -f -- "$cleanup_stage" - ln -s -- "$cleanup_link_target" "$cleanup_stage" || return 1 - move_replace "$cleanup_stage" "$cleanup_executable" || { - rm -f -- "$cleanup_stage" - cleanup_stage="" - return 1 - } - cleanup_stage="" - cleanup_link_matches -} - -atomic_select_release() { - local release="$1" - temporary_link="$deploy_root/.current.${release##*/}.$$" - if [ -e "$temporary_link" ] || [ -L "$temporary_link" ]; then - return 1 - fi - ln -s -- "$release" "$temporary_link" || { - temporary_link="" - return 1 - } - move_replace "$temporary_link" "$current_link" || { - rm -f -- "$temporary_link" - temporary_link="" - return 1 - } - temporary_link="" - [ -L "$current_link" ] && [ "$(/usr/bin/readlink "$current_link")" = "$release" ] -} - -install_managed_file() { - local source="$1" destination="$2" mode="$3" prefix="$4" - managed_file_stage="$(/usr/bin/mktemp "${destination%/*}/.${prefix}.XXXXXX")" || return 1 - if ! install_file "$source" "$managed_file_stage" "$mode"; then - rm -f -- "$managed_file_stage" - managed_file_stage="" - return 1 - fi - if ! root_owned_file_matches "$managed_file_stage" "$mode" \ - || ! files_match "$source" "$managed_file_stage" \ - || ! move_replace "$managed_file_stage" "$destination"; then - rm -f -- "$managed_file_stage" - managed_file_stage="" - return 1 - fi - managed_file_stage="" - root_owned_file_matches "$destination" "$mode" && files_match "$source" "$destination" -} - -install_bootstrap_files() { - local release="$1" - install_managed_file "$release/$environment_relative" "$environment_file" 600 environment \ - || return 1 - bootstrap_environment_installed=1 - install_managed_file "$release/$unit_relative" "$unit_file" 644 nemoclaw-jetson-dispatch.service \ - || return 1 - bootstrap_unit_installed=1 -} - -loopback_service_responds() { - local listeners http_code - systemctl_exec is-active --quiet "$service_name" || return 1 - listeners="$(ss_exec -H -ltn "sport = :$service_port")" || return 1 - [ -n "$listeners" ] || return 1 - printf '%s\n' "$listeners" | /usr/bin/awk -v expected="127.0.0.1:$service_port" ' - NF < 4 || $4 != expected { invalid = 1 } - END { exit invalid || NR == 0 } - ' || return 1 - http_code="$( - curl_exec \ - --disable \ - --noproxy '*' \ - --silent \ - --show-error \ - --output /dev/null \ - --write-out '%{http_code}' \ - --max-time 5 \ - --request POST \ - --header 'Content-Type: application/json' \ - --data '{"schemaVersion":1,"target":"jetson-nvmap-gpu","candidateSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","workflowRunId":"1","workflowRunAttempt":1}' \ - "http://127.0.0.1:$service_port/v1/jobs" - )" || return 1 - [ "$http_code" = 401 ] -} - -verify_loopback_service() { - local _ - for _ in {1..20}; do - loopback_service_responds && return 0 - sleep_exec 0.25 - done - return 1 -} - -start_and_verify_service() { - public_ingress_is_disabled || return 1 - systemctl_exec start "$service_name" || return 1 - if ! public_ingress_is_disabled; then - stop_unverified_service || return 1 - return 1 - fi - verify_loopback_service || return 1 - if ! public_ingress_is_disabled; then - stop_unverified_service || return 1 - return 1 - fi -} - -enable_and_verify_initial_service() { - local load_state - systemctl_exec daemon-reload || return 1 - bootstrap_daemon_reloaded=1 - load_state="$(service_load_state)" || return 1 - [ "$load_state" = loaded ] || return 1 - public_ingress_is_disabled || return 1 - bootstrap_enable_attempted=1 - systemctl_exec enable --now "$service_name" || return 1 - public_ingress_is_disabled || return 1 - verify_loopback_service || return 1 - public_ingress_is_disabled -} - -stop_unverified_service() { - local active_state - systemctl_exec stop "$service_name" || return 1 - active_state="$(service_active_state)" || return 1 - [ "$active_state" = inactive ] || return 1 - device_lock_is_absent -} - -restore_cleanup_selection() { - local cleanup_was_present="$1" - if [ "$cleanup_was_present" = 1 ]; then - cleanup_link_matches - else - rm -f -- "$cleanup_executable" - fi -} - -restore_release_selection() { - local previous_release="$1" - if [ -n "$previous_release" ]; then - atomic_select_release "$previous_release" - return - fi - if [ -L "$current_link" ]; then - rm -f -- "$current_link" - elif [ -e "$current_link" ]; then - return 1 - fi -} - -rollback_deployment() { - local previous_release="$1" cleanup_was_present="$2" service_installed="$3" rollback_failed=0 active_state - if [ "$service_installed" = 1 ]; then - systemctl_exec stop "$service_name" || return 1 - active_state="$(service_active_state)" || return 1 - [ "$active_state" = inactive ] || return 1 - device_lock_is_absent || return 1 - fi - restore_release_selection "$previous_release" || rollback_failed=1 - restore_cleanup_selection "$cleanup_was_present" || rollback_failed=1 - if [ "$service_installed" = 1 ] && [ -n "$previous_release" ]; then - start_and_verify_service || rollback_failed=1 - fi - [ "$rollback_failed" = 0 ] -} - -rollback_initial_deployment() { - local previous_release="$1" cleanup_was_present="$2" active_state load_state rollback_failed=0 - if [ "$bootstrap_enable_attempted" = 1 ]; then - systemctl_exec disable --now "$service_name" || return 1 - active_state="$(service_active_state)" || return 1 - [ "$active_state" = inactive ] || return 1 - device_lock_is_absent || return 1 - fi - - restore_release_selection "$previous_release" || rollback_failed=1 - restore_cleanup_selection "$cleanup_was_present" || rollback_failed=1 - - if [ "$bootstrap_unit_installed" = 1 ]; then - root_owned_file_matches "$unit_file" 644 || return 1 - rm -f -- "$unit_file" || return 1 - fi - if [ "$bootstrap_environment_installed" = 1 ]; then - root_owned_file_matches "$environment_file" 600 || return 1 - rm -f -- "$environment_file" || return 1 - fi - if [ "$bootstrap_daemon_reloaded" = 1 ]; then - systemctl_exec daemon-reload || return 1 - load_state="$(service_load_state)" || return 1 - [ "$load_state" = not-found ] || return 1 - fi - [ "$rollback_failed" = 0 ] -} - -activate_release() { - local release="$1" - atomic_install_cleanup_link || return 1 - atomic_select_release "$release" || return 1 - cleanup_link_matches || return 1 - [ "$(/usr/bin/readlink "$current_link")" = "$release" ] || return 1 -} - -main() { - local sha release load_state service_installed=0 previous_release="" cleanup_was_present=0 - require_root - sha="$(parse_commit "$@")" - print_stage 1 "Validate request and prepared host" - validate_dispatcher_prerequisites - ensure_layout - acquire_deploy_lock - - load_state="$(service_load_state)" || fail "could not inspect $service_name" - if [ "$load_state" = loaded ]; then - service_installed=1 - print_stage 2 "Stop and verify the dispatcher" - require_public_ingress_disabled - stop_installed_service - elif [ "$load_state" != not-found ]; then - fail "$service_name has unsupported load state: $load_state" - else - print_stage 2 "Verify initial deployment state" - require_device_lock_absent - require_public_ingress_disabled - require_bootstrap_destinations_absent - fi - - if [ -e "$current_link" ] || [ -L "$current_link" ]; then - previous_release="$(selected_release)" || fail "could not validate $current_link" - fi - validate_prior_cleanup_selection "$previous_release" - [ -n "$previous_release" ] && cleanup_was_present=1 - if [ "$service_installed" = 1 ] && [ -z "$previous_release" ]; then - fail "$service_name is installed without one managed current release" - fi - print_stage 3 "Fetch and verify the release" - release="$(prepare_release "$sha")" - - print_stage 4 "Select dispatcher and cleanup code" - if [ "$service_installed" = 1 ]; then - require_public_ingress_disabled - fi - if ! activate_release "$release"; then - if rollback_deployment "$previous_release" "$cleanup_was_present" "$service_installed"; then - fail "release $sha activation or verification failed; the previous deployment state was restored" - fi - fail "release $sha activation or verification failed, and rollback did not restore a verified service" - fi - - if [ "$service_installed" = 1 ]; then - print_stage 5 "Start and verify the dispatcher" - if ! start_and_verify_service; then - if rollback_deployment "$previous_release" "$cleanup_was_present" "$service_installed"; then - fail "release $sha activation or verification failed; the previous deployment state was restored" - fi - fail "release $sha activation or verification failed, and rollback did not restore a verified service" - fi - echo "Deployed Colossus Jetson dispatcher release $sha and verified its loopback service" - else - print_stage 5 "Install, enable, and verify the dispatcher" - if ! install_bootstrap_files "$release" || ! enable_and_verify_initial_service; then - if rollback_initial_deployment "$previous_release" "$cleanup_was_present"; then - fail "release $sha initial deployment failed; the release, unit, and environment were rolled back" - fi - fail "release $sha initial deployment failed, and rollback did not restore the prepared host" - fi - echo "Deployed Colossus Jetson dispatcher release $sha and verified its loopback service; public ingress remains disabled" - fi -} - -if [[ "${BASH_SOURCE[0]}" = "$0" ]]; then - main "$@" -fi diff --git a/tools/e2e/colossus-jetson-dispatch.environment b/tools/e2e/colossus-jetson-dispatch.environment deleted file mode 100644 index a3fa0a8e618..00000000000 --- a/tools/e2e/colossus-jetson-dispatch.environment +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -JETSON_DISPATCH_STATE_DIRECTORY=/var/lib/nemoclaw-jetson-dispatch/state -JETSON_DISPATCH_GITHUB_REPOSITORY_ID=1182547092 -JETSON_DISPATCH_PORT=8787 -JETSON_DISPATCH_EXECUTION_TIMEOUT_SECONDS=3000 -JETSON_DISPATCH_CLEANUP_TIMEOUT_SECONDS=300 -JETSON_DISPATCH_TEST_TIMEOUT_SECONDS=2700 -JETSON_DISPATCH_SSH_DESTINATION=nvidia@192.168.55.1 -JETSON_DISPATCH_SSH_IDENTITY_FILE=/var/lib/nemoclaw-jetson-dispatch/id_ed25519 -JETSON_DISPATCH_SSH_KNOWN_HOSTS_FILE=/var/lib/nemoclaw-jetson-dispatch/known_hosts -JETSON_DISPATCH_CLEANUP_EXECUTABLE=/usr/local/libexec/nemoclaw-jetson-cleanup diff --git a/tools/e2e/contracts/v1/jetson-dispatch.json b/tools/e2e/contracts/v1/jetson-dispatch.json new file mode 100644 index 00000000000..29eac93469d --- /dev/null +++ b/tools/e2e/contracts/v1/jetson-dispatch.json @@ -0,0 +1,110 @@ +{ + "license": "Apache-2.0", + "contractVersion": "1.0.0", + "request": { + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflowRunId": "123456789", + "workflowRunAttempt": 1 + }, + "queuedResponse": { + "job": { + "schemaVersion": 1, + "jobId": "4f403de482f4ad3cc04bb1179fef21b61e9761671a6653e05d41b161c450413f", + "request": { + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflowRunId": "123456789", + "workflowRunAttempt": 1 + }, + "state": "queued", + "createdAt": "2026-08-12T00:00:00.000Z", + "cleanup": "pending" + } + }, + "completedStatus": { + "schemaVersion": 1, + "jobId": "4f403de482f4ad3cc04bb1179fef21b61e9761671a6653e05d41b161c450413f", + "request": { + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflowRunId": "123456789", + "workflowRunAttempt": 1 + }, + "state": "completed", + "conclusion": "success", + "createdAt": "2026-08-12T00:00:00.000Z", + "startedAt": "2026-08-12T00:00:01.000Z", + "completedAt": "2026-08-12T00:01:00.000Z", + "device": { + "model": "NVIDIA Jetson test fixture", + "jetpackVersion": "fixture", + "jetsonLinuxRelease": "fixture", + "kernel": "fixture" + }, + "cleanup": "succeeded" + }, + "artifact": { + "status": { + "schemaVersion": 1, + "jobId": "4f403de482f4ad3cc04bb1179fef21b61e9761671a6653e05d41b161c450413f", + "request": { + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflowRunId": "123456789", + "workflowRunAttempt": 1 + }, + "state": "completed", + "conclusion": "success", + "createdAt": "2026-08-12T00:00:00.000Z", + "startedAt": "2026-08-12T00:00:01.000Z", + "completedAt": "2026-08-12T00:01:00.000Z", + "device": { + "model": "NVIDIA Jetson test fixture", + "jetpackVersion": "fixture", + "jetsonLinuxRelease": "fixture", + "kernel": "fixture" + }, + "cleanup": "succeeded" + }, + "log": "fixture log\n", + "artifactArchiveBase64": "dGVzdA==" + }, + "invalidRequests": [ + { + "name": "extra request field", + "value": { + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflowRunId": "123456789", + "workflowRunAttempt": 1, + "command": "untrusted" + } + }, + { + "name": "wrong target", + "value": { + "schemaVersion": 1, + "target": "another-target", + "candidateSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflowRunId": "123456789", + "workflowRunAttempt": 1 + } + }, + { + "name": "noncanonical candidate SHA", + "value": { + "schemaVersion": 1, + "target": "jetson-nvmap-gpu", + "candidateSha": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "workflowRunId": "123456789", + "workflowRunAttempt": 1 + } + } + ] +} diff --git a/tools/e2e/jetson-dispatch-cleanup.sh b/tools/e2e/jetson-dispatch-cleanup.sh deleted file mode 100755 index 44b1563e47e..00000000000 --- a/tools/e2e/jetson-dispatch-cleanup.sh +++ /dev/null @@ -1,635 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -if [ "$#" -ne 0 ]; then - echo "Jetson cleanup accepts no arguments" >&2 - exit 2 -fi - -state_directory=/var/lib/nemoclaw-jetson-dispatch/state -lock_file="$state_directory/device.lock" -identity_file=/var/lib/nemoclaw-jetson-dispatch/id_ed25519 -known_hosts_file=/var/lib/nemoclaw-jetson-dispatch/known_hosts -destination=nvidia@192.168.55.1 - -mapfile -t lock_lines <"$lock_file" -if [ "${#lock_lines[@]}" -ne 1 ]; then - echo "Jetson cleanup lock contains trailing data" >&2 - exit 1 -fi -job_id="${lock_lines[0]}" -if ! [[ "$job_id" =~ ^[a-f0-9]{64}$ ]]; then - echo "Jetson cleanup lock contains an invalid job ID" >&2 - exit 1 -fi - -ssh_args=( - -F /dev/null - -T - -o BatchMode=yes - -o ConnectTimeout=15 - -o IdentitiesOnly=yes - -o ServerAliveCountMax=2 - -o ServerAliveInterval=15 - -o StrictHostKeyChecking=yes - -o "UserKnownHostsFile=$known_hosts_file" - -i "$identity_file" -) - -discovery_output="$( - ssh "${ssh_args[@]}" "$destination" bash -s -- "$job_id" <<'JETSON_DISCOVERY' -set -euo pipefail - -job_id="$1" -[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] - -workspace_root=/var/tmp/nemoclaw-jetson-e2e -workspace="$workspace_root/$job_id" -job_home="$workspace/home" -sandbox_name=e2e-jetson-nvmap -gateway_container=openshell-cluster-nemoclaw -gateway_volume=openshell-cluster-nemoclaw -image_repository=nemoclaw-sandbox-local -service_directory="/tmp/nemoclaw-services-$sandbox_name" - -require_plain_directory_if_present() { - local directory="$1" - if [ -L "$directory" ] || { [ -e "$directory" ] && [ ! -d "$directory" ]; }; then - echo "Refusing an untrusted cleanup directory: $directory" >&2 - exit 1 - fi -} - -require_plain_directory_if_present "$workspace_root" -require_plain_directory_if_present "$workspace" -require_plain_directory_if_present "$job_home" -require_plain_directory_if_present "$job_home/.nemoclaw" -require_plain_directory_if_present "$job_home/.local" -require_plain_directory_if_present "$job_home/.local/state" -require_plain_directory_if_present "$job_home/.local/state/nemoclaw" -require_plain_directory_if_present "$job_home/.local/state/nemoclaw/openshell-docker-gateway" -require_plain_directory_if_present "$service_directory" - -recorded_volumes=() -record_volume() { - local volume="$1" recorded - [[ "$volume" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$ ]] || { - echo "Refusing an invalid recorded Docker volume name" >&2 - exit 1 - } - for recorded in "${recorded_volumes[@]}"; do - [ "$recorded" = "$volume" ] && return 0 - done - recorded_volumes+=("$volume") -} - -record_container_volumes() { - local container="$1" volume volume_output - volume_output="$( - docker container inspect --format \ - '{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}' \ - "$container" - )" || { - echo "Unable to inspect a test-owned Docker container" >&2 - exit 1 - } - while IFS= read -r volume; do - [ -n "$volume" ] || continue - record_volume "$volume" - done <<<"$volume_output" -} - -sandbox_container_output="$( - docker ps -aq \ - --filter label=openshell.ai/managed-by=openshell \ - --filter "label=openshell.ai/sandbox-name=$sandbox_name" -)" || { - echo "Unable to list test-owned Docker containers" >&2 - exit 1 -} -sandbox_containers=() -if [ -n "$sandbox_container_output" ]; then - mapfile -t sandbox_containers <<<"$sandbox_container_output" -fi -for container in "${sandbox_containers[@]}"; do - [[ "$container" =~ ^[a-f0-9]{12,64}$ ]] || { - echo "Refusing an invalid recorded Docker container ID" >&2 - exit 1 - } - container_image="$(docker container inspect --format '{{.Config.Image}}' "$container")" - case "$container_image" in - "$image_repository:$sandbox_name-"*) ;; - *) echo "Refusing a sandbox container whose image is not test-owned" >&2; exit 1 ;; - esac - record_container_volumes "$container" -done -container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { - echo "Unable to list Docker containers" >&2 - exit 1 -} -gateway_container_ids="$( - printf '%s\n' "$container_rows" | awk -F '\t' -v name="$gateway_container" '$2 == name { print $1 }' -)" -if [ -n "$gateway_container_ids" ]; then - if [[ "$gateway_container_ids" == *$'\n'* ]] || ! [[ "$gateway_container_ids" =~ ^[a-f0-9]{12,64}$ ]]; then - echo "Refusing an ambiguous test-owned gateway container" >&2 - exit 1 - fi - record_container_volumes "$gateway_container_ids" -fi -record_volume "$gateway_volume" - -read_recorded_pid() { - local pid_file="$1" - if [ ! -e "$pid_file" ]; then - return 0 - fi - if [ -L "$pid_file" ] || [ ! -f "$pid_file" ]; then - echo "Refusing an untrusted cleanup PID file: $pid_file" >&2 - exit 1 - fi - local pid - IFS= read -r pid <"$pid_file" - [[ "$pid" =~ ^[1-9][0-9]*$ ]] || { - echo "Refusing an invalid cleanup PID file: $pid_file" >&2 - exit 1 - } - printf '%s\n' "$pid" -} - -read_proc_uid() { - awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' "$1/status" 2>/dev/null -} - -read_proc_environment() { - dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' -} - -read_proc_command() { - dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' -} - -handle_proc_read_failure() { - local proc_dir="$1" field="$2" process_uid directory_uid - [ -d "$proc_dir" ] || return 0 - if process_uid="$(read_proc_uid "$proc_dir")"; then - [ "$process_uid" = "$(id -u)" ] || return 0 - else - [ -d "$proc_dir" ] || return 0 - if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then - [ -d "$proc_dir" ] || return 0 - echo "Unable to verify the owner of a live process after a failed $field read" >&2 - exit 1 - fi - [ "$directory_uid" = "$(id -u)" ] || return 0 - fi - echo "Unable to inspect $field for a live same-user process" >&2 - exit 1 -} - -validate_recorded_pid() { - local pid="$1" marker="$2" - [ -n "$pid" ] || return 0 - kill -0 "$pid" 2>/dev/null || return 0 - local process_uid cmdline environment proc_dir="/proc/$pid" - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Refusing to record a live process whose owner cannot be verified" >&2 - exit 1 - fi - [ "$process_uid" = "$(id -u)" ] || { - echo "Refusing to record a process owned by another user" >&2 - exit 1 - } - if ! cmdline="$(read_proc_command "$proc_dir")"; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Refusing to record a live process whose command cannot be verified" >&2 - exit 1 - fi - case "$cmdline" in - *"$marker"*) ;; - *) echo "Refusing to record a PID with an unexpected command" >&2; exit 1 ;; - esac - if ! environment="$(read_proc_environment "$proc_dir")"; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Refusing to record a live process whose environment cannot be verified" >&2 - exit 1 - fi - if ! printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home"; then - echo "Refusing to record a process outside the job home" >&2 - exit 1 - fi -} - -auth_proxy_pid="$(read_recorded_pid "$job_home/.nemoclaw/ollama-auth-proxy.pid")" -gateway_pid="$(read_recorded_pid "$job_home/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid")" -cloudflared_pid="$(read_recorded_pid "$service_directory/cloudflared.pid")" -validate_recorded_pid "$auth_proxy_pid" ollama-auth-proxy. -validate_recorded_pid "$gateway_pid" openshell-gateway -validate_recorded_pid "$cloudflared_pid" cloudflared - -printf '%s\n' nemoclaw-cleanup-evidence-v1-begin -for volume in "${recorded_volumes[@]}"; do - printf 'volume\t%s\n' "$volume" -done -for pid in "$auth_proxy_pid" "$gateway_pid" "$cloudflared_pid"; do - if [ -n "$pid" ]; then - printf 'processId\t%s\n' "$pid" - fi -done -for proc_dir in /proc/[0-9]*; do - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" owner - continue - fi - [ "$process_uid" = "$(id -u)" ] || continue - if ! environment="$(read_proc_environment "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" environment - continue - fi - printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue - if ! cmdline="$(read_proc_command "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" command - continue - fi - case "$cmdline" in - *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) - printf 'processId\t%s\n' "${proc_dir##*/}" - ;; - esac -done -printf '%s\n' nemoclaw-cleanup-evidence-v1-end -JETSON_DISCOVERY -)" - -canonical_evidence="$( - # shellcheck disable=SC2016 - printf '%s\n' "$discovery_output" | /usr/bin/node -e ' - const fs = require("node:fs"); - const path = require("node:path"); - const stateDirectory = process.argv[1]; - const jobId = process.argv[2]; - if (!/^[a-f0-9]{64}$/.test(jobId)) process.exit(1); - const begin = "nemoclaw-cleanup-evidence-v1-begin"; - const end = "nemoclaw-cleanup-evidence-v1-end"; - const lines = fs.readFileSync(0, "utf8").replace(/\r/g, "").trimEnd().split("\n"); - const beginIndex = lines.indexOf(begin); - const endIndex = lines.indexOf(end); - if (beginIndex < 0 || endIndex < 0 || beginIndex !== lines.lastIndexOf(begin) || endIndex !== lines.lastIndexOf(end) || beginIndex >= endIndex) process.exit(1); - const discovered = { schemaVersion: 1, volumes: [], processIds: [] }; - for (const line of lines.slice(beginIndex + 1, endIndex)) { - const fields = line.split("\t"); - if (fields.length !== 2) process.exit(1); - if (fields[0] === "volume" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(fields[1])) discovered.volumes.push(fields[1]); - else if (fields[0] === "processId" && /^[1-9][0-9]*$/.test(fields[1]) && Number.isSafeInteger(Number(fields[1]))) discovered.processIds.push(Number(fields[1])); - else process.exit(1); - } - const cleanupFile = path.join(stateDirectory, `${jobId}.cleanup.json`); - let previous = { schemaVersion: 1, volumes: [], processIds: [] }; - if (fs.existsSync(cleanupFile)) { - const metadata = fs.lstatSync(cleanupFile); - if (!metadata.isFile() || metadata.nlink !== 1 || (metadata.mode & 0o077) !== 0 || metadata.size > 64 * 1024) process.exit(1); - previous = JSON.parse(fs.readFileSync(cleanupFile, "utf8")); - } - const validate = (record) => { - const keys = Object.keys(record).sort(); - if (JSON.stringify(keys) !== JSON.stringify(["processIds", "schemaVersion", "volumes"]) || record.schemaVersion !== 1 || !Array.isArray(record.volumes) || !Array.isArray(record.processIds)) process.exit(1); - if (!record.volumes.every((value) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(value))) process.exit(1); - if (!record.processIds.every((value) => Number.isSafeInteger(value) && value > 0)) process.exit(1); - }; - validate(previous); - validate(discovered); - const merged = { - schemaVersion: 1, - volumes: [...new Set([...previous.volumes, ...discovered.volumes])].sort(), - processIds: [...new Set([...previous.processIds, ...discovered.processIds])].sort((a, b) => a - b), - }; - const serialized = `${JSON.stringify(merged)}\n`; - if (Buffer.byteLength(serialized) > 64 * 1024) process.exit(1); - const temporaryFile = path.join(stateDirectory, `.${jobId}.${process.pid}.cleanup.tmp`); - let descriptor; - try { - descriptor = fs.openSync(temporaryFile, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | (fs.constants.O_NOFOLLOW ?? 0), 0o600); - fs.fchmodSync(descriptor, 0o600); - fs.writeFileSync(descriptor, serialized, "utf8"); - fs.fsyncSync(descriptor); - } finally { - if (descriptor !== undefined) fs.closeSync(descriptor); - } - fs.renameSync(temporaryFile, cleanupFile); - const directoryDescriptor = fs.openSync(stateDirectory, fs.constants.O_RDONLY); - try { fs.fsyncSync(directoryDescriptor); } finally { fs.closeSync(directoryDescriptor); } - process.stdout.write(serialized); - ' "$state_directory" "$job_id" -)" - -cleanup_evidence_base64="$(printf '%s\n' "$canonical_evidence" | base64 --wrap=0)" -ssh "${ssh_args[@]}" "$destination" bash -s -- "$job_id" "$cleanup_evidence_base64" <<'JETSON_CLEANUP' -set -euo pipefail - -job_id="$1" -cleanup_evidence_base64="$2" -[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] - -workspace_root=/var/tmp/nemoclaw-jetson-e2e -workspace="$workspace_root/$job_id" -job_home="$workspace/home" -sandbox_name=e2e-jetson-nvmap -gateway_container=openshell-cluster-nemoclaw -image_repository=nemoclaw-sandbox-local -service_directory="/tmp/nemoclaw-services-$sandbox_name" - -require_plain_directory_if_present() { - local directory="$1" - if [ -L "$directory" ] || { [ -e "$directory" ] && [ ! -d "$directory" ]; }; then - echo "Refusing an untrusted cleanup directory: $directory" >&2 - exit 1 - fi -} - -require_plain_directory_if_present "$workspace_root" -require_plain_directory_if_present "$workspace" -require_plain_directory_if_present "$job_home" -require_plain_directory_if_present "$job_home/.nemoclaw" -require_plain_directory_if_present "$job_home/.local" -require_plain_directory_if_present "$job_home/.local/state" -require_plain_directory_if_present "$job_home/.local/state/nemoclaw" -require_plain_directory_if_present "$job_home/.local/state/nemoclaw/openshell-docker-gateway" -require_plain_directory_if_present "$service_directory" - -cleanup_rows="$( - printf '%s' "$cleanup_evidence_base64" | base64 --decode | node -e ' - const fs = require("node:fs"); - const value = JSON.parse(fs.readFileSync(0, "utf8")); - const keys = Object.keys(value).sort(); - if (JSON.stringify(keys) !== JSON.stringify(["processIds", "schemaVersion", "volumes"]) || value.schemaVersion !== 1 || !Array.isArray(value.volumes) || !Array.isArray(value.processIds)) process.exit(1); - for (const volume of value.volumes) { - if (typeof volume !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(volume)) process.exit(1); - process.stdout.write("volume\t" + volume + "\n"); - } - for (const pid of value.processIds) { - if (!Number.isSafeInteger(pid) || pid < 1) process.exit(1); - process.stdout.write("processId\t" + pid + "\n"); - } - ' -)" -recorded_volumes=() -recorded_process_ids=() -if [ -n "$cleanup_rows" ]; then - while IFS=$'\t' read -r kind identity; do - case "$kind" in - volume) recorded_volumes+=("$identity") ;; - processId) recorded_process_ids+=("$identity") ;; - *) exit 1 ;; - esac - done <<<"$cleanup_rows" -fi - -read_proc_uid() { - awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' "$1/status" 2>/dev/null -} - -read_proc_environment() { - dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' -} - -read_proc_command() { - dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' -} - -handle_proc_read_failure() { - local proc_dir="$1" field="$2" process_uid directory_uid - [ -d "$proc_dir" ] || return 0 - if process_uid="$(read_proc_uid "$proc_dir")"; then - [ "$process_uid" = "$(id -u)" ] || return 0 - else - [ -d "$proc_dir" ] || return 0 - if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then - [ -d "$proc_dir" ] || return 0 - echo "Unable to verify the owner of a live process after a failed $field read" >&2 - exit 1 - fi - [ "$directory_uid" = "$(id -u)" ] || return 0 - fi - echo "Unable to inspect $field for a live same-user process" >&2 - exit 1 -} - -stop_recorded_pid() { - local pid="$1" - kill -0 "$pid" 2>/dev/null || return 0 - local process_uid cmdline environment proc_dir="/proc/$pid" - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Refusing to stop a live process whose owner cannot be verified" >&2 - exit 1 - fi - [ "$process_uid" = "$(id -u)" ] || { - echo "Refusing to stop a process owned by another user" >&2 - exit 1 - } - if ! cmdline="$(read_proc_command "$proc_dir")"; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Refusing to stop a live process whose command cannot be verified" >&2 - exit 1 - fi - case "$cmdline" in - *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) ;; - *) echo "Refusing to stop a recorded PID with an unexpected command" >&2; exit 1 ;; - esac - if ! environment="$(read_proc_environment "$proc_dir")"; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Refusing to stop a live process whose environment cannot be verified" >&2 - exit 1 - fi - if ! printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home"; then - echo "Refusing to stop a process outside the job home" >&2 - exit 1 - fi - if ! kill "$pid" 2>/dev/null; then - kill -0 "$pid" 2>/dev/null || return 0 - echo "Unable to stop a validated cleanup process" >&2 - exit 1 - fi - for _attempt in 1 2 3 4 5 6 7 8 9 10; do - kill -0 "$pid" 2>/dev/null || return 0 - sleep 0.2 - done - kill -KILL "$pid" - for _attempt in 1 2 3 4 5; do - kill -0 "$pid" 2>/dev/null || return 0 - sleep 0.2 - done - echo "Recorded cleanup process $pid did not stop" >&2 - exit 1 -} - -for pid in "${recorded_process_ids[@]}"; do - stop_recorded_pid "$pid" -done - -sandbox_container_output="$( - docker ps -aq \ - --filter label=openshell.ai/managed-by=openshell \ - --filter "label=openshell.ai/sandbox-name=$sandbox_name" -)" || { - echo "Unable to list test-owned Docker containers" >&2 - exit 1 -} -sandbox_containers=() -if [ -n "$sandbox_container_output" ]; then - mapfile -t sandbox_containers <<<"$sandbox_container_output" -fi -for container in "${sandbox_containers[@]}"; do - [[ "$container" =~ ^[a-f0-9]{12,64}$ ]] || exit 1 - container_image="$(docker container inspect --format '{{.Config.Image}}' "$container")" - case "$container_image" in - "$image_repository:$sandbox_name-"*) ;; - *) echo "Refusing a sandbox container whose image is not test-owned" >&2; exit 1 ;; - esac - docker container rm --force "$container" -done -container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { - echo "Unable to list Docker containers" >&2 - exit 1 -} -gateway_container_ids="$( - printf '%s\n' "$container_rows" | awk -F '\t' -v name="$gateway_container" '$2 == name { print $1 }' -)" -if [ -n "$gateway_container_ids" ]; then - if [[ "$gateway_container_ids" == *$'\n'* ]] || ! [[ "$gateway_container_ids" =~ ^[a-f0-9]{12,64}$ ]]; then - echo "Refusing an ambiguous test-owned gateway container" >&2 - exit 1 - fi - docker container rm --force "$gateway_container_ids" -fi -volume_names="$(docker volume ls --format '{{.Name}}')" || { - echo "Unable to list Docker volumes" >&2 - exit 1 -} -for volume in "${recorded_volumes[@]}"; do - if printf '%s\n' "$volume_names" | grep -Fqx "$volume"; then - docker volume rm "$volume" - fi -done - -list_test_owned_images() { - local image_rows repository tag - image_rows="$(docker image ls "$image_repository" --format '{{.Repository}}\t{{.Tag}}')" || { - echo "Unable to list test-owned Docker images" >&2 - return 1 - } - while IFS=$'\t' read -r repository tag; do - [ -n "$repository" ] || continue - if [ "$repository" = "$image_repository" ] && [[ "$tag" =~ ^e2e-jetson-nvmap-[a-z0-9_.-]+$ ]]; then - printf '%s:%s\n' "$repository" "$tag" - fi - done <<<"$image_rows" -} - -job_images_output="$(list_test_owned_images)" -job_images=() -if [ -n "$job_images_output" ]; then - mapfile -t job_images <<<"$job_images_output" -fi -for image in "${job_images[@]}"; do - docker image rm "$image" -done - -if [ -e "$service_directory" ]; then - rm -rf -- "$service_directory" -fi -if [ -e "$workspace" ]; then - rm -rf -- "$workspace" -fi - -test ! -e "$workspace" -test ! -e "$service_directory" -sandbox_container_output="$(docker ps -aq \ - --filter label=openshell.ai/managed-by=openshell \ - --filter "label=openshell.ai/sandbox-name=$sandbox_name")" || { - echo "Unable to verify test-owned Docker container absence" >&2 - exit 1 -} -if [ -n "$sandbox_container_output" ]; then - echo "A test-owned sandbox container remains" >&2 - exit 1 -fi -container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" || { - echo "Unable to verify Docker container absence" >&2 - exit 1 -} -if printf '%s\n' "$container_rows" | awk -F '\t' -v name="$gateway_container" '$2 == name { found = 1 } END { exit found ? 0 : 1 }'; then - echo "The test-owned gateway container remains" >&2 - exit 1 -fi -volume_names="$(docker volume ls --format '{{.Name}}')" || { - echo "Unable to verify Docker volume absence" >&2 - exit 1 -} -for volume in "${recorded_volumes[@]}"; do - if printf '%s\n' "$volume_names" | grep -Fqx "$volume"; then - echo "A recorded test-owned Docker volume remains" >&2 - exit 1 - fi -done -for pid in "${recorded_process_ids[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - echo "A recorded test-owned helper process remains" >&2 - exit 1 - fi -done -for proc_dir in /proc/[0-9]*; do - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" owner - continue - fi - [ "$process_uid" = "$(id -u)" ] || continue - if ! environment="$(read_proc_environment "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" environment - continue - fi - printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue - if ! cmdline="$(read_proc_command "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" command - continue - fi - case "$cmdline" in - *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) - echo "A job-owned helper process remains" >&2 - exit 1 - ;; - esac -done -remaining_job_images="$(list_test_owned_images)" || exit 1 -if [ -n "$remaining_job_images" ]; then - echo "A test-owned Jetson image remains" >&2 - exit 1 -fi -for openshell_component in openshell openshell-gateway openshell-sandbox; do - if PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - command -v "$openshell_component" >/dev/null 2>&1; then - echo "A host-level OpenShell binary remains after cleanup" >&2 - exit 1 - fi - for host_bin in "/usr/local/bin/$openshell_component" "/usr/bin/$openshell_component" "$HOME/.local/bin/$openshell_component"; do - if [ -e "$host_bin" ] || [ -L "$host_bin" ]; then - echo "A host-level OpenShell binary remains after cleanup" >&2 - exit 1 - fi - done -done -JETSON_CLEANUP - -printf '%s\n' nemoclaw-cleanup-evidence-v1-begin -# shellcheck disable=SC2016 -printf '%s\n' "$canonical_evidence" | /usr/bin/node -e ' - const fs = require("node:fs"); - const value = JSON.parse(fs.readFileSync(0, "utf8")); - for (const volume of value.volumes) console.log(`volume\t${volume}`); - for (const pid of value.processIds) console.log(`processId\t${pid}`); -' -printf '%s\n' nemoclaw-cleanup-evidence-v1-end diff --git a/tools/e2e/jetson-dispatch-client.mts b/tools/e2e/jetson-dispatch-client.mts index a74632c86b4..b7d7aeb0725 100644 --- a/tools/e2e/jetson-dispatch-client.mts +++ b/tools/e2e/jetson-dispatch-client.mts @@ -6,16 +6,16 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { + decodeJetsonArtifactArchive, JETSON_DISPATCH_AUDIENCE, JETSON_DISPATCH_TARGET, - parseJetsonDispatchRequest, -} from "./jetson-dispatch-contract.mts"; -import { - decodeJetsonArtifactArchive, type JetsonDispatchArtifact, type JetsonDispatchStatus, MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, -} from "./jetson-dispatch-lifecycle.mts"; + parseJetsonDispatchArtifact, + parseJetsonDispatchRequest, + parseJetsonDispatchStatusResponse, +} from "./jetson-dispatch-contract.mts"; import { writePrivateRegularFile } from "./private-file.mts"; const MAX_STATUS_BYTES = 64 * 1024; @@ -23,7 +23,6 @@ const POLL_INTERVAL_MS = 10_000; const MAX_WAIT_MS = 54 * 60_000; const MAX_CONSECUTIVE_POLL_FAILURES = 3; const OIDC_TOKEN_CACHE_MS = 4 * 60_000; -const JOB_ID_PATTERN = /^[a-f0-9]{64}$/u; function record(value: unknown, name: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { @@ -32,7 +31,7 @@ function record(value: unknown, name: string): Record { return value as Record; } -function dispatcherBaseUrl(value: string | undefined): URL { +export function dispatcherBaseUrl(value: string | undefined): URL { if (!value) throw new Error("JETSON_DISPATCH_URL is required"); const url = new URL(value); if ( @@ -103,15 +102,17 @@ export function createGitHubOidcTokenProvider( const githubOidcToken = createGitHubOidcTokenProvider(); -async function dispatcherRequest(options: { +export async function dispatcherRequest(options: { baseUrl: URL; method: "DELETE" | "GET" | "POST"; path: string; body?: unknown; maxBytes: number; + fetchImpl?: typeof fetch; + tokenProvider?: () => Promise; }): Promise { - const token = await githubOidcToken(); - const response = await fetch(new URL(options.path, options.baseUrl), { + const token = await (options.tokenProvider ?? githubOidcToken)(); + const response = await (options.fetchImpl ?? fetch)(new URL(options.path, options.baseUrl), { method: options.method, headers: { Accept: "application/json", @@ -159,39 +160,6 @@ async function dispatcherRequest(options: { return payload; } -function validateStatus(value: unknown): JetsonDispatchStatus { - const response = record(value, "Jetson dispatcher response"); - const job = record(response.job, "Jetson dispatch job"); - if ( - typeof job.jobId !== "string" || - !JOB_ID_PATTERN.test(job.jobId) || - !["queued", "running", "completed"].includes(String(job.state)) - ) { - throw new Error("Jetson dispatcher returned an invalid job status"); - } - return job as unknown as JetsonDispatchStatus; -} - -function validateArtifact(value: unknown, jobId: string): JetsonDispatchArtifact { - const artifact = record(value, "Jetson dispatch artifact"); - const status = record(artifact.status, "Jetson dispatch artifact status"); - if ( - status.jobId !== jobId || - status.state !== "completed" || - typeof status.conclusion !== "string" || - typeof artifact.log !== "string" - ) { - throw new Error("Jetson dispatcher returned an invalid artifact"); - } - if (artifact.artifactArchiveBase64 !== undefined) { - decodeJetsonArtifactArchive(artifact.artifactArchiveBase64); - } - if (status.conclusion === "success" && artifact.artifactArchiveBase64 === undefined) { - throw new Error("Successful Jetson dispatch did not return its E2E artifact archive"); - } - return artifact as unknown as JetsonDispatchArtifact; -} - function delay(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } @@ -211,6 +179,9 @@ export async function pollJetsonDispatch(options: { const wait = options.wait ?? delay; let consecutiveFailures = 0; let status = options.initialStatus; + if (status.jobId !== options.jobId) { + throw new Error("Jetson dispatcher status does not match the accepted job"); + } while (status.state !== "completed") { if (options.stopping?.()) throw new Error("Jetson dispatch cancellation requested"); if (now() >= options.deadlineMs) { @@ -224,14 +195,18 @@ export async function pollJetsonDispatch(options: { } await wait(POLL_INTERVAL_MS); try { - status = validateStatus( + status = parseJetsonDispatchStatusResponse( await request({ baseUrl: options.baseUrl, method: "GET", path: `v1/jobs/${options.jobId}`, maxBytes: MAX_STATUS_BYTES, }), + options.initialStatus.request, ); + if (status.jobId !== options.jobId) { + throw new Error("Jetson dispatcher status does not match the accepted job"); + } consecutiveFailures = 0; console.log(`Jetson dispatch state: ${status.state}`); } catch (error) { @@ -286,7 +261,7 @@ async function main(): Promise { process.on("SIGINT", cancel); process.on("SIGTERM", cancel); - const dispatched = validateStatus( + const dispatched = parseJetsonDispatchStatusResponse( await dispatcherRequest({ baseUrl, method: "POST", @@ -294,6 +269,7 @@ async function main(): Promise { body: request, maxBytes: MAX_STATUS_BYTES, }), + request, ); jobId = dispatched.jobId; console.log(`Jetson dispatch accepted as ${jobId}`); @@ -312,7 +288,7 @@ async function main(): Promise { path: `v1/jobs/${jobId}/artifact`, maxBytes: MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES, }); - const artifact = validateArtifact(artifactValue, jobId); + const artifact: JetsonDispatchArtifact = parseJetsonDispatchArtifact(artifactValue, jobId); const { artifactArchiveBase64, ...artifactReceipt } = artifact; writePrivateRegularFile( path.join(artifactDirectory, "jetson-dispatch.json"), diff --git a/tools/e2e/jetson-dispatch-contract.mts b/tools/e2e/jetson-dispatch-contract.mts index 7a8c668d19a..1632c03630f 100644 --- a/tools/e2e/jetson-dispatch-contract.mts +++ b/tools/e2e/jetson-dispatch-contract.mts @@ -1,23 +1,34 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createPublicKey, createVerify, type JsonWebKeyInput, type KeyObject } from "node:crypto"; +import { createHash } from "node:crypto"; +export const JETSON_DISPATCH_CONTRACT_VERSION = "1.0.0"; export const JETSON_DISPATCH_AUDIENCE = "nemoclaw-jetson-dispatch"; export const JETSON_DISPATCH_TARGET = "jetson-nvmap-gpu"; -export const JETSON_DISPATCH_REPOSITORY = "NVIDIA/NemoClaw"; -export const JETSON_DISPATCH_WORKFLOW_REF = - "NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main"; -const GITHUB_OIDC_ISSUER = "https://token.actions.githubusercontent.com"; -const GITHUB_OIDC_JWKS_URL = `${GITHUB_OIDC_ISSUER}/.well-known/jwks`; const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const JOB_ID_PATTERN = /^[a-f0-9]{64}$/u; const POSITIVE_INTEGER_PATTERN = /^[1-9][0-9]*$/u; -const MAX_TOKEN_BYTES = 16 * 1024; -const MAX_TOKEN_LIFETIME_SECONDS = 15 * 60; -const CLOCK_SKEW_SECONDS = 30; -const JWKS_CACHE_MS = 5 * 60_000; -const UNKNOWN_KEY_REFRESH_INTERVAL_MS = 60_000; +const DISPATCH_CONCLUSIONS: readonly string[] = [ + "cancelled", + "cleanup-failed", + "failure", + "success", + "timed-out", +]; +const COMPLETED_CLEANUP_STATES: readonly string[] = ["failed", "succeeded"]; +const MAX_ERROR_CHARACTERS = 500; +const MAX_DEVICE_IDENTITY_CHARACTERS = 500; +export const MAX_JETSON_DISPATCH_LOG_BYTES = 4 * 1024 * 1024; +export const MAX_JETSON_ARTIFACT_ARCHIVE_BYTES = 1024 * 1024; +const MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS = + Math.ceil(MAX_JETSON_ARTIFACT_ARCHIVE_BYTES / 3) * 4; +const MAX_JETSON_DISPATCH_ARTIFACT_JSON_OVERHEAD_BYTES = 256 * 1024; +export const MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES = + MAX_JETSON_DISPATCH_LOG_BYTES * 6 + + MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS + + MAX_JETSON_DISPATCH_ARTIFACT_JSON_OVERHEAD_BYTES; export interface JetsonDispatchRequest { schemaVersion: 1; @@ -27,20 +38,39 @@ export interface JetsonDispatchRequest { workflowRunAttempt: number; } -export interface GitHubOidcIdentity { - repository: typeof JETSON_DISPATCH_REPOSITORY; - repositoryId: string; - runId: string; - runAttempt: number; - workflowSha: string; - tokenId: string; +export type JetsonDispatchConclusion = + | "cancelled" + | "cleanup-failed" + | "failure" + | "success" + | "timed-out"; + +export interface JetsonDeviceIdentity { + model: string; + jetpackVersion: string; + jetsonLinuxRelease: string; + kernel: string; } -export interface GitHubOidcPolicy { - repositoryId: string; +export interface JetsonDispatchStatus { + schemaVersion: 1; + jobId: string; + request: JetsonDispatchRequest; + state: "queued" | "running" | "completed"; + conclusion?: JetsonDispatchConclusion; + createdAt: string; + startedAt?: string; + completedAt?: string; + device?: JetsonDeviceIdentity; + cleanup: "pending" | "succeeded" | "failed"; + error?: string; } -export type SigningKeyResolver = (keyId: string) => Promise; +export interface JetsonDispatchArtifact { + status: JetsonDispatchStatus; + log: string; + artifactArchiveBase64?: string; +} function record(value: unknown, name: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { @@ -49,6 +79,23 @@ function record(value: unknown, name: string): Record { return value as Record; } +function requireFields( + value: Record, + name: string, + required: string[], + optional: string[] = [], +): void { + const allowed = new Set([...required, ...optional]); + if ( + required.some((field) => !Object.hasOwn(value, field)) || + Object.keys(value).some((field) => !allowed.has(field)) + ) { + throw new Error( + `${name} fields do not match Jetson dispatch contract ${JETSON_DISPATCH_CONTRACT_VERSION}`, + ); + } +} + function positiveInteger(value: unknown, name: string): number { if (!Number.isSafeInteger(value) || (value as number) < 1) { throw new Error(`${name} must be a positive safe integer`); @@ -65,20 +112,13 @@ function positiveIntegerString(value: unknown, name: string): string { export function parseJetsonDispatchRequest(value: unknown): JetsonDispatchRequest { const request = record(value, "dispatch request"); - const expectedKeys = [ + requireFields(request, "dispatch request", [ "candidateSha", "schemaVersion", "target", "workflowRunAttempt", "workflowRunId", - ]; - const actualKeys = Object.keys(request).sort(); - if ( - actualKeys.length !== expectedKeys.length || - actualKeys.some((key, index) => key !== expectedKeys[index]) - ) { - throw new Error("dispatch request fields must match the fixed Jetson contract"); - } + ]); if (request.schemaVersion !== 1) { throw new Error("dispatch request schemaVersion must be 1"); } @@ -88,187 +128,213 @@ export function parseJetsonDispatchRequest(value: unknown): JetsonDispatchReques if (typeof request.candidateSha !== "string" || !SHA_PATTERN.test(request.candidateSha)) { throw new Error("candidateSha must be a lowercase 40-character commit SHA"); } - const workflowRunId = positiveIntegerString(request.workflowRunId, "workflowRunId"); - const workflowRunAttempt = positiveInteger(request.workflowRunAttempt, "workflowRunAttempt"); return { schemaVersion: 1, target: JETSON_DISPATCH_TARGET, candidateSha: request.candidateSha, - workflowRunId, - workflowRunAttempt, + workflowRunId: positiveIntegerString(request.workflowRunId, "workflowRunId"), + workflowRunAttempt: positiveInteger(request.workflowRunAttempt, "workflowRunAttempt"), }; } -function decodeJwtPart(encoded: string, name: string): Record { - if (!/^[A-Za-z0-9_-]+$/u.test(encoded)) throw new Error(`OIDC ${name} encoding is invalid`); - let value: unknown; - try { - value = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")); - } catch { - throw new Error(`OIDC ${name} is not valid JSON`); - } - return record(value, `OIDC ${name}`); +function expectedJobId(request: JetsonDispatchRequest): string { + return createHash("sha256") + .update( + `${request.schemaVersion}:${request.target}:${request.candidateSha}:${request.workflowRunId}:${request.workflowRunAttempt}`, + "utf8", + ) + .digest("hex"); } -function stringClaim(claims: Record, name: string): string { - const value = claims[name]; - if (typeof value !== "string" || value.length === 0) { - throw new Error(`OIDC ${name} claim is invalid`); +function parseTimestamp(value: unknown, name: string): string { + if ( + typeof value !== "string" || + value.length !== 24 || + Number.isNaN(Date.parse(value)) || + new Date(value).toISOString() !== value + ) { + throw new Error(`${name} must be an ISO timestamp`); } return value; } -function numericDateClaim(claims: Record, name: string): number { - const value = claims[name]; - if (!Number.isSafeInteger(value) || (value as number) < 1) { - throw new Error(`OIDC ${name} claim is invalid`); +function parseDeviceIdentity(value: unknown): JetsonDeviceIdentity { + const device = record(value, "Jetson device identity"); + requireFields(device, "Jetson device identity", [ + "jetpackVersion", + "jetsonLinuxRelease", + "kernel", + "model", + ]); + for (const field of ["jetpackVersion", "jetsonLinuxRelease", "kernel", "model"]) { + const entry = device[field]; + if ( + typeof entry !== "string" || + entry.length === 0 || + entry.length > MAX_DEVICE_IDENTITY_CHARACTERS || + /[\u0000-\u001f\u007f]/u.test(entry) + ) { + throw new Error(`Jetson device ${field} is invalid`); + } } - return value as number; + return { + model: device.model as string, + jetpackVersion: device.jetpackVersion as string, + jetsonLinuxRelease: device.jetsonLinuxRelease as string, + kernel: device.kernel as string, + }; } -export async function verifyGitHubOidcToken(options: { - token: string; - request: JetsonDispatchRequest; - policy: GitHubOidcPolicy; - resolveSigningKey: SigningKeyResolver; - nowMs?: number; -}): Promise { +export function parseJetsonDispatchStatus(value: unknown): JetsonDispatchStatus { + const status = record(value, "Jetson dispatch status"); + const baseFields = ["cleanup", "createdAt", "jobId", "request", "schemaVersion", "state"]; + const request = parseJetsonDispatchRequest(status.request); if ( - typeof options.token !== "string" || - options.token.length === 0 || - Buffer.byteLength(options.token) > MAX_TOKEN_BYTES + status.schemaVersion !== 1 || + typeof status.jobId !== "string" || + !JOB_ID_PATTERN.test(status.jobId) || + status.jobId !== expectedJobId(request) ) { - throw new Error("OIDC token size is invalid"); - } - if (!POSITIVE_INTEGER_PATTERN.test(options.policy.repositoryId)) { - throw new Error("trusted repository ID must be a positive decimal integer"); + throw new Error("Jetson dispatch status does not match its request and job ID"); } - const parts = options.token.split("."); - if (parts.length !== 3 || parts.some((part) => part.length === 0)) { - throw new Error("OIDC token must be a compact JWT"); - } - const [encodedHeader, encodedClaims, encodedSignature] = parts as [string, string, string]; - const header = decodeJwtPart(encodedHeader, "header"); - if (header.alg !== "RS256" || header.typ !== "JWT") { - throw new Error("OIDC token must use an RS256 JWT header"); - } - const keyId = stringClaim(header, "kid"); - if (keyId.length > 256 || /[\u0000-\u001f\u007f]/u.test(keyId)) { - throw new Error("OIDC kid header is invalid"); + const createdAt = parseTimestamp(status.createdAt, "createdAt"); + if (status.state === "queued") { + requireFields(status, "queued Jetson dispatch status", baseFields); + if (status.cleanup !== "pending") { + throw new Error("queued Jetson dispatch cleanup must be pending"); + } + return { + schemaVersion: 1, + jobId: status.jobId, + request, + state: "queued", + createdAt, + cleanup: "pending", + }; } - - const claims = decodeJwtPart(encodedClaims, "claims"); - if (!/^[A-Za-z0-9_-]+$/u.test(encodedSignature)) { - throw new Error("OIDC signature encoding is invalid"); + if (status.state === "running") { + requireFields(status, "running Jetson dispatch status", [...baseFields, "startedAt"]); + const startedAt = parseTimestamp(status.startedAt, "startedAt"); + if (status.cleanup !== "pending" || startedAt < createdAt) { + throw new Error("running Jetson dispatch status is invalid"); + } + return { + schemaVersion: 1, + jobId: status.jobId, + request, + state: "running", + createdAt, + startedAt, + cleanup: "pending", + }; } - const signature = Buffer.from(encodedSignature, "base64url"); - const verifier = createVerify("RSA-SHA256"); - verifier.update(`${encodedHeader}.${encodedClaims}`, "ascii"); - verifier.end(); - const signingKey = await options.resolveSigningKey(keyId); - if (!verifier.verify(signingKey, signature)) { - throw new Error("OIDC signature verification failed"); + if (status.state !== "completed") { + throw new Error("Jetson dispatch status state is invalid"); } - - const nowSeconds = Math.floor((options.nowMs ?? Date.now()) / 1000); - const issuedAt = numericDateClaim(claims, "iat"); - const notBefore = numericDateClaim(claims, "nbf"); - const expiresAt = numericDateClaim(claims, "exp"); + requireFields( + status, + "completed Jetson dispatch status", + [...baseFields, "completedAt", "conclusion", "startedAt"], + ["device", "error"], + ); if ( - issuedAt > nowSeconds + CLOCK_SKEW_SECONDS || - notBefore > nowSeconds + CLOCK_SKEW_SECONDS || - expiresAt <= nowSeconds - CLOCK_SKEW_SECONDS || - expiresAt <= issuedAt || - expiresAt - issuedAt > MAX_TOKEN_LIFETIME_SECONDS + typeof status.conclusion !== "string" || + !DISPATCH_CONCLUSIONS.includes(status.conclusion) || + typeof status.cleanup !== "string" || + !COMPLETED_CLEANUP_STATES.includes(status.cleanup) || + (status.cleanup === "failed" && status.conclusion !== "cleanup-failed") ) { - throw new Error("OIDC token is outside its allowed validity window"); + throw new Error("completed Jetson dispatch result is invalid"); + } + const startedAt = parseTimestamp(status.startedAt, "startedAt"); + const completedAt = parseTimestamp(status.completedAt, "completedAt"); + if (startedAt < createdAt || completedAt < startedAt) { + throw new Error("completed Jetson dispatch timestamps are invalid"); } - - const workflowSha = stringClaim(claims, "workflow_sha"); - const workflowRunId = positiveIntegerString(claims.run_id, "OIDC run_id claim"); - const workflowRunAttempt = positiveIntegerString(claims.run_attempt, "OIDC run_attempt claim"); if ( - stringClaim(claims, "iss") !== GITHUB_OIDC_ISSUER || - stringClaim(claims, "aud") !== JETSON_DISPATCH_AUDIENCE || - stringClaim(claims, "repository") !== JETSON_DISPATCH_REPOSITORY || - stringClaim(claims, "repository_id") !== options.policy.repositoryId || - stringClaim(claims, "workflow_ref") !== JETSON_DISPATCH_WORKFLOW_REF || - stringClaim(claims, "ref") !== "refs/heads/main" || - stringClaim(claims, "event_name") !== "workflow_dispatch" || - stringClaim(claims, "runner_environment") !== "github-hosted" || - !SHA_PATTERN.test(workflowSha) || - stringClaim(claims, "sha") !== workflowSha || - workflowRunId !== options.request.workflowRunId || - Number(workflowRunAttempt) !== options.request.workflowRunAttempt + status.error !== undefined && + (typeof status.error !== "string" || + status.error.length === 0 || + status.error.length > MAX_ERROR_CHARACTERS || + /[\u0000-\u001f\u007f]/u.test(status.error)) ) { - throw new Error("OIDC claims do not match the trusted Jetson controller"); + throw new Error("completed Jetson dispatch error is invalid"); + } + const device = status.device === undefined ? undefined : parseDeviceIdentity(status.device); + if (status.conclusion === "success" && device === undefined) { + throw new Error("successful Jetson dispatch must include device identity"); } - return { - repository: JETSON_DISPATCH_REPOSITORY, - repositoryId: options.policy.repositoryId, - runId: workflowRunId, - runAttempt: options.request.workflowRunAttempt, - workflowSha, - tokenId: stringClaim(claims, "jti"), + schemaVersion: 1, + jobId: status.jobId, + request, + state: "completed", + conclusion: status.conclusion as JetsonDispatchConclusion, + createdAt, + startedAt, + completedAt, + ...(device === undefined ? {} : { device }), + cleanup: status.cleanup as "failed" | "succeeded", + ...(status.error === undefined ? {} : { error: status.error }), }; } -type FetchLike = typeof fetch; - -export function createGitHubJwksResolver(fetchImpl: FetchLike = fetch): SigningKeyResolver { - let cachedAt = 0; - let cachedKeys = new Map(); - let refreshInFlight: Promise | undefined; - - async function refresh(): Promise { - const response = await fetchImpl(GITHUB_OIDC_JWKS_URL, { - headers: { Accept: "application/json" }, - signal: AbortSignal.timeout(5_000), - }); - if (!response.ok) throw new Error(`GitHub OIDC JWKS returned HTTP ${response.status}`); - const payload = record(await response.json(), "GitHub OIDC JWKS response"); - if (!Array.isArray(payload.keys) || payload.keys.length === 0 || payload.keys.length > 20) { - throw new Error("GitHub OIDC JWKS response has an invalid key set"); - } - const nextKeys = new Map(); - for (const value of payload.keys) { - const key = record(value, "GitHub OIDC JWK"); - if ( - key.kty !== "RSA" || - key.use !== "sig" || - key.alg !== "RS256" || - typeof key.kid !== "string" || - key.kid.length === 0 || - typeof key.n !== "string" || - typeof key.e !== "string" - ) { - continue; - } - nextKeys.set(key.kid, createPublicKey({ key, format: "jwk" } as JsonWebKeyInput)); - } - if (nextKeys.size === 0) throw new Error("GitHub OIDC JWKS contains no RS256 signing key"); - cachedKeys = nextKeys; - cachedAt = Date.now(); +export function parseJetsonDispatchStatusResponse( + value: unknown, + expectedRequest?: JetsonDispatchRequest, +): JetsonDispatchStatus { + const response = record(value, "Jetson dispatcher response"); + requireFields(response, "Jetson dispatcher response", ["job"]); + const status = parseJetsonDispatchStatus(response.job); + if (expectedRequest !== undefined && status.jobId !== expectedJobId(expectedRequest)) { + throw new Error("Jetson dispatcher response does not match the submitted request"); } + return status; +} - async function refreshOnce(): Promise { - refreshInFlight ??= refresh().finally(() => { - refreshInFlight = undefined; - }); - await refreshInFlight; +export function decodeJetsonArtifactArchive(value: unknown): Buffer { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value) + ) { + throw new Error("Jetson artifact archive must be bounded canonical base64"); + } + const decoded = Buffer.from(value, "base64"); + if (decoded.length > MAX_JETSON_ARTIFACT_ARCHIVE_BYTES || decoded.toString("base64") !== value) { + throw new Error("Jetson artifact archive must be bounded canonical base64"); } + return decoded; +} - return async (keyId) => { - const now = Date.now(); - const refreshedExpiredCache = cachedAt === 0 || now - cachedAt > JWKS_CACHE_MS; - if (refreshedExpiredCache) await refreshOnce(); - let key = cachedKeys.get(keyId); - if (!key && !refreshedExpiredCache && now - cachedAt > UNKNOWN_KEY_REFRESH_INTERVAL_MS) { - await refreshOnce(); - key = cachedKeys.get(keyId); - } - if (!key) throw new Error("OIDC signing key is unknown"); - return key; +export function parseJetsonDispatchArtifact( + value: unknown, + expectedArtifactJobId: string, +): JetsonDispatchArtifact { + const artifact = record(value, "Jetson dispatch artifact"); + requireFields(artifact, "Jetson dispatch artifact", ["log", "status"], ["artifactArchiveBase64"]); + const status = parseJetsonDispatchStatus(artifact.status); + if (status.jobId !== expectedArtifactJobId || status.state !== "completed") { + throw new Error("Jetson dispatch artifact does not match its completed job"); + } + if ( + typeof artifact.log !== "string" || + Buffer.byteLength(artifact.log) > MAX_JETSON_DISPATCH_LOG_BYTES + ) { + throw new Error("Jetson dispatch artifact log is invalid"); + } + if (artifact.artifactArchiveBase64 !== undefined) { + decodeJetsonArtifactArchive(artifact.artifactArchiveBase64); + } + if (status.conclusion === "success" && artifact.artifactArchiveBase64 === undefined) { + throw new Error("successful Jetson dispatch must include its artifact archive"); + } + return { + status, + log: artifact.log, + ...(artifact.artifactArchiveBase64 === undefined + ? {} + : { artifactArchiveBase64: artifact.artifactArchiveBase64 as string }), }; } diff --git a/tools/e2e/jetson-dispatch-lifecycle.mts b/tools/e2e/jetson-dispatch-lifecycle.mts deleted file mode 100644 index 35cd03a0a79..00000000000 --- a/tools/e2e/jetson-dispatch-lifecycle.mts +++ /dev/null @@ -1,657 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; -import { - type GitHubOidcIdentity, - type JetsonDispatchRequest, - parseJetsonDispatchRequest, -} from "./jetson-dispatch-contract.mts"; -import { - createPrivateRegularFile, - readPrivateRegularFile, - writePrivateRegularFile, -} from "./private-file.mts"; - -export const MAX_JETSON_DISPATCH_LOG_BYTES = 4 * 1024 * 1024; -const MAX_RETAINED_JOB_STATUSES = 128; -const MAX_JETSON_DISPATCH_STATUS_BYTES = 16 * 1024; -const JOB_ID_PATTERN = /^[a-f0-9]{64}$/u; -const STATUS_FILE_PATTERN = /^([a-f0-9]{64})\.json$/u; -export const MAX_JETSON_ARTIFACT_ARCHIVE_BYTES = 1024 * 1024; -const MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS = - Math.ceil(MAX_JETSON_ARTIFACT_ARCHIVE_BYTES / 3) * 4; -const MAX_JETSON_DISPATCH_ARTIFACT_JSON_OVERHEAD_BYTES = 256 * 1024; -export const MAX_JETSON_DISPATCH_ARTIFACT_RESPONSE_BYTES = - MAX_JETSON_DISPATCH_LOG_BYTES * 6 + - MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS + - MAX_JETSON_DISPATCH_ARTIFACT_JSON_OVERHEAD_BYTES; - -export type JetsonDispatchConclusion = - | "cancelled" - | "cleanup-failed" - | "failure" - | "success" - | "timed-out"; - -export interface JetsonDeviceIdentity { - model: string; - jetpackVersion: string; - jetsonLinuxRelease: string; - kernel: string; -} - -export interface JetsonWorkerResult { - device: JetsonDeviceIdentity; - log: string; - artifactArchiveBase64?: string; -} - -export interface JetsonDispatchWorker { - run( - request: JetsonDispatchRequest, - options: { jobId: string; signal: AbortSignal }, - ): Promise; - cleanup(options: { jobId: string; signal: AbortSignal }): Promise; -} - -export interface JetsonDispatchStatus { - schemaVersion: 1; - jobId: string; - request: JetsonDispatchRequest; - state: "queued" | "running" | "completed"; - conclusion?: JetsonDispatchConclusion; - createdAt: string; - startedAt?: string; - completedAt?: string; - device?: JetsonDeviceIdentity; - cleanup: "pending" | "succeeded" | "failed"; - error?: string; -} - -export interface JetsonDispatchArtifact { - status: JetsonDispatchStatus; - log: string; - artifactArchiveBase64?: string; -} - -export class JetsonDispatchBusyError extends Error {} - -export class JetsonDispatchNotFoundError extends Error {} - -function safeError(error: unknown): string { - const message = error instanceof Error ? error.message : "Jetson dispatch failed"; - return message.replace(/[\u0000-\u001f\u007f]/gu, " ").slice(0, 500); -} - -function appendBoundedError(previous: string | undefined, newest: string): string { - const latest = newest.replace(/[\u0000-\u001f\u007f]/gu, " ").slice(0, 500); - if (!previous) return latest; - const separator = "; "; - const available = 500 - separator.length - latest.length; - if (available <= 0) return latest; - const earlier = previous.replace(/[\u0000-\u001f\u007f]/gu, " "); - const boundedEarlier = - earlier.length <= available - ? earlier - : available > 3 - ? `${earlier.slice(0, available - 3)}...` - : earlier.slice(0, available); - return `${boundedEarlier}${separator}${latest}`; -} - -function workerErrorLog(error: unknown): string | undefined { - if (!error || typeof error !== "object" || !("log" in error)) return undefined; - const log = (error as { log?: unknown }).log; - if (typeof log !== "string" || Buffer.byteLength(log) > MAX_JETSON_DISPATCH_LOG_BYTES) { - return undefined; - } - return log; -} - -export function decodeJetsonArtifactArchive(value: unknown): Buffer { - if ( - typeof value !== "string" || - value.length === 0 || - value.length > MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS || - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value) - ) { - throw new Error("Jetson artifact archive must be bounded canonical base64"); - } - const decoded = Buffer.from(value, "base64"); - if (decoded.length > MAX_JETSON_ARTIFACT_ARCHIVE_BYTES || decoded.toString("base64") !== value) { - throw new Error("Jetson artifact archive must be bounded canonical base64"); - } - return decoded; -} - -function workerErrorArtifactArchive(error: unknown): string | undefined { - if (!error || typeof error !== "object" || !("artifactArchiveBase64" in error)) { - return undefined; - } - const value = (error as { artifactArchiveBase64?: unknown }).artifactArchiveBase64; - try { - decodeJetsonArtifactArchive(value); - return value as string; - } catch { - return undefined; - } -} - -function ensurePrivateDirectory(directory: string): void { - fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); - const stat = fs.lstatSync(directory); - if (!stat.isDirectory() || stat.isSymbolicLink()) { - throw new Error(`${directory} must be a private directory`); - } - const currentUid = process.getuid?.(); - if (currentUid !== undefined && stat.uid !== currentUid) { - throw new Error(`${directory} must be owned by the dispatcher user`); - } - fs.chmodSync(directory, 0o700); -} - -function fsyncDirectory(directory: string): void { - const descriptor = fs.openSync( - directory, - fs.constants.O_RDONLY | (fs.constants.O_DIRECTORY ?? 0) | (fs.constants.O_NOFOLLOW ?? 0), - ); - try { - if (!fs.fstatSync(descriptor).isDirectory()) { - throw new Error(`${directory} must be a directory`); - } - fs.fsyncSync(descriptor); - } finally { - fs.closeSync(descriptor); - } -} - -function dispatchJobId(request: JetsonDispatchRequest): string { - return createHash("sha256") - .update( - `${request.schemaVersion}:${request.target}:${request.candidateSha}:${request.workflowRunId}:${request.workflowRunAttempt}`, - "utf8", - ) - .digest("hex"); -} - -function parseTimestamp(value: unknown, name: string): string { - if ( - typeof value !== "string" || - value.length !== 24 || - Number.isNaN(Date.parse(value)) || - new Date(value).toISOString() !== value - ) { - throw new Error(`${name} must be an ISO timestamp`); - } - return value; -} - -function parseDeviceIdentity(value: unknown): JetsonDeviceIdentity | undefined { - if (value === undefined) return undefined; - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("device must be an object"); - } - const device = value as Record; - const fields = ["jetpackVersion", "jetsonLinuxRelease", "kernel", "model"]; - if (fields.some((field) => typeof device[field] !== "string")) { - throw new Error("device fields are invalid"); - } - return { - model: device.model as string, - jetpackVersion: device.jetpackVersion as string, - jetsonLinuxRelease: device.jetsonLinuxRelease as string, - kernel: device.kernel as string, - }; -} - -function parsePersistedStatus(contents: string, expectedJobId: string): JetsonDispatchStatus { - let value: unknown; - try { - value = JSON.parse(contents); - } catch { - throw new Error("persisted Jetson dispatch status is not valid JSON"); - } - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("persisted Jetson dispatch status must be an object"); - } - const status = value as Record; - const request = parseJetsonDispatchRequest(status.request); - if ( - status.schemaVersion !== 1 || - status.jobId !== expectedJobId || - dispatchJobId(request) !== expectedJobId - ) { - throw new Error("persisted Jetson dispatch status does not match its job ID"); - } - const createdAt = parseTimestamp(status.createdAt, "createdAt"); - if (status.state === "queued" || status.state === "running") { - if (status.cleanup !== "pending") { - throw new Error("persisted incomplete Jetson dispatch cleanup is invalid"); - } - return { - schemaVersion: 1, - jobId: expectedJobId, - request, - state: status.state, - createdAt, - ...(status.state === "running" - ? { startedAt: parseTimestamp(status.startedAt, "startedAt") } - : {}), - cleanup: "pending", - }; - } - if (status.state !== "completed") { - throw new Error("persisted Jetson dispatch status has an invalid state"); - } - if ( - !["cancelled", "cleanup-failed", "failure", "success", "timed-out"].includes( - status.conclusion as string, - ) || - !["failed", "succeeded"].includes(status.cleanup as string) || - (status.error !== undefined && typeof status.error !== "string") - ) { - throw new Error("persisted Jetson dispatch result fields are invalid"); - } - const startedAt = parseTimestamp(status.startedAt, "startedAt"); - const completedAt = parseTimestamp(status.completedAt, "completedAt"); - const device = parseDeviceIdentity(status.device); - return { - schemaVersion: 1, - jobId: expectedJobId, - request, - state: "completed", - conclusion: status.conclusion as JetsonDispatchConclusion, - createdAt, - startedAt, - completedAt, - ...(device === undefined ? {} : { device }), - cleanup: status.cleanup as "failed" | "succeeded", - ...(status.error === undefined ? {} : { error: status.error as string }), - }; -} - -function withTimeout( - timeoutMs: number, - operation: (signal: AbortSignal) => Promise, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(new Error("operation timed out")), timeoutMs); - timer.unref(); - return operation(controller.signal).finally(() => clearTimeout(timer)); -} - -export class JetsonDispatchCoordinator { - readonly #stateDirectory: string; - readonly #lockPath: string; - readonly #worker: JetsonDispatchWorker; - readonly #executionTimeoutMs: number; - readonly #cleanupTimeoutMs: number; - readonly #jobs = new Map(); - #activeJobId: string | undefined; - #activeAbort: AbortController | undefined; - #activeRun: Promise | undefined; - #cancelRequested = false; - #initialized = false; - #lockRecoveryRequired = false; - - constructor(options: { - stateDirectory: string; - worker: JetsonDispatchWorker; - executionTimeoutMs: number; - cleanupTimeoutMs: number; - }) { - if (!path.isAbsolute(options.stateDirectory)) { - throw new Error("Jetson dispatcher state directory must be absolute"); - } - if ( - !Number.isSafeInteger(options.executionTimeoutMs) || - options.executionTimeoutMs < 60_000 || - options.executionTimeoutMs > 55 * 60_000 - ) { - throw new Error("Jetson execution timeout must be between 1 and 55 minutes"); - } - if ( - !Number.isSafeInteger(options.cleanupTimeoutMs) || - options.cleanupTimeoutMs < 10_000 || - options.cleanupTimeoutMs > 10 * 60_000 - ) { - throw new Error("Jetson cleanup timeout must be between 10 seconds and 10 minutes"); - } - this.#stateDirectory = options.stateDirectory; - this.#lockPath = path.join(options.stateDirectory, "device.lock"); - this.#worker = options.worker; - this.#executionTimeoutMs = options.executionTimeoutMs; - this.#cleanupTimeoutMs = options.cleanupTimeoutMs; - } - - async initialize(): Promise { - if (this.#initialized) return; - ensurePrivateDirectory(this.#stateDirectory); - const staleLock = readPrivateRegularFile(this.#lockPath, { - allowMissing: true, - maxBytes: 1024, - }); - if (staleLock !== null) { - if (!/^[a-f0-9]{64}\n?$/u.test(staleLock)) { - throw new Error("Jetson device lock contains an invalid job ID"); - } - const jobId = staleLock.endsWith("\n") ? staleLock.slice(0, -1) : staleLock; - await withTimeout(this.#cleanupTimeoutMs, (signal) => - this.#worker.cleanup({ jobId, signal }), - ); - const recoveredStatus = this.#readStatus(jobId); - if (recoveredStatus && recoveredStatus.state !== "completed") { - recoveredStatus.state = "completed"; - recoveredStatus.conclusion = "failure"; - recoveredStatus.startedAt ??= recoveredStatus.createdAt; - recoveredStatus.completedAt = new Date().toISOString(); - recoveredStatus.cleanup = "succeeded"; - recoveredStatus.error = "Jetson dispatcher restarted before terminal status was persisted"; - this.#persist(recoveredStatus); - } - this.#removeDeviceLock(jobId); - } - this.#restoreCompletedStatuses(); - this.#initialized = true; - } - - dispatch(request: JetsonDispatchRequest, identity: GitHubOidcIdentity): JetsonDispatchStatus { - this.#requireInitialized(); - if (this.#lockRecoveryRequired) { - throw new JetsonDispatchBusyError("Jetson device lock requires recovery"); - } - if ( - identity.runId !== request.workflowRunId || - identity.runAttempt !== request.workflowRunAttempt - ) { - throw new Error("dispatch identity does not match the workflow run"); - } - const jobId = dispatchJobId(request); - const existing = this.#findStatus(jobId); - if (existing) return structuredClone(existing); - if (this.#activeJobId) { - throw new JetsonDispatchBusyError("Jetson device is already running another job"); - } - this.#evictOldestCompletedJob(); - - try { - createPrivateRegularFile(this.#lockPath, `${jobId}\n`); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "EEXIST") { - throw new JetsonDispatchBusyError("Jetson device lock requires recovery"); - } - throw error; - } - const status: JetsonDispatchStatus = { - schemaVersion: 1, - jobId, - request: structuredClone(request), - state: "queued", - createdAt: new Date().toISOString(), - cleanup: "pending", - }; - this.#clearPriorResultFiles(jobId); - this.#persist(status); - fsyncDirectory(this.#stateDirectory); - this.#jobs.set(jobId, status); - this.#activeJobId = jobId; - this.#activeRun = this.#run(status); - return structuredClone(status); - } - - status(jobId: string): JetsonDispatchStatus { - this.#requireInitialized(); - const status = this.#findStatus(jobId); - if (!status) throw new JetsonDispatchNotFoundError("Jetson dispatch job was not found"); - return structuredClone(status); - } - - request(jobId: string): JetsonDispatchRequest { - return this.status(jobId).request; - } - - cancel(jobId: string): JetsonDispatchStatus { - this.#requireInitialized(); - const status = this.#findStatus(jobId); - if (!status) throw new JetsonDispatchNotFoundError("Jetson dispatch job was not found"); - if (status.state !== "completed" && this.#activeJobId === jobId) { - this.#cancelRequested = true; - this.#activeAbort?.abort(new Error("dispatch cancelled")); - } - return structuredClone(status); - } - - async shutdown(): Promise { - if (this.#activeJobId) this.cancel(this.#activeJobId); - await this.#activeRun; - if (this.#lockRecoveryRequired) { - throw new Error("Jetson device lock state requires operator recovery"); - } - if ( - readPrivateRegularFile(this.#lockPath, { - allowMissing: true, - maxBytes: 1024, - }) !== null - ) { - throw new Error("Jetson device lock still requires cleanup recovery"); - } - } - - artifact(jobId: string): JetsonDispatchArtifact { - const status = this.status(jobId); - if (status.state !== "completed") - throw new JetsonDispatchBusyError("Jetson job is not complete"); - const log = - readPrivateRegularFile(this.#logPath(jobId), { - allowMissing: true, - maxBytes: MAX_JETSON_DISPATCH_LOG_BYTES, - }) ?? ""; - const artifactArchiveBase64 = readPrivateRegularFile(this.#artifactPath(jobId), { - allowMissing: true, - maxBytes: MAX_JETSON_ARTIFACT_ARCHIVE_BASE64_CHARACTERS, - }); - if (artifactArchiveBase64 !== null) decodeJetsonArtifactArchive(artifactArchiveBase64); - return { - status, - log, - ...(artifactArchiveBase64 === null ? {} : { artifactArchiveBase64 }), - }; - } - - async #run(status: JetsonDispatchStatus): Promise { - const controller = new AbortController(); - this.#activeAbort = controller; - this.#cancelRequested = false; - let timedOut = false; - let conclusion: JetsonDispatchConclusion = "failure"; - const timer = setTimeout(() => { - timedOut = true; - controller.abort(new Error("Jetson execution timed out")); - }, this.#executionTimeoutMs); - timer.unref(); - - try { - status.state = "running"; - status.startedAt = new Date().toISOString(); - this.#persist(status); - const result = await this.#worker.run(status.request, { - jobId: status.jobId, - signal: controller.signal, - }); - if (Buffer.byteLength(result.log) > MAX_JETSON_DISPATCH_LOG_BYTES) { - throw new Error(`Jetson log exceeds ${MAX_JETSON_DISPATCH_LOG_BYTES} bytes`); - } - writePrivateRegularFile(this.#logPath(status.jobId), result.log); - if (result.artifactArchiveBase64 !== undefined) { - decodeJetsonArtifactArchive(result.artifactArchiveBase64); - writePrivateRegularFile(this.#artifactPath(status.jobId), result.artifactArchiveBase64); - } - status.device = result.device; - conclusion = "success"; - } catch (error) { - status.error = safeError(error); - conclusion = timedOut ? "timed-out" : this.#cancelRequested ? "cancelled" : "failure"; - try { - writePrivateRegularFile(this.#logPath(status.jobId), workerErrorLog(error) ?? ""); - const artifactArchiveBase64 = workerErrorArtifactArchive(error); - if (artifactArchiveBase64 !== undefined) { - writePrivateRegularFile(this.#artifactPath(status.jobId), artifactArchiveBase64); - } - } catch (resultError) { - status.error = appendBoundedError( - status.error, - `Result persistence failed: ${safeError(resultError)}`, - ); - } - } finally { - clearTimeout(timer); - try { - await withTimeout(this.#cleanupTimeoutMs, (signal) => - this.#worker.cleanup({ jobId: status.jobId, signal }), - ); - status.cleanup = "succeeded"; - } catch (error) { - status.cleanup = "failed"; - const cleanupError = `Jetson cleanup failed: ${safeError(error)}`; - status.error = appendBoundedError(status.error, cleanupError); - conclusion = "cleanup-failed"; - } - status.conclusion = conclusion; - status.state = "completed"; - status.completedAt = new Date().toISOString(); - let terminalStatusPersisted = false; - try { - this.#persist(status); - terminalStatusPersisted = true; - } catch (error) { - const persistenceError = `Final status persistence failed: ${safeError(error)}`; - status.error = appendBoundedError(status.error, persistenceError); - if (status.conclusion !== "cleanup-failed") status.conclusion = "failure"; - } - if (status.cleanup === "succeeded" && terminalStatusPersisted) { - try { - this.#removeDeviceLock(status.jobId); - } catch (error) { - const lockError = `Jetson lock removal failed: ${safeError(error)}`; - status.error = appendBoundedError(status.error, lockError); - status.conclusion = "cleanup-failed"; - try { - this.#persist(status); - } catch (persistError) { - const persistenceError = `Lock failure persistence failed: ${safeError(persistError)}`; - status.error = appendBoundedError(status.error, persistenceError); - } - } - } - this.#activeAbort = undefined; - this.#activeRun = undefined; - this.#activeJobId = undefined; - this.#cancelRequested = false; - } - } - - #persist(status: JetsonDispatchStatus): void { - writePrivateRegularFile(this.#statusPath(status.jobId), `${JSON.stringify(status, null, 2)}\n`); - } - - #removeDeviceLock(jobId: string): void { - fs.unlinkSync(this.#lockPath); - try { - fsyncDirectory(this.#stateDirectory); - } catch (error) { - try { - this.#restoreDeviceLock(jobId); - } catch (restoreError) { - this.#lockRecoveryRequired = true; - throw new Error( - `Jetson lock directory persistence failed: ${safeError(error)}; lock restoration failed: ${safeError(restoreError)}`, - ); - } - throw error; - } - } - - #restoreDeviceLock(jobId: string): void { - try { - createPrivateRegularFile(this.#lockPath, `${jobId}\n`); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const existing = readPrivateRegularFile(this.#lockPath, { maxBytes: 1024 }); - if (existing !== `${jobId}\n`) { - throw new Error("Jetson device lock changed while durability was restored"); - } - } - fsyncDirectory(this.#stateDirectory); - } - - #statusPath(jobId: string): string { - return path.join(this.#stateDirectory, `${jobId}.json`); - } - - #logPath(jobId: string): string { - return path.join(this.#stateDirectory, `${jobId}.log`); - } - - #artifactPath(jobId: string): string { - return path.join(this.#stateDirectory, `${jobId}.artifacts.tar.gz.b64`); - } - - #clearPriorResultFiles(jobId: string): void { - fs.rmSync(this.#logPath(jobId), { force: true }); - fs.rmSync(this.#artifactPath(jobId), { force: true }); - } - - #findStatus(jobId: string): JetsonDispatchStatus | undefined { - const existing = this.#jobs.get(jobId); - if (existing || !JOB_ID_PATTERN.test(jobId)) return existing; - const restored = this.#readStatus(jobId); - if (restored?.state === "completed") { - this.#evictOldestCompletedJob(); - this.#jobs.set(jobId, restored); - } - return restored?.state === "completed" ? restored : undefined; - } - - #readStatus(jobId: string): JetsonDispatchStatus | null { - try { - const contents = readPrivateRegularFile(this.#statusPath(jobId), { - allowMissing: true, - maxBytes: MAX_JETSON_DISPATCH_STATUS_BYTES, - }); - return contents === null ? null : parsePersistedStatus(contents, jobId); - } catch (error) { - throw new Error(`Jetson dispatch status ${jobId}.json is invalid: ${safeError(error)}`); - } - } - - #restoreCompletedStatuses(): void { - const completed = fs - .readdirSync(this.#stateDirectory) - .flatMap((name) => { - const match = STATUS_FILE_PATTERN.exec(name); - if (!match) return []; - const status = this.#readStatus(match[1]!); - return status?.state === "completed" ? [status] : []; - }) - .sort((left, right) => left.completedAt!.localeCompare(right.completedAt!)) - .slice(-MAX_RETAINED_JOB_STATUSES); - for (const status of completed) this.#jobs.set(status.jobId, status); - } - - #evictOldestCompletedJob(): void { - if (this.#jobs.size < MAX_RETAINED_JOB_STATUSES) return; - for (const [jobId, status] of this.#jobs) { - if (status.state === "completed") { - this.#jobs.delete(jobId); - return; - } - } - } - - #requireInitialized(): void { - if (!this.#initialized) throw new Error("Jetson dispatcher is not initialized"); - } -} diff --git a/tools/e2e/jetson-dispatch-service.mts b/tools/e2e/jetson-dispatch-service.mts deleted file mode 100644 index 738a71fc126..00000000000 --- a/tools/e2e/jetson-dispatch-service.mts +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import { - createGitHubJwksResolver, - type GitHubOidcIdentity, - parseJetsonDispatchRequest, - type SigningKeyResolver, - verifyGitHubOidcToken, -} from "./jetson-dispatch-contract.mts"; -import { - JetsonDispatchBusyError, - JetsonDispatchCoordinator, - JetsonDispatchNotFoundError, -} from "./jetson-dispatch-lifecycle.mts"; -import { loadSshJetsonWorkerConfig, SshJetsonDispatchWorker } from "./jetson-dispatch-worker.mts"; - -const MAX_REQUEST_BYTES = 8 * 1024; -const JOB_PATH_PATTERN = /^\/v1\/jobs\/([a-f0-9]{64})(?:\/(artifact))?$/u; - -function positiveIntegerEnvironment( - value: string | undefined, - name: string, - minimum: number, - maximum: number, -): number { - if (!value || !/^[1-9][0-9]*$/u.test(value)) { - throw new Error(`${name} must be a positive integer`); - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new Error(`${name} must be between ${minimum} and ${maximum}`); - } - return parsed; -} - -function bearerToken(request: IncomingMessage): string { - const authorization = request.headers.authorization; - const match = - typeof authorization === "string" ? /^Bearer ([A-Za-z0-9._-]+)$/u.exec(authorization) : null; - if (!match) throw new Error("request requires a bearer token"); - return match[1]!; -} - -async function readJsonBody(request: IncomingMessage): Promise { - if (request.headers["content-type"]?.split(";", 1)[0]?.trim() !== "application/json") { - throw new Error("request Content-Type must be application/json"); - } - const chunks: Buffer[] = []; - let bytes = 0; - for await (const chunk of request) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - bytes += buffer.length; - if (bytes > MAX_REQUEST_BYTES) throw new Error("request body is too large"); - chunks.push(buffer); - } - if (bytes === 0) throw new Error("request body is required"); - try { - return JSON.parse(Buffer.concat(chunks).toString("utf8")); - } catch { - throw new Error("request body must be valid JSON"); - } -} - -function sendJson(response: ServerResponse, status: number, value: unknown): void { - const body = `${JSON.stringify(value)}\n`; - response.writeHead(status, { - "Cache-Control": "no-store", - "Content-Length": Buffer.byteLength(body), - "Content-Type": "application/json; charset=utf-8", - "X-Content-Type-Options": "nosniff", - }); - response.end(body); -} - -export function createJetsonDispatchServer(options: { - coordinator: JetsonDispatchCoordinator; - repositoryId: string; - resolveSigningKey: SigningKeyResolver; -}): Server { - async function authenticate( - token: string, - dispatchRequest: ReturnType, - ): Promise { - return verifyGitHubOidcToken({ - token, - request: dispatchRequest, - policy: { repositoryId: options.repositoryId }, - resolveSigningKey: options.resolveSigningKey, - }); - } - - return createServer((request, response) => { - void (async () => { - if (!request.url) { - sendJson(response, 404, { error: "not found" }); - return; - } - let token: string; - try { - token = bearerToken(request); - } catch { - sendJson(response, 401, { error: "request authentication failed" }); - return; - } - const createJob = request.method === "POST" && request.url === "/v1/jobs"; - const match = JOB_PATH_PATTERN.exec(request.url); - if (!createJob && (!match || !["DELETE", "GET"].includes(request.method ?? ""))) { - sendJson(response, 404, { error: "not found" }); - return; - } - - let dispatchRequest; - let jobId: string | undefined; - let artifact = false; - if (createJob) { - try { - dispatchRequest = parseJetsonDispatchRequest(await readJsonBody(request)); - } catch (error) { - sendJson(response, 400, { - error: error instanceof Error ? error.message : "invalid request", - }); - return; - } - } else { - jobId = match![1]!; - artifact = match![2] === "artifact"; - try { - dispatchRequest = options.coordinator.request(jobId); - } catch { - sendJson(response, 401, { error: "request authentication failed" }); - return; - } - } - - let identity; - try { - identity = await authenticate(token, dispatchRequest); - } catch { - sendJson(response, 401, { error: "request authentication failed" }); - return; - } - if (createJob) { - sendJson(response, 202, { job: options.coordinator.dispatch(dispatchRequest, identity) }); - return; - } - - const existingJobId = jobId!; - if (request.method === "DELETE") { - if (artifact) { - sendJson(response, 405, { error: "method not allowed" }); - return; - } - sendJson(response, 202, { job: options.coordinator.cancel(existingJobId) }); - return; - } - sendJson( - response, - 200, - artifact - ? options.coordinator.artifact(existingJobId) - : { job: options.coordinator.status(existingJobId) }, - ); - })().catch((error) => { - if (response.headersSent) { - response.destroy(); - return; - } - if (error instanceof JetsonDispatchBusyError) { - sendJson(response, 409, { error: error.message }); - return; - } - if (error instanceof JetsonDispatchNotFoundError) { - sendJson(response, 404, { error: error.message }); - return; - } - console.error("Jetson dispatch service failed"); - sendJson(response, 500, { error: "Jetson dispatch service failed" }); - }); - }); -} - -async function main(): Promise { - const stateDirectory = process.env.JETSON_DISPATCH_STATE_DIRECTORY ?? ""; - if (!path.isAbsolute(stateDirectory)) { - throw new Error("JETSON_DISPATCH_STATE_DIRECTORY must be absolute"); - } - const repositoryId = process.env.JETSON_DISPATCH_GITHUB_REPOSITORY_ID ?? ""; - if (!/^[1-9][0-9]*$/u.test(repositoryId)) { - throw new Error("JETSON_DISPATCH_GITHUB_REPOSITORY_ID must be a positive integer"); - } - const executionTimeoutSeconds = positiveIntegerEnvironment( - process.env.JETSON_DISPATCH_EXECUTION_TIMEOUT_SECONDS, - "JETSON_DISPATCH_EXECUTION_TIMEOUT_SECONDS", - 120, - 55 * 60, - ); - const cleanupTimeoutSeconds = positiveIntegerEnvironment( - process.env.JETSON_DISPATCH_CLEANUP_TIMEOUT_SECONDS, - "JETSON_DISPATCH_CLEANUP_TIMEOUT_SECONDS", - 10, - 10 * 60, - ); - const workerConfig = loadSshJetsonWorkerConfig({ stateDirectory }); - if (executionTimeoutSeconds < workerConfig.testTimeoutSeconds + 30) { - throw new Error("Jetson execution timeout must exceed the remote test timeout by 30 seconds"); - } - const coordinator = new JetsonDispatchCoordinator({ - stateDirectory, - worker: new SshJetsonDispatchWorker(workerConfig), - executionTimeoutMs: executionTimeoutSeconds * 1_000, - cleanupTimeoutMs: cleanupTimeoutSeconds * 1_000, - }); - await coordinator.initialize(); - const server = createJetsonDispatchServer({ - coordinator, - repositoryId, - resolveSigningKey: createGitHubJwksResolver(), - }); - server.headersTimeout = 10_000; - server.requestTimeout = 15_000; - server.keepAliveTimeout = 5_000; - const port = positiveIntegerEnvironment( - process.env.JETSON_DISPATCH_PORT ?? "8787", - "JETSON_DISPATCH_PORT", - 1, - 65_535, - ); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(port, "127.0.0.1", resolve); - }); - console.log(`Jetson dispatch service listening on 127.0.0.1:${port}`); - - let stopping = false; - const stop = (): void => { - if (stopping) return; - stopping = true; - void new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }) - .then(() => coordinator.shutdown()) - .catch((error) => { - console.error(error instanceof Error ? error.message : "Jetson dispatch shutdown failed"); - process.exitCode = 1; - }); - }; - process.on("SIGINT", stop); - process.on("SIGTERM", stop); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - console.error(error instanceof Error ? error.message : "Jetson dispatch service failed"); - process.exitCode = 1; - }); -} diff --git a/tools/e2e/jetson-dispatch-worker.mts b/tools/e2e/jetson-dispatch-worker.mts deleted file mode 100644 index d0c517bc556..00000000000 --- a/tools/e2e/jetson-dispatch-worker.mts +++ /dev/null @@ -1,1056 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawn } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; - -import type { JetsonDispatchRequest } from "./jetson-dispatch-contract.mts"; -import type { - JetsonDeviceIdentity, - JetsonDispatchWorker, - JetsonWorkerResult, -} from "./jetson-dispatch-lifecycle.mts"; -import { - decodeJetsonArtifactArchive, - MAX_JETSON_ARTIFACT_ARCHIVE_BYTES, -} from "./jetson-dispatch-lifecycle.mts"; -import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.mts"; - -const MAX_PROCESS_OUTPUT_BYTES = 4 * 1024 * 1024; -const JETSON_SSH_DESTINATION = "nvidia@192.168.55.1"; -const JETSON_CLEANUP_EXECUTABLE = "/usr/local/libexec/nemoclaw-jetson-cleanup"; -const JETSON_CLEANUP_TARGET = - "/opt/nemoclaw-jetson-dispatch/current/tools/e2e/jetson-dispatch-cleanup.sh"; -const SAFE_PROCESS_ENV = { - LANG: "C.UTF-8", - PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", -}; -const MAX_BASELINE_OUTPUT_BYTES = 4 * 1024; -const MAX_BASELINE_RECORD_BYTES = 8 * 1024; -const MAX_OLLAMA_MODELS = 64; -const MAX_OLLAMA_MODELS_BYTES = 3 * 1024; -const MAX_CLEANUP_EVIDENCE_BYTES = 64 * 1024; -const CLEANUP_EVIDENCE_BEGIN = "nemoclaw-cleanup-evidence-v1-begin"; -const CLEANUP_EVIDENCE_END = "nemoclaw-cleanup-evidence-v1-end"; - -interface JetsonCleanupEvidence { - schemaVersion: 1; - volumes: string[]; - processIds: number[]; -} - -const DEVICE_IDENTITY_SCRIPT = String.raw`set -eu -clean_line() { tr -d '\000\r\n\t'; } -printf 'model\t' -if [ -r /proc/device-tree/model ]; then clean_line /dev/null 2>&1; then - jetpack_version="$(dpkg-query -W -f='${"${Version}"}' nvidia-jetpack 2>/dev/null || true)" - if [ -n "$jetpack_version" ]; then printf '%s' "$jetpack_version" | clean_line; else printf 'unavailable'; fi -else - printf 'unavailable' -fi -printf '\njetsonLinuxRelease\t' -if [ -r /etc/nv_tegra_release ]; then sed -n '1p' /etc/nv_tegra_release | clean_line; else printf 'unavailable'; fi -printf '\nkernel\t' -uname -r | clean_line -printf '\n' -`; - -const PRESERVED_BASELINE_SCRIPT = String.raw`set -euo pipefail -export LC_ALL=C -clean_line() { tr -d '\000\r\n\t'; } -node_path="$(command -v node)" -npm_path="$(command -v npm)" -ollama_path="$(command -v ollama)" -for openshell_component in openshell openshell-gateway openshell-sandbox; do - if command -v "$openshell_component" >/dev/null 2>&1; then - echo "Jetson dispatch requires OpenShell to be absent before the job" >&2 - exit 1 - fi - for host_bin in "/usr/local/bin/$openshell_component" "/usr/bin/$openshell_component" "$HOME/.local/bin/$openshell_component"; do - if [ -e "$host_bin" ] || [ -L "$host_bin" ]; then - echo "Jetson dispatch requires host-level OpenShell binaries to be absent" >&2 - exit 1 - fi - done -done -node_version="$(node --version)" -node -e ' - const [major, minor] = process.versions.node.split(".").map(Number); - if (major < 22 || (major === 22 && minor < 19)) process.exit(1); -' -npm_version="$(npm --version)" -npm_major="${"${npm_version%%.*}"}" -[[ "$npm_major" =~ ^[0-9]+$ ]] && (( npm_major >= 10 )) -test -c /dev/nvmap -docker_runtimes="$(docker info --format '{{json .Runtimes}}')" -case "$docker_runtimes" in - *nvidia*) ;; - *) echo "Docker does not report the NVIDIA runtime" >&2; exit 1 ;; -esac -ollama_models_base64="$( - ollama list | - awk 'NR == 1 { if ($1 != "NAME" || $2 != "ID") exit 1; next } { if (NF < 2) exit 1; print $1 "\t" $2 } END { if (NR == 0) exit 1 }' | - LC_ALL=C sort | - base64 --wrap=0 -)" -test_owned_images="$(docker image ls nemoclaw-sandbox-local --format '{{.Repository}}\t{{.Tag}}' | awk '$1 == "nemoclaw-sandbox-local" && index($2, "e2e-jetson-nvmap-") == 1 { print $1 ":" $2 }')" -[ -z "$test_owned_images" ] || { echo "A test-owned Jetson image remains before dispatch" >&2; exit 1; } -printf 'nodePath\t'; printf '%s' "$node_path" | clean_line -printf '\nnodeVersion\t'; printf '%s' "$node_version" | clean_line -printf '\nnpmPath\t'; printf '%s' "$npm_path" | clean_line -printf '\nnpmVersion\t'; printf '%s' "$npm_version" | clean_line -printf '\nollamaPath\t'; printf '%s' "$ollama_path" | clean_line -printf '\nollamaModelsBase64\t'; printf '%s' "$ollama_models_base64" | clean_line -printf '\nopenshellState\tabsent' -printf '\n' -`; - -const CLEANUP_VERIFICATION_SCRIPT = - String.raw`set -euo pipefail -job_id="$1" -cleanup_evidence_base64="$2" -[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] -workspace="/var/tmp/nemoclaw-jetson-e2e/$job_id" -job_home="$workspace/home" -sandbox_name=e2e-jetson-nvmap -gateway_name=nemoclaw -recorded_volumes=() -recorded_process_ids=() -cleanup_rows="$( - printf '%s' "$cleanup_evidence_base64" | base64 --decode | node -e ' - const fs = require("node:fs"); - const value = JSON.parse(fs.readFileSync(0, "utf8")); - if (value.schemaVersion !== 1 || !Array.isArray(value.volumes) || !Array.isArray(value.processIds)) process.exit(1); - for (const volume of value.volumes) { - if (typeof volume !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/.test(volume)) process.exit(1); - process.stdout.write("volume\t" + volume + "\n"); - } - for (const pid of value.processIds) { - if (!Number.isSafeInteger(pid) || pid < 1) process.exit(1); - process.stdout.write("processId\t" + pid + "\n"); - } - ' -)" -if [ -n "$cleanup_rows" ]; then - while IFS=$'\t' read -r kind identity; do - case "$kind" in - volume) recorded_volumes+=("$identity") ;; - processId) recorded_process_ids+=("$identity") ;; - *) echo "Cleanup evidence contains an invalid identity" >&2; exit 1 ;; - esac - done <<<"$cleanup_rows" -fi -test ! -e "$workspace" -test ! -L "$workspace" -sandbox_container_output="$(docker ps -aq \ - --filter label=openshell.ai/managed-by=openshell \ - --filter "label=openshell.ai/sandbox-name=$sandbox_name")" -if [ -n "$sandbox_container_output" ]; then - echo "A test-owned OpenShell sandbox container remains" >&2 - exit 1 -fi -container_rows="$(docker container ls --all --no-trunc --format '{{.ID}}\t{{.Names}}')" -if printf '%s\n' "$container_rows" | awk -F '\t' -v name="openshell-cluster-$gateway_name" '$2 == name { found = 1 } END { exit found ? 0 : 1 }'; then - echo "The test-owned OpenShell gateway container remains" >&2 - exit 1 -fi -volume_names="$(docker volume ls --format '{{.Name}}')" -if printf '%s\n' "$volume_names" | grep -Fqx "openshell-cluster-$gateway_name"; then - echo "The test-owned OpenShell gateway volume remains" >&2 - exit 1 -fi -for volume in "${"${recorded_volumes[@]}"}"; do - if printf '%s\n' "$volume_names" | grep -Fqx "$volume"; then - echo "A recorded test-owned Docker volume remains" >&2 - exit 1 - fi -done -for pid in "${"${recorded_process_ids[@]}"}"; do - if kill -0 "$pid" 2>/dev/null; then - echo "A recorded test-owned helper process remains" >&2 - exit 1 - fi -done -read_proc_uid() { - awk '/^Uid:/ { print $2; found = 1; exit } END { exit found ? 0 : 1 }' "$1/status" 2>/dev/null -} -read_proc_environment() { - dd if="$1/environ" status=none 2>/dev/null | tr '\000' '\n' -} -read_proc_command() { - dd if="$1/cmdline" status=none 2>/dev/null | tr '\000' ' ' -} -handle_proc_read_failure() { - local proc_dir="$1" field="$2" process_uid directory_uid - [ -d "$proc_dir" ] || return 0 - if process_uid="$(read_proc_uid "$proc_dir")"; then - [ "$process_uid" = "$(id -u)" ] || return 0 - else - [ -d "$proc_dir" ] || return 0 - if ! directory_uid="$(stat -c %u "$proc_dir" 2>/dev/null)"; then - [ -d "$proc_dir" ] || return 0 - echo "Unable to verify the owner of a live process after a failed $field read" >&2 - exit 1 - fi - [ "$directory_uid" = "$(id -u)" ] || return 0 - fi - echo "Unable to inspect $field for a live same-user process" >&2 - exit 1 -} -for proc_dir in /proc/[0-9]*; do - if ! process_uid="$(read_proc_uid "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" owner - continue - fi - [ "$process_uid" = "$(id -u)" ] || continue - if ! environment="$(read_proc_environment "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" environment - continue - fi - printf '%s\n' "$environment" | grep -Fqx "HOME=$job_home" || continue - if ! cmdline="$(read_proc_command "$proc_dir")"; then - handle_proc_read_failure "$proc_dir" command - continue - fi - case "$cmdline" in - *ollama-auth-proxy.* | *openshell-gateway* | *openshell-forward* | *openshell\ forward* | *cloudflared*) - echo "A job-owned helper process remains" >&2 - exit 1 - ;; - esac -done -` + PRESERVED_BASELINE_SCRIPT; - -const JETSON_TEST_SCRIPT = String.raw`set -euo pipefail -candidate_sha="$1" -job_id="$2" -timeout_seconds="$3" -[[ "$candidate_sha" =~ ^[a-f0-9]{40}$ ]] -[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] -[[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] - -workspace_root=/var/tmp/nemoclaw-jetson-e2e -if [ -L "$workspace_root" ]; then - echo "Jetson E2E workspace root must not be a symbolic link" >&2 - exit 1 -fi -install -d -m 0700 "$workspace_root" -workspace="$workspace_root/$job_id" -if [ -e "$workspace" ]; then - echo "Jetson E2E workspace already exists; cleanup recovery is required" >&2 - exit 1 -fi -install -d -m 0700 "$workspace" -install -d -m 0700 \ - "$workspace/home" \ - "$workspace/home/.cache" \ - "$workspace/home/.config" \ - "$workspace/home/.local/bin" \ - "$workspace/home/.local/share" \ - "$workspace/home/.local/state" \ - "$workspace/npm-prefix" \ - "$workspace/runtime" \ - "$workspace/tmp" - -export HOME="$workspace/home" -export TMPDIR="$workspace/tmp" -export XDG_CACHE_HOME="$workspace/home/.cache" -export XDG_CONFIG_HOME="$workspace/home/.config" -export XDG_DATA_HOME="$workspace/home/.local/share" -export XDG_STATE_HOME="$workspace/home/.local/state" -export XDG_BIN_HOME="$workspace/home/.local/bin" -export XDG_RUNTIME_DIR="$workspace/runtime" -export npm_config_prefix="$workspace/npm-prefix" -export PATH="$XDG_BIN_HOME:$workspace/npm-prefix/bin:$PATH" -export NEMOCLAW_JETSON_WORKSPACE="$workspace" -unset DBUS_SESSION_BUS_ADDRESS NEMOCLAW_OPENSHELL_BIN NEMOCLAW_OPENSHELL_GATEWAY_BIN - -if [ -w /usr/local/bin ] || [ -w /usr/bin ]; then - echo "Jetson job user must not be able to install OpenShell into a host binary directory" >&2 - exit 1 -fi - -git init --quiet "$workspace/repository" -git -C "$workspace/repository" remote add origin https://github.com/NVIDIA/NemoClaw.git -git -C "$workspace/repository" fetch --quiet --depth=1 --no-tags origin "$candidate_sha" -fetched_sha="$(git -C "$workspace/repository" rev-parse FETCH_HEAD)" -[[ "$fetched_sha" == "$candidate_sha" ]] -git -C "$workspace/repository" checkout --quiet --detach "$candidate_sha" -cd "$workspace/repository" - -node -e ' - const [major, minor] = process.versions.node.split(".").map(Number); - if (major < 22 || (major === 22 && minor < 19)) process.exit(1); -' || { echo "Jetson E2E requires Node.js 22.19.0 or later" >&2; exit 1; } -npm_major="$(npm --version | cut -d . -f 1)" -[[ "$npm_major" =~ ^[0-9]+$ ]] && (( npm_major >= 10 )) || { - echo "Jetson E2E requires npm 10 or later" >&2 - exit 1 -} -npm ci --ignore-scripts -npm run build:cli - -export E2E_JOB=1 -export E2E_TARGET_ID=jetson-nvmap-gpu -export E2E_ARTIFACT_DIR="$workspace/e2e-artifacts/live/jetson-nvmap-gpu" -export NEMOCLAW_CLI_BIN="$workspace/repository/bin/nemoclaw.js" -export NEMOCLAW_RUN_LIVE_E2E=1 -export NEMOCLAW_NON_INTERACTIVE=1 -export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 -export NEMOCLAW_SANDBOX_NAME=e2e-jetson-nvmap -export NEMOCLAW_RECREATE_SANDBOX=1 -export NEMOCLAW_PROVIDER=ollama -export OPENSHELL_GATEWAY=nemoclaw - -timeout --foreground --signal=TERM --kill-after=30s "${"${timeout_seconds}"}s" \ - npx tsx tools/e2e/live-vitest-invocation.mts run \ - --test-path test/e2e/live/jetson-nvmap-gpu.test.ts - -for installed_command in nemoclaw openshell openshell-gateway openshell-sandbox; do - installed_path="$(command -v "$installed_command")" - canonical_path="$(realpath -e "$installed_path")" - case "$canonical_path" in - "$workspace"/*) ;; - *) echo "$installed_command resolved outside the Jetson job workspace" >&2; exit 1 ;; - esac -done -`; - -const COLLECT_ARTIFACT_SCRIPT = String.raw`set -euo pipefail -job_id="$1" -[[ "$job_id" =~ ^[a-f0-9]{64}$ ]] -workspace="/var/tmp/nemoclaw-jetson-e2e/$job_id" - -artifact_directory="$workspace/e2e-artifacts/live/jetson-nvmap-gpu" -if [ ! -d "$artifact_directory" ] || [ -L "$artifact_directory" ]; then - exit 0 -fi -artifact_bytes="$(du --apparent-size --block-size=1 --summarize "$artifact_directory" | cut -f1)" -[[ "$artifact_bytes" =~ ^[0-9]+$ ]] -if (( artifact_bytes > ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} )); then - echo "Jetson E2E artifacts exceed ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} bytes" >&2 - exit 1 -fi -archive="$workspace/jetson-e2e-artifacts.tar.gz" -tar --one-file-system --create --gzip --file "$archive" --directory "$artifact_directory" -- . -archive_bytes="$(stat --format='%s' "$archive")" -if (( archive_bytes > ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} )); then - echo "Jetson E2E artifact archive exceeds ${MAX_JETSON_ARTIFACT_ARCHIVE_BYTES} bytes" >&2 - exit 1 -fi -base64 --wrap=0 "$archive" -`; - -export interface SshJetsonWorkerConfig { - cleanupExecutable: string; - destination: string; - identityFile: string; - knownHostsFile: string; - stateDirectory: string; - testTimeoutSeconds: number; -} - -function requireSecureFile( - file: string, - options: { executable?: boolean; private?: boolean }, -): void { - if (!path.isAbsolute(file) || /[\u0000-\u001f\u007f]/u.test(file)) { - throw new Error(`${file} must be an absolute path without control characters`); - } - const stat = fs.lstatSync(file); - if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`${file} must be a regular file`); - if ((stat.mode & 0o022) !== 0) throw new Error(`${file} must not be group- or world-writable`); - if (options.private && (stat.mode & 0o077) !== 0) { - throw new Error(`${file} must be readable only by its owner`); - } - const currentUid = process.getuid?.(); - if (currentUid !== undefined && stat.uid !== currentUid && stat.uid !== 0) { - throw new Error(`${file} must be owned by root or the dispatcher user`); - } - if (options.executable && (stat.mode & 0o111) === 0) { - throw new Error(`${file} must be executable`); - } -} - -function requireSecureCleanupExecutable(file: string): void { - if (file !== JETSON_CLEANUP_EXECUTABLE) { - requireSecureFile(file, { executable: true }); - return; - } - const linkStat = fs.lstatSync(file); - if (!linkStat.isSymbolicLink()) { - requireSecureFile(file, { executable: true }); - return; - } - if (linkStat.uid !== 0) { - throw new Error(`${file} symbolic link must be owned by root`); - } - if (fs.readlinkSync(file) !== JETSON_CLEANUP_TARGET) { - throw new Error(`${file} must select the managed current-release cleanup program`); - } - requireSecureFile(JETSON_CLEANUP_TARGET, { executable: true }); -} - -function positiveIntegerEnvironment( - value: string | undefined, - name: string, - minimum: number, - maximum: number, -): number { - if (!value || !/^[1-9][0-9]*$/u.test(value)) { - throw new Error(`${name} must be a positive integer`); - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new Error(`${name} must be between ${minimum} and ${maximum}`); - } - return parsed; -} - -export function loadSshJetsonWorkerConfig( - options: { stateDirectory: string }, - env: NodeJS.ProcessEnv = process.env, -): SshJetsonWorkerConfig { - if (!path.isAbsolute(options.stateDirectory)) { - throw new Error("Jetson dispatcher state directory must be absolute"); - } - const destination = env.JETSON_DISPATCH_SSH_DESTINATION ?? ""; - if (destination !== JETSON_SSH_DESTINATION) { - throw new Error(`JETSON_DISPATCH_SSH_DESTINATION must be ${JETSON_SSH_DESTINATION}`); - } - const identityFile = env.JETSON_DISPATCH_SSH_IDENTITY_FILE ?? ""; - const knownHostsFile = env.JETSON_DISPATCH_SSH_KNOWN_HOSTS_FILE ?? ""; - const cleanupExecutable = env.JETSON_DISPATCH_CLEANUP_EXECUTABLE ?? ""; - requireSecureFile(identityFile, { private: true }); - requireSecureFile(knownHostsFile, {}); - requireSecureCleanupExecutable(cleanupExecutable); - return { - cleanupExecutable, - destination, - identityFile, - knownHostsFile, - stateDirectory: options.stateDirectory, - testTimeoutSeconds: positiveIntegerEnvironment( - env.JETSON_DISPATCH_TEST_TIMEOUT_SECONDS, - "JETSON_DISPATCH_TEST_TIMEOUT_SECONDS", - 60, - 50 * 60, - ), - }; -} - -interface ProcessResult { - stdout: string; - stderr: string; -} - -export class ProcessFailure extends Error { - readonly result: ProcessResult; - - constructor(message: string, result: ProcessResult) { - super(message); - this.result = result; - } -} - -function runProcess(options: { - executable: string; - args: string[]; - input?: string; - signal: AbortSignal; -}): Promise { - return new Promise((resolve, reject) => { - const child = spawn(options.executable, options.args, { - env: SAFE_PROCESS_ENV, - stdio: ["pipe", "pipe", "pipe"], - }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - let outputBytes = 0; - let settled = false; - let killTimer: NodeJS.Timeout | undefined; - - const finish = (error?: Error, result?: ProcessResult): void => { - if (settled) return; - settled = true; - options.signal.removeEventListener("abort", abort); - if (error) reject(error); - else resolve(result!); - }; - const stop = (): void => { - child.kill("SIGTERM"); - killTimer = setTimeout(() => child.kill("SIGKILL"), 5_000); - killTimer.unref(); - }; - const abort = (): void => stop(); - const collect = (chunks: Buffer[], chunk: Buffer): void => { - if (settled) return; - outputBytes += chunk.length; - if (outputBytes > MAX_PROCESS_OUTPUT_BYTES) { - stop(); - finish(new Error(`Jetson command output exceeds ${MAX_PROCESS_OUTPUT_BYTES} bytes`)); - return; - } - chunks.push(chunk); - }; - - options.signal.addEventListener("abort", abort, { once: true }); - if (options.signal.aborted) abort(); - child.stdout.on("data", (chunk: Buffer) => collect(stdout, chunk)); - child.stderr.on("data", (chunk: Buffer) => collect(stderr, chunk)); - child.stdin.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") { - stop(); - finish(error); - } - }); - child.once("error", (error) => finish(error)); - child.once("close", (code, signal) => { - if (killTimer) clearTimeout(killTimer); - if (settled) return; - const result = { - stdout: Buffer.concat(stdout).toString("utf8"), - stderr: Buffer.concat(stderr).toString("utf8"), - }; - if (options.signal.aborted) { - finish(new ProcessFailure("Jetson command was interrupted", result)); - return; - } - if (code !== 0) { - finish( - new ProcessFailure( - `Jetson command exited with ${code ?? `signal ${signal ?? "unknown"}`}: ${result.stderr.slice(-500)}`, - result, - ), - ); - return; - } - finish(undefined, result); - }); - if (options.input !== undefined) child.stdin.end(options.input, "utf8"); - else child.stdin.end(); - }); -} - -function parseDeviceIdentity(output: string): JetsonDeviceIdentity { - if (Buffer.byteLength(output) > 8 * 1024) throw new Error("Jetson identity output is too large"); - const entries = new Map(); - for (const line of output.trimEnd().split("\n")) { - const separator = line.indexOf("\t"); - if (separator < 1) throw new Error("Jetson identity output is malformed"); - const key = line.slice(0, separator); - const value = line.slice(separator + 1); - if (entries.has(key) || value.length === 0 || /[\u0000-\u001f\u007f]/u.test(value)) { - throw new Error("Jetson identity output is malformed"); - } - entries.set(key, value.slice(0, 500)); - } - const expected = ["jetpackVersion", "jetsonLinuxRelease", "kernel", "model"]; - if (entries.size !== expected.length || expected.some((key) => !entries.has(key))) { - throw new Error("Jetson identity output is incomplete"); - } - return { - model: entries.get("model")!, - jetpackVersion: entries.get("jetpackVersion")!, - jetsonLinuxRelease: entries.get("jetsonLinuxRelease")!, - kernel: entries.get("kernel")!, - }; -} - -interface JetsonPreservedBaseline { - nodePath: string; - nodeVersion: string; - npmPath: string; - npmVersion: string; - ollamaPath: string; - ollamaModels: JetsonOllamaModel[]; - openshellState: "absent"; -} - -interface JetsonOllamaModel { - name: string; - id: string; -} - -function parseOllamaModelsBase64(value: string): JetsonOllamaModel[] { - if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value)) { - throw new Error("Jetson protected-baseline Ollama model data is malformed"); - } - const decoded = Buffer.from(value, "base64"); - if ( - decoded.length > MAX_OLLAMA_MODELS_BYTES || - decoded.toString("base64") !== value || - decoded.includes(0) - ) { - throw new Error("Jetson protected-baseline Ollama model data is malformed or oversized"); - } - const rows = decoded.length === 0 ? [] : decoded.toString("utf8").trimEnd().split("\n"); - if (rows.length > MAX_OLLAMA_MODELS) { - throw new Error("Jetson protected-baseline Ollama model data is malformed or oversized"); - } - const models = rows.map((row) => { - const fields = row.split("\t"); - if ( - fields.length !== 2 || - !/^[A-Za-z0-9][A-Za-z0-9._/:@-]{0,254}$/u.test(fields[0]!) || - !/^[a-f0-9]{12,64}$/u.test(fields[1]!) - ) { - throw new Error("Jetson protected-baseline Ollama model data is malformed"); - } - return { name: fields[0]!, id: fields[1]! }; - }); - const sorted = [...models].sort((left, right) => { - if (left.name !== right.name) return left.name < right.name ? -1 : 1; - if (left.id === right.id) return 0; - return left.id < right.id ? -1 : 1; - }); - const names = new Set(); - if ( - models.some((model) => { - if (names.has(model.name)) return true; - names.add(model.name); - return false; - }) || - models.some( - (model, index) => model.name !== sorted[index]!.name || model.id !== sorted[index]!.id, - ) - ) { - throw new Error("Jetson protected-baseline Ollama model data is duplicate or ambiguous"); - } - return models; -} - -function validateOllamaModels(value: unknown): JetsonOllamaModel[] { - if (!Array.isArray(value) || value.length > MAX_OLLAMA_MODELS) { - throw new Error("Jetson protected-baseline Ollama model record is malformed or oversized"); - } - const serialized = value - .map((model) => { - if (!model || typeof model !== "object" || Array.isArray(model)) { - throw new Error("Jetson protected-baseline Ollama model record is malformed"); - } - const record = model as Record; - if ( - Object.keys(record).sort().join(",") !== "id,name" || - typeof record.name !== "string" || - typeof record.id !== "string" - ) { - throw new Error("Jetson protected-baseline Ollama model record is malformed"); - } - return `${record.name}\t${record.id}`; - }) - .join("\n"); - if (Buffer.byteLength(serialized) > MAX_OLLAMA_MODELS_BYTES) { - throw new Error("Jetson protected-baseline Ollama model record is malformed or oversized"); - } - return parseOllamaModelsBase64(Buffer.from(serialized).toString("base64")); -} - -function parsePreservedBaseline(output: string): JetsonPreservedBaseline { - if (Buffer.byteLength(output) > MAX_BASELINE_OUTPUT_BYTES) { - throw new Error("Jetson protected-baseline output is too large"); - } - const entries = new Map(); - for (const line of output.trimEnd().split("\n")) { - const separator = line.indexOf("\t"); - if (separator < 1) throw new Error("Jetson protected-baseline output is malformed"); - const key = line.slice(0, separator); - const value = line.slice(separator + 1); - if ( - entries.has(key) || - (value.length === 0 && key !== "ollamaModelsBase64") || - /[\u0000-\u001f\u007f]/u.test(value) - ) { - throw new Error("Jetson protected-baseline output is malformed"); - } - entries.set(key, value); - } - const expected = [ - "nodePath", - "nodeVersion", - "npmPath", - "npmVersion", - "ollamaPath", - "ollamaModelsBase64", - "openshellState", - ] as const; - if (entries.size !== expected.length || expected.some((key) => !entries.has(key))) { - throw new Error("Jetson protected-baseline output is incomplete"); - } - const baseline: JetsonPreservedBaseline = { - nodePath: entries.get("nodePath")!, - nodeVersion: entries.get("nodeVersion")!, - npmPath: entries.get("npmPath")!, - npmVersion: entries.get("npmVersion")!, - ollamaPath: entries.get("ollamaPath")!, - ollamaModels: parseOllamaModelsBase64(entries.get("ollamaModelsBase64")!), - openshellState: entries.get("openshellState") as "absent", - }; - if (baseline.openshellState !== "absent") { - throw new Error("Jetson protected-baseline OpenShell state is malformed"); - } - if (Buffer.byteLength(`${JSON.stringify(baseline)}\n`) > MAX_BASELINE_RECORD_BYTES) { - throw new Error("Jetson protected-baseline record is too large"); - } - return baseline; -} - -function parsePreservedBaselineJson(output: string): JetsonPreservedBaseline { - if (Buffer.byteLength(output) > MAX_BASELINE_RECORD_BYTES) { - throw new Error("Jetson protected-baseline record is too large"); - } - let parsed: unknown; - try { - parsed = JSON.parse(output); - } catch { - throw new Error("Jetson protected-baseline record is malformed"); - } - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error("Jetson protected-baseline record is malformed"); - } - const record = parsed as Record; - const expected = [ - "nodePath", - "nodeVersion", - "npmPath", - "npmVersion", - "ollamaPath", - "ollamaModels", - "openshellState", - ]; - if ( - Object.keys(record).length !== expected.length || - expected.some((key) => key !== "ollamaModels" && typeof record[key] !== "string") - ) { - throw new Error("Jetson protected-baseline record is incomplete"); - } - const ollamaModels = validateOllamaModels(record.ollamaModels); - const baselineOutput = [ - `nodePath\t${record.nodePath}`, - `nodeVersion\t${record.nodeVersion}`, - `npmPath\t${record.npmPath}`, - `npmVersion\t${record.npmVersion}`, - `ollamaPath\t${record.ollamaPath}`, - `ollamaModelsBase64\t${Buffer.from( - ollamaModels.map((model) => `${model.name}\t${model.id}`).join("\n"), - ).toString("base64")}`, - `openshellState\t${record.openshellState}`, - ].join("\n"); - return parsePreservedBaseline(baselineOutput); -} - -function protectedHostBaselineMatches( - expected: JetsonPreservedBaseline, - actual: JetsonPreservedBaseline, -): boolean { - return ( - expected.nodePath === actual.nodePath && - expected.nodeVersion === actual.nodeVersion && - expected.npmPath === actual.npmPath && - expected.npmVersion === actual.npmVersion && - expected.ollamaPath === actual.ollamaPath && - expected.openshellState === actual.openshellState - ); -} - -function preservesOllamaModels( - expected: JetsonPreservedBaseline, - actual: JetsonPreservedBaseline, -): boolean { - const actualModels = new Map(actual.ollamaModels.map((model) => [model.name, model.id])); - return expected.ollamaModels.every((model) => actualModels.get(model.name) === model.id); -} - -function validateCleanupEvidence(value: unknown): JetsonCleanupEvidence { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("Jetson cleanup evidence is malformed"); - } - const record = value as Record; - if ( - Object.keys(record).length !== 3 || - record.schemaVersion !== 1 || - !Array.isArray(record.volumes) || - !Array.isArray(record.processIds) - ) { - throw new Error("Jetson cleanup evidence is malformed"); - } - const volumes = record.volumes; - const processIds = record.processIds; - if ( - volumes.some( - (volume) => typeof volume !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(volume), - ) || - processIds.some((pid) => typeof pid !== "number" || !Number.isSafeInteger(pid) || pid < 1) - ) { - throw new Error("Jetson cleanup evidence contains an invalid resource identity"); - } - return { - schemaVersion: 1, - volumes: [...new Set(volumes as string[])].sort(), - processIds: [...new Set(processIds as number[])].sort((left, right) => left - right), - }; -} - -function parseCleanupEvidenceJson(output: string): JetsonCleanupEvidence { - if (Buffer.byteLength(output) > MAX_CLEANUP_EVIDENCE_BYTES) { - throw new Error("Jetson cleanup evidence is too large"); - } - try { - return validateCleanupEvidence(JSON.parse(output)); - } catch (error) { - if (error instanceof Error && error.message.startsWith("Jetson cleanup evidence")) throw error; - throw new Error("Jetson cleanup evidence is malformed"); - } -} - -function parseCleanupCommandEvidence(output: string): JetsonCleanupEvidence { - if (Buffer.byteLength(output) > MAX_PROCESS_OUTPUT_BYTES) { - throw new Error("Jetson cleanup output is too large"); - } - const lines = output.replace(/\r/gu, "").split("\n"); - while (lines.at(-1) === "") lines.pop(); - const beginIndex = lines.indexOf(CLEANUP_EVIDENCE_BEGIN); - const endIndex = lines.indexOf(CLEANUP_EVIDENCE_END); - if ( - beginIndex < 0 || - endIndex < 0 || - beginIndex !== lines.lastIndexOf(CLEANUP_EVIDENCE_BEGIN) || - endIndex !== lines.lastIndexOf(CLEANUP_EVIDENCE_END) || - beginIndex >= endIndex - ) { - throw new Error("Jetson cleanup did not return bounded resource evidence"); - } - const volumes: string[] = []; - const processIds: number[] = []; - for (const line of lines.slice(beginIndex + 1, endIndex)) { - const fields = line.split("\t"); - if (fields.length !== 2) throw new Error("Jetson cleanup evidence is malformed"); - const [kind, identity] = fields; - if (kind === "volume") { - volumes.push(identity!); - } else if (kind === "processId" && /^[1-9][0-9]*$/u.test(identity!)) { - processIds.push(Number(identity)); - } else { - throw new Error("Jetson cleanup evidence is malformed"); - } - } - return validateCleanupEvidence({ schemaVersion: 1, volumes, processIds }); -} - -export class SshJetsonDispatchWorker implements JetsonDispatchWorker { - readonly #config: SshJetsonWorkerConfig; - readonly #runProcess: typeof runProcess; - - constructor(config: SshJetsonWorkerConfig, processRunner: typeof runProcess = runProcess) { - this.#config = config; - this.#runProcess = processRunner; - } - - async run( - request: JetsonDispatchRequest, - options: { jobId: string; signal: AbortSignal }, - ): Promise { - fs.rmSync(this.#baselinePath(options.jobId), { force: true }); - const baselineResult = await this.#runProcess({ - executable: "ssh", - args: [...this.#sshArgs(), this.#config.destination, "bash", "-s"], - input: PRESERVED_BASELINE_SCRIPT, - signal: options.signal, - }); - const baseline = parsePreservedBaseline(baselineResult.stdout); - writePrivateRegularFile(this.#baselinePath(options.jobId), `${JSON.stringify(baseline)}\n`); - const identityResult = await this.#runProcess({ - executable: "ssh", - args: [...this.#sshArgs(), this.#config.destination, "bash", "-s"], - input: DEVICE_IDENTITY_SCRIPT, - signal: options.signal, - }); - const device = parseDeviceIdentity(identityResult.stdout); - let testResult: ProcessResult | undefined; - let testError: unknown; - try { - testResult = await this.#runProcess({ - executable: "ssh", - args: [ - ...this.#sshArgs(), - this.#config.destination, - "bash", - "-s", - "--", - request.candidateSha, - options.jobId, - String(this.#config.testTimeoutSeconds), - ], - input: JETSON_TEST_SCRIPT, - signal: options.signal, - }); - } catch (error) { - testError = error; - } - let artifactArchiveBase64: string | undefined; - let artifactCollectionError: unknown; - if (!options.signal.aborted) { - try { - artifactArchiveBase64 = await this.#collectArtifact(options.jobId, options.signal); - } catch (error) { - artifactCollectionError = error; - } - } - if (testError instanceof ProcessFailure) { - const collectionFailure = - artifactCollectionError instanceof Error - ? `; artifact collection failed: ${artifactCollectionError.message}` - : ""; - const failure = new Error(`${testError.message}${collectionFailure}`) as Error & { - artifactArchiveBase64?: string; - log: string; - }; - failure.log = [ - "=== Jetson identity ===", - identityResult.stdout.trimEnd(), - "=== Jetson E2E stdout ===", - testError.result.stdout.trimEnd(), - "=== Jetson E2E stderr ===", - testError.result.stderr.trimEnd(), - "", - ].join("\n"); - if (artifactArchiveBase64 !== undefined) { - failure.artifactArchiveBase64 = artifactArchiveBase64; - } - throw failure; - } - if (testError) throw testError; - if (!testResult) throw new Error("Jetson E2E did not return a process result"); - if (artifactCollectionError) { - const collectionMessage = - artifactCollectionError instanceof Error - ? artifactCollectionError.message - : "artifact collection failed"; - const failure = new Error(collectionMessage) as Error & { log: string }; - failure.log = [ - "=== Jetson identity ===", - identityResult.stdout.trimEnd(), - "=== Jetson E2E stdout ===", - testResult.stdout.trimEnd(), - "=== Jetson E2E stderr ===", - testResult.stderr.trimEnd(), - "=== Jetson artifact collection error ===", - collectionMessage, - "", - ].join("\n"); - throw failure; - } - if (artifactArchiveBase64 === undefined) { - throw new Error("Successful Jetson E2E did not produce a bounded artifact archive"); - } - return { - artifactArchiveBase64, - device, - log: [ - "=== Jetson identity ===", - identityResult.stdout.trimEnd(), - "=== Jetson E2E stdout ===", - testResult.stdout.trimEnd(), - "=== Jetson E2E stderr ===", - testResult.stderr.trimEnd(), - "", - ].join("\n"), - }; - } - - async #collectArtifact(jobId: string, signal: AbortSignal): Promise { - const result = await this.#runProcess({ - executable: "ssh", - args: [...this.#sshArgs(), this.#config.destination, "bash", "-s", "--", jobId], - input: COLLECT_ARTIFACT_SCRIPT, - signal, - }); - if (result.stdout.length === 0) return undefined; - decodeJetsonArtifactArchive(result.stdout); - return result.stdout; - } - - async cleanup(options: { jobId: string; signal: AbortSignal }): Promise { - const cleanupResult = await this.#runProcess({ - executable: this.#config.cleanupExecutable, - args: [], - signal: options.signal, - }); - const reportedEvidence = parseCleanupCommandEvidence(cleanupResult.stdout); - const evidencePath = this.#cleanupEvidencePath(options.jobId); - const persistedRaw = readPrivateRegularFile(evidencePath, { - allowMissing: true, - maxBytes: MAX_CLEANUP_EVIDENCE_BYTES, - }); - if (persistedRaw === null) throw new Error("Jetson cleanup evidence is missing"); - const persistedEvidence = parseCleanupEvidenceJson(persistedRaw); - if (JSON.stringify(reportedEvidence) !== JSON.stringify(persistedEvidence)) { - throw new Error("Jetson cleanup output differs from its durable resource evidence"); - } - const serializedEvidence = `${JSON.stringify(persistedEvidence)}\n`; - const expectedRaw = readPrivateRegularFile(this.#baselinePath(options.jobId), { - allowMissing: true, - maxBytes: MAX_BASELINE_RECORD_BYTES, - }); - const verification = await this.#runProcess({ - executable: "ssh", - args: [ - ...this.#sshArgs(), - this.#config.destination, - "bash", - "-s", - "--", - options.jobId, - Buffer.from(serializedEvidence).toString("base64"), - ], - input: CLEANUP_VERIFICATION_SCRIPT, - signal: options.signal, - }); - const actual = parsePreservedBaseline(verification.stdout); - if (expectedRaw === null) return; - const expected = parsePreservedBaselineJson(expectedRaw); - if (!protectedHostBaselineMatches(expected, actual)) { - throw new Error("Jetson protected tool baseline differs after cleanup"); - } - if (!preservesOllamaModels(expected, actual)) { - throw new Error("Jetson cleanup did not preserve every pre-existing Ollama model"); - } - } - - #baselinePath(jobId: string): string { - if (!/^[a-f0-9]{64}$/u.test(jobId)) throw new Error("Jetson job ID is invalid"); - return path.join(this.#config.stateDirectory, `${jobId}.baseline.json`); - } - - #cleanupEvidencePath(jobId: string): string { - if (!/^[a-f0-9]{64}$/u.test(jobId)) throw new Error("Jetson job ID is invalid"); - return path.join(this.#config.stateDirectory, `${jobId}.cleanup.json`); - } - - #sshArgs(): string[] { - return [ - "-F", - "/dev/null", - "-T", - "-o", - "BatchMode=yes", - "-o", - "ConnectTimeout=15", - "-o", - "IdentitiesOnly=yes", - "-o", - "ServerAliveCountMax=2", - "-o", - "ServerAliveInterval=15", - "-o", - "StrictHostKeyChecking=yes", - "-o", - `UserKnownHostsFile=${this.#config.knownHostsFile}`, - "-i", - this.#config.identityFile, - ]; - } -} diff --git a/tools/e2e/nemoclaw-jetson-dispatch.service b/tools/e2e/nemoclaw-jetson-dispatch.service deleted file mode 100644 index 9c5834aae6b..00000000000 --- a/tools/e2e/nemoclaw-jetson-dispatch.service +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[Unit] -Description=NemoClaw Jetson dispatcher -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=nemoclaw-jetson-dispatch -Group=nemoclaw-jetson-dispatch -UMask=0077 -WorkingDirectory=/opt/nemoclaw-jetson-dispatch/current -EnvironmentFile=/etc/nemoclaw-jetson-dispatch/environment -ExecStart=/usr/bin/node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-service.mts -Restart=on-failure -RestartSec=5 -TimeoutStopSec=360 -NoNewPrivileges=true -PrivateTmp=true -ProtectHome=true -ProtectSystem=strict -ReadWritePaths=/var/lib/nemoclaw-jetson-dispatch - -[Install] -WantedBy=multi-user.target diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 12d565143e7..470a3067d7a 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -3968,14 +3968,12 @@ function validateAllowJetsonDispatchInput(errors: string[], dispatchInputs: Work } const description = stringValue(input.description); if ( - !description.includes("Colossus dispatcher") || - !description.includes("Cloudflare Tunnel") || - !description.includes("cleanup helper") || + !description.includes("operator-owned dispatch backend") || !description.includes("JETSON_DISPATCH_URL") || - !description.includes("test/e2e/docs/jetson-colossus-dispatch.md") + !description.includes("test/e2e/docs/jetson-dispatch.md") ) { errors.push( - "workflow_dispatch allow_jetson_dispatch input must require the Colossus dispatcher, tunnel, cleanup helper, repository URL variable, and deployment checks", + "workflow_dispatch allow_jetson_dispatch input must require the operator-owned backend, repository URL variable, and controller documentation", ); } } @@ -3999,13 +3997,13 @@ function validateJetsonControllerBoundary(errors: string[], jobs: WorkflowRecord errors.push("jetson-nvmap-gpu controller timeout must be 60 minutes"); if ( !isDeepStrictEqual(asRecord(job.concurrency), { - group: "jetson-nvmap-gpu-colossus", + group: "jetson-nvmap-gpu-dispatch", queue: "max", "cancel-in-progress": false, }) ) { errors.push( - "jetson-nvmap-gpu concurrency must queue every dispatch on one fixed device without cancellation", + "jetson-nvmap-gpu concurrency must queue every operator-backend dispatch without cancellation", ); } if (!isDeepStrictEqual(asRecord(job.permissions), { contents: "read", "id-token": "write" })) { @@ -4042,7 +4040,7 @@ function validateJetsonControllerBoundary(errors: string[], jobs: WorkflowRecord errors.push("jetson-nvmap-gpu controller must use Node.js 22"); } } - const dispatch = namedStep(steps, "Dispatch exact commit to Jetson through Colossus"); + const dispatch = namedStep(steps, "Dispatch exact commit to Jetson through operator backend"); if ( dispatch?.run !== "node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-client.mts" ||