From 564e248cfc550bac199d3dbbbc9689b4e824e4c4 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 14 Aug 2026 23:42:33 -0700 Subject: [PATCH 01/13] fix(e2e): model portable Podman service activation --- .github/workflows/portable-profile-e2e.yaml | 41 +----- .../portable-profile-systemctl-shim.sh | 81 ++++++++++++ .../portable-profile-rootless-linux.test.ts | 60 ++------- .../portable-profile-systemctl-shim.test.ts | 119 ++++++++++++++++++ test/helpers/vitest-watch-triggers.ts | 8 +- test/vitest-watch-triggers.test.ts | 10 ++ 6 files changed, 232 insertions(+), 87 deletions(-) create mode 100755 test/e2e/fixtures/portable-profile-systemctl-shim.sh create mode 100644 test/e2e/support/portable-profile-systemctl-shim.test.ts diff --git a/.github/workflows/portable-profile-e2e.yaml b/.github/workflows/portable-profile-e2e.yaml index 2d7c72ce324..de77b606938 100644 --- a/.github/workflows/portable-profile-e2e.yaml +++ b/.github/workflows/portable-profile-e2e.yaml @@ -20,6 +20,7 @@ on: - "src/lib/domain/sandbox/image-tag.ts" - "src/lib/sandbox/**" - "test/e2e/fixtures/availability-env.ts" + - "test/e2e/fixtures/portable-profile-systemctl-shim.sh" - "test/e2e/live/full-e2e.test.ts" - "test/e2e/live/launch-agent-turn.ts" - "test/e2e/live/portable-profile-gateway-proof.ts" @@ -139,47 +140,11 @@ jobs: shim_dir="${RUNNER_TEMP}/nemoclaw-portable-bin" install -d -m 700 "$shim_dir" - cat >"$shim_dir/systemctl" <<'SHIM' - #!/usr/bin/env bash - set -euo pipefail - runtime_dir="${XDG_RUNTIME_DIR:?}" - service_dir="${runtime_dir}/podman" - socket_path="${service_dir}/podman.sock" - pid_file="${runtime_dir}/nemoclaw-podman-service.pid" - log_file="${runtime_dir}/nemoclaw-podman-service.log" - case "$*" in - "--user set-environment "*) exit 0 ;; - "--user try-restart podman.service") - if [[ -f "$pid_file" ]]; then - kill "$(<"$pid_file")" 2>/dev/null || true - rm -f "$pid_file" "$socket_path" - fi - ;; - "--user enable --now podman.socket") - install -d -m 755 "$service_dir" - nohup podman system service --time=0 "unix://$socket_path" >"$log_file" 2>&1 & - echo $! >"$pid_file" - for _ in $(seq 1 100); do - if [[ -S "$socket_path" ]]; then - chmod 660 "$socket_path" - exit 0 - fi - sleep 0.1 - done - cat "$log_file" >&2 || true - exit 1 - ;; - *) - echo "unexpected user-service command: $*" >&2 - exit 64 - ;; - esac - SHIM - chmod 700 "$shim_dir/systemctl" + install -m 700 test/e2e/fixtures/portable-profile-systemctl-shim.sh "$shim_dir/systemctl" export PATH="$shim_dir:$PATH" export XDG_RUNTIME_DIR="$runtime_dir" - systemctl --user enable --now podman.socket + systemctl --user start podman.socket printf '%s\n' "$shim_dir" >>"$GITHUB_PATH" printf 'XDG_RUNTIME_DIR=%s\n' "$runtime_dir" >>"$GITHUB_ENV" printf 'DOCKER_HOST=unix://%s/podman/podman.sock\n' "$runtime_dir" >>"$GITHUB_ENV" diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh new file mode 100755 index 00000000000..b08edf731fd --- /dev/null +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +runtime_dir="${XDG_RUNTIME_DIR:?}" +service_dir="${runtime_dir}/podman" +socket_path="${service_dir}/podman.sock" +pid_file="${runtime_dir}/nemoclaw-podman-service.pid" +log_file="${runtime_dir}/nemoclaw-podman-service.log" + +service_is_active() { + [[ -f "$pid_file" ]] || return 1 + local pid + pid="$(<"$pid_file")" + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1 + kill -0 "$pid" 2>/dev/null && [[ -S "$socket_path" ]] +} + +stop_service() { + if [[ -f "$pid_file" ]]; then + local pid + pid="$(<"$pid_file")" + if [[ "$pid" =~ ^[1-9][0-9]*$ ]]; then + kill "$pid" 2>/dev/null || true + fi + fi + rm -f "$pid_file" "$socket_path" +} + +start_socket() { + if service_is_active; then + return 0 + fi + + stop_service + install -d -m 700 "$service_dir" + nohup podman system service --time=0 "unix://$socket_path" >"$log_file" 2>&1 & + echo $! >"$pid_file" + + for ((attempt = 0; attempt < 100; attempt += 1)); do + if service_is_active; then + chmod 660 "$socket_path" + return 0 + fi + if ! kill -0 "$(<"$pid_file")" 2>/dev/null; then + break + fi + sleep 0.1 + done + + stop_service + cat "$log_file" >&2 || true + return 1 +} + +case "$*" in + "--user set-environment NETAVARK_FW=iptables CONTAINERS_CONF="*) + exit 0 + ;; + "--user try-restart podman.service") + if service_is_active; then + stop_service + start_socket + fi + ;; + "--user is-active --quiet podman.service") + if service_is_active; then + exit 0 + fi + exit 3 + ;; + "--user start podman.socket") + start_socket + ;; + *) + echo "unexpected user-service command: $*" >&2 + exit 64 + ;; +esac diff --git a/test/e2e/live/portable-profile-rootless-linux.test.ts b/test/e2e/live/portable-profile-rootless-linux.test.ts index f386652e62d..d5f3cea6f30 100644 --- a/test/e2e/live/portable-profile-rootless-linux.test.ts +++ b/test/e2e/live/portable-profile-rootless-linux.test.ts @@ -98,49 +98,11 @@ async function waitForRegistry(attempt = 0): Promise { function writeSystemctlShim(binDir: string): void { const shim = path.join(binDir, "systemctl"); - fs.writeFileSync( + fs.copyFileSync( + path.join(process.cwd(), "test/e2e/fixtures/portable-profile-systemctl-shim.sh"), shim, - `#!/usr/bin/env bash -set -euo pipefail -runtime_dir="\${XDG_RUNTIME_DIR:?}" -service_dir="\${runtime_dir}/podman" -socket_path="\${service_dir}/podman.sock" -pid_file="\${runtime_dir}/nemoclaw-podman-service.pid" -log_file="\${runtime_dir}/nemoclaw-podman-service.log" - -case "$*" in - "--user set-environment NETAVARK_FW=iptables CONTAINERS_CONF="*) - exit 0 - ;; - "--user try-restart podman.service") - if [[ -f "\${pid_file}" ]]; then - kill "$(<"\${pid_file}")" 2>/dev/null || true - rm -f "\${pid_file}" "\${socket_path}" - fi - exit 0 - ;; - "--user enable --now podman.socket") - install -d -m 755 "\${service_dir}" - nohup podman system service --time=0 "unix://\${socket_path}" >"\${log_file}" 2>&1 & - echo $! >"\${pid_file}" - for _ in $(seq 1 100); do - if [[ -S "\${socket_path}" ]]; then - chmod 660 "\${socket_path}" - exit 0 - fi - sleep 0.1 - done - cat "\${log_file}" >&2 || true - exit 1 - ;; - *) - echo "unexpected user-service command: $*" >&2 - exit 64 - ;; -esac -`, - { encoding: "utf-8", mode: 0o700 }, ); + fs.chmodSync(shim, 0o700); } function selectInstallerPodmanRuntime(repoRoot: string): string { @@ -321,9 +283,13 @@ async function main(progress: TestProgress): Promise { } } -test("portable profile rootless environment completes the local image and fixed-host route contracts", { - meta: { e2ePhases: PORTABLE_PROFILE_E2E_PHASES }, - timeout: 120_000, -}, async ({ progress }) => { - await main(progress); -}); +test( + "portable profile rootless environment completes the local image and fixed-host route contracts", + { + meta: { e2ePhases: PORTABLE_PROFILE_E2E_PHASES }, + timeout: 120_000, + }, + async ({ progress }) => { + await main(progress); + }, +); diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts new file mode 100644 index 00000000000..c8610f88829 --- /dev/null +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { readYaml, type Workflow, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); +const SHIM_SOURCE = path.join(REPO_ROOT, "test/e2e/fixtures/portable-profile-systemctl-shim.sh"); + +function writeExecutable(filePath: string, source: string): void { + fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o700 }); +} + +function portableLaunchProvisionStep(): WorkflowStep { + const workflow = readYaml(".github/workflows/portable-profile-e2e.yaml"); + const step = workflow.jobs["portable-launch"]?.steps?.find( + (candidate) => candidate.name === "Provision restricted rootless Linux runtime", + ); + expect(step).toBeDefined(); + return step!; +} + +describe("portable profile systemctl fixture", () => { + it( + "reports inactive status, activates the socket, and reports active status (#9006)", + { + timeout: 15_000, + }, + () => { + const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); + const binDir = path.join(directory, "bin"); + const runtimeDir = path.join(directory, "runtime"); + const shim = path.join(binDir, "systemctl"); + fs.mkdirSync(binDir); + fs.mkdirSync(runtimeDir); + fs.copyFileSync(SHIM_SOURCE, shim); + fs.chmodSync(shim, 0o700); + writeExecutable( + path.join(binDir, "podman"), + `#!${process.execPath} +const net = require("node:net"); +const socketPath = process.argv.at(-1).replace("unix://", ""); +const server = net.createServer(); +server.listen(socketPath); +const stop = () => server.close(() => process.exit(0)); +process.on("SIGINT", stop); +process.on("SIGTERM", stop); +`, + ); + const env = { + ...process.env, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + XDG_RUNTIME_DIR: runtimeDir, + }; + const systemctl = (args: string[]) => + spawnSync(shim, args, { encoding: "utf8", env, timeout: 15_000 }); + + try { + expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(3); + expect( + systemctl([ + "--user", + "set-environment", + "NETAVARK_FW=iptables", + `CONTAINERS_CONF=${path.join(directory, "containers.conf")}`, + ]).status, + ).toBe(0); + const activation = systemctl(["--user", "start", "podman.socket"]); + expect(activation.status, activation.stderr).toBe(0); + expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(0); + } finally { + const pidFile = path.join(runtimeDir, "nemoclaw-podman-service.pid"); + if (fs.existsSync(pidFile)) { + const pid = Number(fs.readFileSync(pidFile, "utf8").trim()); + if (Number.isInteger(pid)) { + process.kill(pid, "SIGTERM"); + } + } + fs.rmSync(directory, { force: true, recursive: true }); + } + }, + ); + + it("rejects an unexpected user-service command (#9006)", () => { + const runtimeDir = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); + try { + const result = spawnSync(SHIM_SOURCE, ["--user", "restart", "podman.socket"], { + encoding: "utf8", + env: { ...process.env, XDG_RUNTIME_DIR: runtimeDir }, + }); + expect(result.status).toBe(64); + expect(result.stderr).toContain( + "unexpected user-service command: --user restart podman.socket", + ); + } finally { + fs.rmSync(runtimeDir, { force: true, recursive: true }); + } + }); + + it("binds both portable profile lanes to the same systemctl fixture (#9006)", () => { + const provision = portableLaunchProvisionStep().run ?? ""; + expect(provision).toContain( + 'install -m 700 test/e2e/fixtures/portable-profile-systemctl-shim.sh "$shim_dir/systemctl"', + ); + expect(provision).toContain("systemctl --user start podman.socket"); + expect( + fs.readFileSync( + path.join(REPO_ROOT, "test/e2e/live/portable-profile-rootless-linux.test.ts"), + "utf8", + ), + ).toContain('"test/e2e/fixtures/portable-profile-systemctl-shim.sh"'); + }); +}); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 1dba947af84..b39aa62a14f 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -151,6 +151,11 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/e2e-standard-profile\.yaml$/, testsToRun: runTests("test/e2e/support/standard-profile-workflow-boundary.test.ts"), }, + { + pattern: + /(?:^|\/)(?:\.github\/workflows\/portable-profile-e2e\.yaml|test\/e2e\/fixtures\/portable-profile-systemctl-shim\.sh)$/, + testsToRun: runTests("test/e2e/support/portable-profile-systemctl-shim.test.ts"), + }, { pattern: /(?:^|\/)\.github\/(?:actions\/docker-auth-(?:cleanup|setup)\/action\.yaml|scripts\/docker-auth-(?:cleanup|setup)\.sh)$/, @@ -184,8 +189,7 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ testsToRun: runTests("test/e2e-main-retry-workflow.test.ts"), }, { - pattern: - /(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|platform-vitest-main)\.yaml$/, + pattern: /(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|platform-vitest-main)\.yaml$/, testsToRun: runTests("test/hosted-runner-recovery-workflow.test.ts"), }, { diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 70afdbfbead..df9cc3f58c2 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -64,6 +64,8 @@ const OPAQUE_INPUTS = [ "test/e2e/docs/parity-inventory.generated.json", ".github/workflows/e2e.yaml", ".github/workflows/e2e-standard-profile.yaml", + ".github/workflows/portable-profile-e2e.yaml", + "test/e2e/fixtures/portable-profile-systemctl-shim.sh", ".github/actions/docker-auth-setup/action.yaml", ".github/actions/docker-auth-cleanup/action.yaml", ".github/scripts/docker-auth-setup.sh", @@ -166,6 +168,14 @@ describe("Vitest opaque-input watch triggers", () => { expect(triggeredBy(".github/workflows/e2e-standard-profile.yaml")).toEqual([ "test/e2e/support/standard-profile-workflow-boundary.test.ts", ]); + for (const portableProfilePath of [ + ".github/workflows/portable-profile-e2e.yaml", + "test/e2e/fixtures/portable-profile-systemctl-shim.sh", + ]) { + expect(triggeredBy(portableProfilePath)).toEqual([ + "test/e2e/support/portable-profile-systemctl-shim.test.ts", + ]); + } for (const authPath of [ ".github/actions/docker-auth-setup/action.yaml", ".github/actions/docker-auth-cleanup/action.yaml", From 35f0e75e92465feafe72360b4b027023e53ea604 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 00:10:01 -0700 Subject: [PATCH 02/13] test(e2e): keep portable service cleanup linear --- .../portable-profile-systemctl-shim.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index c8610f88829..2d3efb49df9 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -28,7 +28,7 @@ function portableLaunchProvisionStep(): WorkflowStep { describe("portable profile systemctl fixture", () => { it( - "reports inactive status, activates the socket, and reports active status (#9006)", + "reports the service inactive, starts the socket, and preserves active status across try-restart (#9006)", { timeout: 15_000, }, @@ -74,13 +74,17 @@ process.on("SIGTERM", stop); const activation = systemctl(["--user", "start", "podman.socket"]); expect(activation.status, activation.stderr).toBe(0); expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(0); + const refresh = systemctl(["--user", "try-restart", "podman.service"]); + expect(refresh.status, refresh.stderr).toBe(0); + expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(0); } finally { const pidFile = path.join(runtimeDir, "nemoclaw-podman-service.pid"); - if (fs.existsSync(pidFile)) { + try { const pid = Number(fs.readFileSync(pidFile, "utf8").trim()); - if (Number.isInteger(pid)) { - process.kill(pid, "SIGTERM"); - } + expect(pid).toBeGreaterThan(0); + process.kill(pid, "SIGTERM"); + } catch (error) { + expect(["ENOENT", "ESRCH"]).toContain((error as NodeJS.ErrnoException).code); } fs.rmSync(directory, { force: true, recursive: true }); } @@ -103,7 +107,7 @@ process.on("SIGTERM", stop); } }); - it("binds both portable profile lanes to the same systemctl fixture (#9006)", () => { + it("binds the portable-launch workflow and rootless Linux test to one systemctl fixture (#9006)", () => { const provision = portableLaunchProvisionStep().run ?? ""; expect(provision).toContain( 'install -m 700 test/e2e/fixtures/portable-profile-systemctl-shim.sh "$shim_dir/systemctl"', From 0bbca0b213a536ddf4eb8c6cdea0342d2e75682c Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 00:30:38 -0700 Subject: [PATCH 03/13] test(e2e): exercise portable shim installation --- .github/workflows/portable-profile-e2e.yaml | 1 + .../fixtures/portable-profile-systemctl.ts | 17 ++++++++++ .../portable-profile-rootless-linux.test.ts | 12 ++----- .../portable-profile-systemctl-shim.test.ts | 31 ++++++++----------- 4 files changed, 33 insertions(+), 28 deletions(-) create mode 100644 test/e2e/fixtures/portable-profile-systemctl.ts diff --git a/.github/workflows/portable-profile-e2e.yaml b/.github/workflows/portable-profile-e2e.yaml index de77b606938..6dd5041f48d 100644 --- a/.github/workflows/portable-profile-e2e.yaml +++ b/.github/workflows/portable-profile-e2e.yaml @@ -21,6 +21,7 @@ on: - "src/lib/sandbox/**" - "test/e2e/fixtures/availability-env.ts" - "test/e2e/fixtures/portable-profile-systemctl-shim.sh" + - "test/e2e/fixtures/portable-profile-systemctl.ts" - "test/e2e/live/full-e2e.test.ts" - "test/e2e/live/launch-agent-turn.ts" - "test/e2e/live/portable-profile-gateway-proof.ts" diff --git a/test/e2e/fixtures/portable-profile-systemctl.ts b/test/e2e/fixtures/portable-profile-systemctl.ts new file mode 100644 index 00000000000..6563b7a9cb6 --- /dev/null +++ b/test/e2e/fixtures/portable-profile-systemctl.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const SYSTEMCTL_SHIM_SOURCE = fileURLToPath( + new URL("./portable-profile-systemctl-shim.sh", import.meta.url), +); + +export function installPortableProfileSystemctlShim(binDir: string): string { + const systemctl = path.join(binDir, "systemctl"); + fs.copyFileSync(SYSTEMCTL_SHIM_SOURCE, systemctl); + fs.chmodSync(systemctl, 0o700); + return systemctl; +} diff --git a/test/e2e/live/portable-profile-rootless-linux.test.ts b/test/e2e/live/portable-profile-rootless-linux.test.ts index d5f3cea6f30..a2882163be8 100644 --- a/test/e2e/live/portable-profile-rootless-linux.test.ts +++ b/test/e2e/live/portable-profile-rootless-linux.test.ts @@ -14,6 +14,7 @@ import * as importedPortableHostPreparation from "../../../src/lib/onboard/exper import * as importedSandboxPrebuild from "../../../src/lib/onboard/sandbox-prebuild.ts"; import * as importedBuildContext from "../../../src/lib/sandbox/build-context.ts"; import { test } from "../fixtures/e2e-test.ts"; +import { installPortableProfileSystemctlShim } from "../fixtures/portable-profile-systemctl.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import { verifyPinnedPodmanGatewayStarts } from "./portable-profile-gateway-proof.ts"; @@ -96,15 +97,6 @@ async function waitForRegistry(attempt = 0): Promise { ); } -function writeSystemctlShim(binDir: string): void { - const shim = path.join(binDir, "systemctl"); - fs.copyFileSync( - path.join(process.cwd(), "test/e2e/fixtures/portable-profile-systemctl-shim.sh"), - shim, - ); - fs.chmodSync(shim, 0o700); -} - function selectInstallerPodmanRuntime(repoRoot: string): string { const payload = path.join(repoRoot, "scripts", "install.sh"); const script = [ @@ -127,7 +119,7 @@ async function main(progress: TestProgress): Promise { const configHome = path.join(home, ".config"); const runtimeDir = `/run/user/${String(process.getuid?.())}`; fs.mkdirSync(binDir, { recursive: true, mode: 0o700 }); - writeSystemctlShim(binDir); + installPortableProfileSystemctlShim(binDir); Object.assign(process.env, { HOME: home, diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index 2d3efb49df9..d4e0503ccaa 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -4,15 +4,12 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; +import { installPortableProfileSystemctlShim } from "../fixtures/portable-profile-systemctl.ts"; import { readYaml, type Workflow, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; -const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); -const SHIM_SOURCE = path.join(REPO_ROOT, "test/e2e/fixtures/portable-profile-systemctl-shim.sh"); - function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o700 }); } @@ -28,7 +25,7 @@ function portableLaunchProvisionStep(): WorkflowStep { describe("portable profile systemctl fixture", () => { it( - "reports the service inactive, starts the socket, and preserves active status across try-restart (#9006)", + "installs a mode-0700 shim that reports inactive status, starts the socket, and stays active across try-restart (#9006)", { timeout: 15_000, }, @@ -36,11 +33,10 @@ describe("portable profile systemctl fixture", () => { const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); const binDir = path.join(directory, "bin"); const runtimeDir = path.join(directory, "runtime"); - const shim = path.join(binDir, "systemctl"); fs.mkdirSync(binDir); fs.mkdirSync(runtimeDir); - fs.copyFileSync(SHIM_SOURCE, shim); - fs.chmodSync(shim, 0o700); + const shim = installPortableProfileSystemctlShim(binDir); + expect(fs.statSync(shim).mode & 0o777).toBe(0o700); writeExecutable( path.join(binDir, "podman"), `#!${process.execPath} @@ -92,9 +88,14 @@ process.on("SIGTERM", stop); ); it("rejects an unexpected user-service command (#9006)", () => { - const runtimeDir = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); + const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); + const binDir = path.join(directory, "bin"); + const runtimeDir = path.join(directory, "runtime"); try { - const result = spawnSync(SHIM_SOURCE, ["--user", "restart", "podman.socket"], { + fs.mkdirSync(binDir); + fs.mkdirSync(runtimeDir); + const shim = installPortableProfileSystemctlShim(binDir); + const result = spawnSync(shim, ["--user", "restart", "podman.socket"], { encoding: "utf8", env: { ...process.env, XDG_RUNTIME_DIR: runtimeDir }, }); @@ -103,21 +104,15 @@ process.on("SIGTERM", stop); "unexpected user-service command: --user restart podman.socket", ); } finally { - fs.rmSync(runtimeDir, { force: true, recursive: true }); + fs.rmSync(directory, { force: true, recursive: true }); } }); - it("binds the portable-launch workflow and rootless Linux test to one systemctl fixture (#9006)", () => { + it("binds the portable-launch workflow to the shared systemctl fixture (#9006)", () => { const provision = portableLaunchProvisionStep().run ?? ""; expect(provision).toContain( 'install -m 700 test/e2e/fixtures/portable-profile-systemctl-shim.sh "$shim_dir/systemctl"', ); expect(provision).toContain("systemctl --user start podman.socket"); - expect( - fs.readFileSync( - path.join(REPO_ROOT, "test/e2e/live/portable-profile-rootless-linux.test.ts"), - "utf8", - ), - ).toContain('"test/e2e/fixtures/portable-profile-systemctl-shim.sh"'); }); }); From f5d2667f2e5a2d2f40181fae70ca64018b2f20c9 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 04:22:44 -0700 Subject: [PATCH 04/13] test(e2e): model cold portable socket activation Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.sh | 234 ++++++++++--- .../portable-profile-systemctl-shim.test.ts | 331 ++++++++++++++---- 2 files changed, 461 insertions(+), 104 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index b08edf731fd..2f192c758aa 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -7,44 +7,66 @@ set -euo pipefail runtime_dir="${XDG_RUNTIME_DIR:?}" service_dir="${runtime_dir}/podman" socket_path="${service_dir}/podman.sock" -pid_file="${runtime_dir}/nemoclaw-podman-service.pid" +activator_pid_file="${runtime_dir}/nemoclaw-podman-socket-activator.pid" +service_pid_file="${runtime_dir}/nemoclaw-podman-service.pid" log_file="${runtime_dir}/nemoclaw-podman-service.log" -service_is_active() { +pid_is_active() { + local pid_file="$1" [[ -f "$pid_file" ]] || return 1 local pid pid="$(<"$pid_file")" [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1 - kill -0 "$pid" 2>/dev/null && [[ -S "$socket_path" ]] + kill -0 "$pid" 2>/dev/null } -stop_service() { - if [[ -f "$pid_file" ]]; then - local pid - pid="$(<"$pid_file")" - if [[ "$pid" =~ ^[1-9][0-9]*$ ]]; then - kill "$pid" 2>/dev/null || true +service_is_active() { + pid_is_active "$service_pid_file" && [[ -S "$socket_path" ]] +} + +socket_is_ready() { + [[ -S "$socket_path" ]] \ + && { service_is_active || pid_is_active "$activator_pid_file"; } +} + +stop_pid() { + local pid_file="$1" + [[ -f "$pid_file" ]] || return 0 + local pid + pid="$(<"$pid_file")" + if [[ "$pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + for ((attempt = 0; attempt < 100; attempt += 1)); do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.05 + done + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null || true fi fi - rm -f "$pid_file" "$socket_path" + rm -f "$pid_file" } -start_socket() { - if service_is_active; then - return 0 - fi +stop_service() { + stop_pid "$service_pid_file" + rm -f "$socket_path" +} +stop_runtime() { stop_service - install -d -m 700 "$service_dir" - nohup podman system service --time=0 "unix://$socket_path" >"$log_file" 2>&1 & - echo $! >"$pid_file" + stop_pid "$activator_pid_file" + rm -f "$socket_path" +} +wait_for_service() { for ((attempt = 0; attempt < 100; attempt += 1)); do if service_is_active; then chmod 660 "$socket_path" return 0 fi - if ! kill -0 "$(<"$pid_file")" 2>/dev/null; then + if ! pid_is_active "$service_pid_file"; then break fi sleep 0.1 @@ -55,27 +77,159 @@ start_socket() { return 1 } -case "$*" in - "--user set-environment NETAVARK_FW=iptables CONTAINERS_CONF="*) - exit 0 - ;; - "--user try-restart podman.service") - if service_is_active; then - stop_service - start_socket +start_service() { + stop_service + install -d -m 700 "$service_dir" + nohup podman system service --time=0 "unix://$socket_path" >>"$log_file" 2>&1 & + echo $! >"$service_pid_file" + wait_for_service +} + +start_socket() { + if socket_is_ready; then + return 0 + fi + + stop_runtime + install -d -m 700 "$service_dir" + NEMOCLAW_PODMAN_LOG_FILE="$log_file" + export NEMOCLAW_PODMAN_LOG_FILE + nohup node - "$socket_path" "$service_pid_file" "$activator_pid_file" \ + >>"$log_file" 2>&1 <<'NODE' & +const { spawn } = require("node:child_process"); +const fs = require("node:fs"); +const net = require("node:net"); + +const [socketPath, servicePidFile, activatorPidFile] = process.argv.slice(2); +const logFile = process.env.NEMOCLAW_PODMAN_LOG_FILE; +let activationStarted = false; + +function removeActivatorState() { + fs.rmSync(activatorPidFile, { force: true }); +} + +function serviceIsRunning(service) { + return service.exitCode === null && service.signalCode === null; +} + +async function activate(client, server) { + activationStarted = true; + client.destroy(); + server.close(); + fs.rmSync(socketPath, { force: true }); + const output = fs.openSync(logFile, "a"); + const service = spawn("podman", ["system", "service", "--time=0", `unix://${socketPath}`], { + detached: true, + stdio: ["ignore", output, output], + }); + fs.closeSync(output); + if (!service.pid) throw new Error("Podman service did not report a process ID."); + fs.writeFileSync(servicePidFile, `${service.pid}\n`, { mode: 0o600 }); + service.unref(); + + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + if (fs.statSync(socketPath).isSocket()) { + fs.chmodSync(socketPath, 0o660); + removeActivatorState(); + process.exit(0); + } + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + if (!serviceIsRunning(service)) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + if (serviceIsRunning(service)) service.kill("SIGTERM"); + fs.rmSync(servicePidFile, { force: true }); + removeActivatorState(); + throw new Error("Podman service did not create its activation socket."); +} + +const server = net.createServer((client) => { + if (activationStarted) { + client.destroy(); + return; + } + void activate(client, server).catch((error) => { + console.error(error); + process.exit(1); + }); +}); + +server.listen(socketPath, () => fs.chmodSync(socketPath, 0o660)); +const stop = () => { + server.close(); + fs.rmSync(socketPath, { force: true }); + removeActivatorState(); + process.exit(0); +}; +process.on("SIGINT", stop); +process.on("SIGTERM", stop); +NODE + echo $! >"$activator_pid_file" + + for ((attempt = 0; attempt < 100; attempt += 1)); do + if socket_is_ready; then + return 0 fi - ;; - "--user is-active --quiet podman.service") - if service_is_active; then - exit 0 + if ! pid_is_active "$activator_pid_file"; then + break fi - exit 3 - ;; - "--user start podman.socket") - start_socket - ;; - *) - echo "unexpected user-service command: $*" >&2 - exit 64 - ;; -esac + sleep 0.1 + done + + stop_runtime + cat "$log_file" >&2 || true + return 1 +} + +if [[ "$#" -eq 4 && + "$1" == "--user" && + "$2" == "set-environment" && + "$3" == "NETAVARK_FW=iptables" && + "$4" == CONTAINERS_CONF=?* ]]; then + exit 0 +fi + +if [[ "$#" -eq 3 && + "$1" == "--user" && + "$2" == "try-restart" && + "$3" == "podman.service" ]]; then + if service_is_active; then + start_service + fi + exit 0 +fi + +if [[ "$#" -eq 4 && + "$1" == "--user" && + "$2" == "is-active" && + "$3" == "--quiet" && + "$4" == "podman.service" ]]; then + if service_is_active; then + exit 0 + fi + exit 3 +fi + +if [[ "$#" -eq 3 && + "$1" == "--user" && + "$2" == "start" && + "$3" == "podman.socket" ]]; then + start_socket + exit 0 +fi + +if [[ "$#" -eq 4 && + "$1" == "--user" && + "$2" == "enable" && + "$3" == "--now" && + "$4" == "podman.socket" ]]; then + start_socket + exit 0 +fi + +echo "unexpected user-service command: $*" >&2 +exit 64 diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index d4e0503ccaa..1222a2fe2b5 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; +import net from "node:net"; import path from "node:path"; import { describe, expect, it } from "vitest"; @@ -10,10 +11,189 @@ import { describe, expect, it } from "vitest"; import { installPortableProfileSystemctlShim } from "../fixtures/portable-profile-systemctl.ts"; import { readYaml, type Workflow, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; +const INSTALLER_PAYLOAD = path.join(import.meta.dirname, "..", "..", "..", "scripts", "install.sh"); + +interface FixtureScope { + readonly binDir: string; + readonly directory: string; + readonly env: NodeJS.ProcessEnv; + readonly runtimeDir: string; + readonly shim: string; + readonly socketPath: string; +} + function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o700 }); } +function createFixture(): FixtureScope { + const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); + const binDir = path.join(directory, "bin"); + const runtimeDir = path.join(directory, "runtime"); + const socketPath = path.join(runtimeDir, "podman", "podman.sock"); + fs.mkdirSync(binDir); + fs.mkdirSync(runtimeDir); + const shim = installPortableProfileSystemctlShim(binDir); + writeExecutable( + path.join(binDir, "podman"), + `#!${process.execPath} +const fs = require("node:fs"); +const net = require("node:net"); +const args = process.argv.slice(2); +if (args[0] === "info") { + process.stdout.write(process.env.FAKE_PODMAN_SOCKET + "\\n"); + process.exit(0); +} +if ( + args.length !== 4 || + args[0] !== "system" || + args[1] !== "service" || + args[2] !== "--time=0" || + !args[3].startsWith("unix://") +) { + process.exit(64); +} +const socketPath = args[3].slice("unix://".length); +fs.rmSync(socketPath, { force: true }); +const server = net.createServer((socket) => { + socket.once("data", () => socket.end("ready")); +}); +server.listen(socketPath); +const stop = () => server.close(() => process.exit(0)); +process.on("SIGINT", stop); +process.on("SIGTERM", stop); +`, + ); + writeExecutable(path.join(binDir, "docker"), "#!/usr/bin/env bash\nexit 0\n"); + return { + binDir, + directory, + env: { + ...process.env, + FAKE_PODMAN_SOCKET: socketPath, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + XDG_RUNTIME_DIR: runtimeDir, + }, + runtimeDir, + shim, + socketPath, + }; +} + +function systemctl(scope: FixtureScope, args: string[]): ReturnType { + return spawnSync(scope.shim, args, { + encoding: "utf8", + env: scope.env, + timeout: 15_000, + }); +} + +function serviceStatus(scope: FixtureScope): number | null { + return systemctl(scope, ["--user", "is-active", "--quiet", "podman.service"]).status; +} + +function activateThroughSocket(socketPath: string): Promise { + return new Promise((resolve, reject) => { + const client = net.createConnection(socketPath); + let output = ""; + let settled = false; + const finish = (): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolve(output); + }; + const timeout = setTimeout(() => { + client.destroy(); + reject(new Error("Timed out waiting for the activated Podman service.")); + }, 15_000); + client.setEncoding("utf8"); + client.once("connect", () => client.write("activate")); + client.on("data", (chunk) => { + output += chunk; + }); + client.once("close", finish); + client.once("error", (error) => { + if ((error as NodeJS.ErrnoException).code === "ECONNRESET") { + finish(); + return; + } + clearTimeout(timeout); + reject(error); + }); + }); +} + +async function waitForServiceStatus(scope: FixtureScope, expected: number): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (serviceStatus(scope) === expected) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + expect(serviceStatus(scope)).toBe(expected); +} + +function readFixturePids(scope: FixtureScope): number[] { + return ["nemoclaw-podman-socket-activator.pid", "nemoclaw-podman-service.pid"].flatMap((name) => { + try { + const pid = Number(fs.readFileSync(path.join(scope.runtimeDir, name), "utf8").trim()); + return Number.isInteger(pid) && pid > 0 ? [pid] : []; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + }); +} + +function pidIsActive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } +} + +async function waitForExit(pid: number): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (!pidIsActive(pid)) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +async function cleanFixture(scope: FixtureScope): Promise { + const pids = readFixturePids(scope); + for (const pid of pids) { + try { + process.kill(pid, "SIGTERM"); + } catch (error) { + expect((error as NodeJS.ErrnoException).code).toBe("ESRCH"); + } + } + await Promise.all(pids.map(waitForExit)); + for (const pid of pids) { + if (pidIsActive(pid)) process.kill(pid, "SIGKILL"); + } + await Promise.all(pids.map(waitForExit)); + expect(pids.every((pid) => !pidIsActive(pid))).toBe(true); + fs.rmSync(scope.directory, { force: true, recursive: true }); +} + +function runInstallerOverride(scope: FixtureScope): ReturnType { + const script = [ + `source "${INSTALLER_PAYLOAD}" >/dev/null 2>&1 || true`, + "uname() { printf 'Linux\\n'; }", + 'export NEMOCLAW_EXPERIMENTAL_PROFILE="portable"', + "prepare_portable_experimental_runtime_override", + 'printf "DOCKER_HOST=%s\\n" "$DOCKER_HOST"', + ].join("\n"); + return spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: scope.env, + timeout: 15_000, + }); +} + function portableLaunchProvisionStep(): WorkflowStep { const workflow = readYaml(".github/workflows/portable-profile-e2e.yaml"); const step = workflow.jobs["portable-launch"]?.steps?.find( @@ -25,86 +205,109 @@ function portableLaunchProvisionStep(): WorkflowStep { describe("portable profile systemctl fixture", () => { it( - "installs a mode-0700 shim that reports inactive status, starts the socket, and stays active across try-restart (#9006)", - { - timeout: 15_000, + "installs a mode-0700 shim that keeps the socket cold until the first client and preserves active status across try-restart (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + try { + expect(fs.statSync(scope.shim).mode & 0o777).toBe(0o700); + expect(serviceStatus(scope)).toBe(3); + expect(systemctl(scope, ["--user", "try-restart", "podman.service"]).status).toBe(0); + expect(serviceStatus(scope)).toBe(3); + expect( + systemctl(scope, [ + "--user", + "set-environment", + "NETAVARK_FW=iptables", + `CONTAINERS_CONF=${path.join(scope.directory, "containers.conf")}`, + ]).status, + ).toBe(0); + + const activation = systemctl(scope, ["--user", "start", "podman.socket"]); + expect(activation.status, String(activation.stderr)).toBe(0); + expect(fs.statSync(scope.socketPath).isSocket()).toBe(true); + expect(serviceStatus(scope)).toBe(3); + expect(await activateThroughSocket(scope.socketPath)).toBe(""); + await waitForServiceStatus(scope, 0); + expect(serviceStatus(scope)).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const firstPid = fs.readFileSync(servicePidFile, "utf8").trim(); + const refresh = systemctl(scope, ["--user", "try-restart", "podman.service"]); + expect(refresh.status, String(refresh.stderr)).toBe(0); + expect(serviceStatus(scope)).toBe(0); + expect(fs.readFileSync(servicePidFile, "utf8").trim()).not.toBe(firstPid); + expect(pidIsActive(Number(firstPid))).toBe(false); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + } finally { + await cleanFixture(scope); + } }, - () => { - const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); - const binDir = path.join(directory, "bin"); - const runtimeDir = path.join(directory, "runtime"); - fs.mkdirSync(binDir); - fs.mkdirSync(runtimeDir); - const shim = installPortableProfileSystemctlShim(binDir); - expect(fs.statSync(shim).mode & 0o777).toBe(0o700); - writeExecutable( - path.join(binDir, "podman"), - `#!${process.execPath} -const net = require("node:net"); -const socketPath = process.argv.at(-1).replace("unix://", ""); -const server = net.createServer(); -server.listen(socketPath); -const stop = () => server.close(() => process.exit(0)); -process.on("SIGINT", stop); -process.on("SIGTERM", stop); -`, - ); - const env = { - ...process.env, - PATH: `${binDir}:${process.env.PATH ?? ""}`, - XDG_RUNTIME_DIR: runtimeDir, - }; - const systemctl = (args: string[]) => - spawnSync(shim, args, { encoding: "utf8", env, timeout: 15_000 }); + ); + it( + "runs the installer enable --now and CLI host-preparation commands through cold activation (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); try { - expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(3); + const installer = runInstallerOverride(scope); + expect(installer.status, String(installer.stderr)).toBe(0); + expect(installer.stdout).toContain(`DOCKER_HOST=unix://${scope.socketPath}`); + expect(fs.statSync(scope.socketPath).isSocket()).toBe(true); + expect(serviceStatus(scope)).toBe(3); + expect( - systemctl([ + systemctl(scope, [ "--user", "set-environment", "NETAVARK_FW=iptables", - `CONTAINERS_CONF=${path.join(directory, "containers.conf")}`, + `CONTAINERS_CONF=${path.join(scope.directory, "containers.conf")}`, ]).status, ).toBe(0); - const activation = systemctl(["--user", "start", "podman.socket"]); - expect(activation.status, activation.stderr).toBe(0); - expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(0); - const refresh = systemctl(["--user", "try-restart", "podman.service"]); - expect(refresh.status, refresh.stderr).toBe(0); - expect(systemctl(["--user", "is-active", "--quiet", "podman.service"]).status).toBe(0); + expect(systemctl(scope, ["--user", "try-restart", "podman.service"]).status).toBe(0); + expect(serviceStatus(scope)).toBe(3); + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + expect(serviceStatus(scope)).toBe(3); + expect(await activateThroughSocket(scope.socketPath)).toBe(""); + await waitForServiceStatus(scope, 0); + expect(serviceStatus(scope)).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); } finally { - const pidFile = path.join(runtimeDir, "nemoclaw-podman-service.pid"); - try { - const pid = Number(fs.readFileSync(pidFile, "utf8").trim()); - expect(pid).toBeGreaterThan(0); - process.kill(pid, "SIGTERM"); - } catch (error) { - expect(["ENOENT", "ESRCH"]).toContain((error as NodeJS.ErrnoException).code); - } - fs.rmSync(directory, { force: true, recursive: true }); + await cleanFixture(scope); } }, ); - it("rejects an unexpected user-service command (#9006)", () => { - const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); - const binDir = path.join(directory, "bin"); - const runtimeDir = path.join(directory, "runtime"); + it("rejects malformed or extended user-service commands (#9006)", () => { + const scope = createFixture(); try { - fs.mkdirSync(binDir); - fs.mkdirSync(runtimeDir); - const shim = installPortableProfileSystemctlShim(binDir); - const result = spawnSync(shim, ["--user", "restart", "podman.socket"], { - encoding: "utf8", - env: { ...process.env, XDG_RUNTIME_DIR: runtimeDir }, - }); - expect(result.status).toBe(64); - expect(result.stderr).toContain( - "unexpected user-service command: --user restart podman.socket", - ); + const driftedCommands = [ + ["--user", "restart", "podman.socket"], + ["--user", "set-environment", "NETAVARK_FW=iptables", "CONTAINERS_CONF="], + [ + "--user", + "set-environment", + "NETAVARK_FW=iptables", + `CONTAINERS_CONF=${path.join(scope.directory, "containers.conf")}`, + "trailing", + ], + [ + "--user set-environment", + "NETAVARK_FW=iptables", + `CONTAINERS_CONF=${path.join(scope.directory, "containers.conf")}`, + ], + ["--user", "start", "podman.socket", "trailing"], + ["--user", "enable", "podman.socket"], + ]; + for (const args of driftedCommands) { + const result = systemctl(scope, args); + expect(result.status, args.join(" ")).toBe(64); + expect(result.stderr).toContain("unexpected user-service command:"); + } } finally { - fs.rmSync(directory, { force: true, recursive: true }); + fs.rmSync(scope.directory, { force: true, recursive: true }); } }); From 385355270eabbdbb42bb60b31cfcb8f03cb62e78 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 04:35:44 -0700 Subject: [PATCH 05/13] test(e2e): keep portable activation handoff linear Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.sh | 4 +- .../portable-profile-systemctl-shim.test.ts | 53 ++++++------------- 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index 2f192c758aa..27194243793 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -21,7 +21,9 @@ pid_is_active() { } service_is_active() { - pid_is_active "$service_pid_file" && [[ -S "$socket_path" ]] + ! pid_is_active "$activator_pid_file" \ + && pid_is_active "$service_pid_file" \ + && [[ -S "$socket_path" ]] } socket_is_ready() { diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index 1222a2fe2b5..b31c1f692d4 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -6,7 +6,7 @@ import fs from "node:fs"; import net from "node:net"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { installPortableProfileSystemctlShim } from "../fixtures/portable-profile-systemctl.ts"; import { readYaml, type Workflow, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; @@ -96,10 +96,7 @@ function activateThroughSocket(socketPath: string): Promise { return new Promise((resolve, reject) => { const client = net.createConnection(socketPath); let output = ""; - let settled = false; const finish = (): void => { - if (settled) return; - settled = true; clearTimeout(timeout); resolve(output); }; @@ -113,35 +110,23 @@ function activateThroughSocket(socketPath: string): Promise { output += chunk; }); client.once("close", finish); - client.once("error", (error) => { - if ((error as NodeJS.ErrnoException).code === "ECONNRESET") { - finish(); - return; - } - clearTimeout(timeout); - reject(error); - }); + client.once("error", finish); }); } async function waitForServiceStatus(scope: FixtureScope, expected: number): Promise { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (serviceStatus(scope) === expected) return; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - expect(serviceStatus(scope)).toBe(expected); + await vi.waitFor(() => expect(serviceStatus(scope)).toBe(expected), { + interval: 50, + timeout: 5_000, + }); } function readFixturePids(scope: FixtureScope): number[] { - return ["nemoclaw-podman-socket-activator.pid", "nemoclaw-podman-service.pid"].flatMap((name) => { - try { - const pid = Number(fs.readFileSync(path.join(scope.runtimeDir, name), "utf8").trim()); - return Number.isInteger(pid) && pid > 0 ? [pid] : []; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; - throw error; - } - }); + return ["nemoclaw-podman-socket-activator.pid", "nemoclaw-podman-service.pid"] + .map((name) => path.join(scope.runtimeDir, name)) + .filter((pidFile) => fs.existsSync(pidFile)) + .map((pidFile) => Number(fs.readFileSync(pidFile, "utf8").trim())) + .filter((pid) => Number.isInteger(pid) && pid > 0); } function pidIsActive(pid: number): boolean { @@ -149,16 +134,16 @@ function pidIsActive(pid: number): boolean { process.kill(pid, 0); return true; } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; - throw error; + expect((error as NodeJS.ErrnoException).code).toBe("ESRCH"); + return false; } } async function waitForExit(pid: number): Promise { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (!pidIsActive(pid)) return; - await new Promise((resolve) => setTimeout(resolve, 20)); - } + await vi.waitFor(() => expect(pidIsActive(pid)).toBe(false), { + interval: 20, + timeout: 2_000, + }); } async function cleanFixture(scope: FixtureScope): Promise { @@ -171,10 +156,6 @@ async function cleanFixture(scope: FixtureScope): Promise { } } await Promise.all(pids.map(waitForExit)); - for (const pid of pids) { - if (pidIsActive(pid)) process.kill(pid, "SIGKILL"); - } - await Promise.all(pids.map(waitForExit)); expect(pids.every((pid) => !pidIsActive(pid))).toBe(true); fs.rmSync(scope.directory, { force: true, recursive: true }); } From 209cece15580ae18d47f0a43b92d4cd1761fbb6a Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 04:56:47 -0700 Subject: [PATCH 06/13] test(e2e): preserve portable socket authority Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.sh | 91 +++++++++++-------- .../portable-profile-systemctl-shim.test.ts | 24 ++++- 2 files changed, 74 insertions(+), 41 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index 27194243793..b01f2228301 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -7,6 +7,7 @@ set -euo pipefail runtime_dir="${XDG_RUNTIME_DIR:?}" service_dir="${runtime_dir}/podman" socket_path="${service_dir}/podman.sock" +backend_socket_path="${service_dir}/nemoclaw-podman-service.sock" activator_pid_file="${runtime_dir}/nemoclaw-podman-socket-activator.pid" service_pid_file="${runtime_dir}/nemoclaw-podman-service.pid" log_file="${runtime_dir}/nemoclaw-podman-service.log" @@ -21,14 +22,14 @@ pid_is_active() { } service_is_active() { - ! pid_is_active "$activator_pid_file" \ + pid_is_active "$activator_pid_file" \ && pid_is_active "$service_pid_file" \ - && [[ -S "$socket_path" ]] + && [[ -S "$socket_path" ]] \ + && [[ -S "$backend_socket_path" ]] } socket_is_ready() { - [[ -S "$socket_path" ]] \ - && { service_is_active || pid_is_active "$activator_pid_file"; } + [[ -S "$socket_path" ]] && pid_is_active "$activator_pid_file" } stop_pid() { @@ -53,13 +54,13 @@ stop_pid() { stop_service() { stop_pid "$service_pid_file" - rm -f "$socket_path" + rm -f "$backend_socket_path" } stop_runtime() { stop_service stop_pid "$activator_pid_file" - rm -f "$socket_path" + rm -f "$socket_path" "$backend_socket_path" } wait_for_service() { @@ -82,7 +83,7 @@ wait_for_service() { start_service() { stop_service install -d -m 700 "$service_dir" - nohup podman system service --time=0 "unix://$socket_path" >>"$log_file" 2>&1 & + nohup podman system service --time=0 "unix://$backend_socket_path" >>"$log_file" 2>&1 & echo $! >"$service_pid_file" wait_for_service } @@ -96,67 +97,85 @@ start_socket() { install -d -m 700 "$service_dir" NEMOCLAW_PODMAN_LOG_FILE="$log_file" export NEMOCLAW_PODMAN_LOG_FILE - nohup node - "$socket_path" "$service_pid_file" "$activator_pid_file" \ + nohup node - "$socket_path" "$backend_socket_path" "$service_pid_file" \ + "$activator_pid_file" \ >>"$log_file" 2>&1 <<'NODE' & const { spawn } = require("node:child_process"); const fs = require("node:fs"); const net = require("node:net"); -const [socketPath, servicePidFile, activatorPidFile] = process.argv.slice(2); +const [socketPath, backendSocketPath, servicePidFile, activatorPidFile] = process.argv.slice(2); const logFile = process.env.NEMOCLAW_PODMAN_LOG_FILE; -let activationStarted = false; +let activationPromise; function removeActivatorState() { fs.rmSync(activatorPidFile, { force: true }); } +function pidIsActive() { + try { + const pid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + process.kill(pid, 0); + return Number.isInteger(pid) && pid > 0; + } catch { + return false; + } +} + +function backendIsReady() { + try { + return pidIsActive() && fs.statSync(backendSocketPath).isSocket(); + } catch (error) { + if (error.code !== "ENOENT") throw error; + return false; + } +} + function serviceIsRunning(service) { return service.exitCode === null && service.signalCode === null; } -async function activate(client, server) { - activationStarted = true; - client.destroy(); - server.close(); - fs.rmSync(socketPath, { force: true }); +async function startService() { + if (backendIsReady()) return; + fs.rmSync(backendSocketPath, { force: true }); const output = fs.openSync(logFile, "a"); - const service = spawn("podman", ["system", "service", "--time=0", `unix://${socketPath}`], { - detached: true, - stdio: ["ignore", output, output], - }); + const service = spawn( + "podman", + ["system", "service", "--time=0", `unix://${backendSocketPath}`], + { detached: true, stdio: ["ignore", output, output] }, + ); fs.closeSync(output); if (!service.pid) throw new Error("Podman service did not report a process ID."); fs.writeFileSync(servicePidFile, `${service.pid}\n`, { mode: 0o600 }); service.unref(); for (let attempt = 0; attempt < 100; attempt += 1) { - try { - if (fs.statSync(socketPath).isSocket()) { - fs.chmodSync(socketPath, 0o660); - removeActivatorState(); - process.exit(0); - } - } catch (error) { - if (error.code !== "ENOENT") throw error; - } + if (backendIsReady()) return; if (!serviceIsRunning(service)) break; await new Promise((resolve) => setTimeout(resolve, 100)); } if (serviceIsRunning(service)) service.kill("SIGTERM"); fs.rmSync(servicePidFile, { force: true }); - removeActivatorState(); - throw new Error("Podman service did not create its activation socket."); + fs.rmSync(backendSocketPath, { force: true }); + throw new Error("Podman service did not create its backend socket."); +} + +async function proxy(client) { + activationPromise ??= startService().finally(() => { + activationPromise = undefined; + }); + await activationPromise; + const backend = net.createConnection(backendSocketPath); + backend.once("connect", () => client.pipe(backend).pipe(client)); + backend.once("error", () => client.destroy()); + client.once("error", () => backend.destroy()); } const server = net.createServer((client) => { - if (activationStarted) { - client.destroy(); - return; - } - void activate(client, server).catch((error) => { + void proxy(client).catch((error) => { console.error(error); - process.exit(1); + client.destroy(); }); }); diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index b31c1f692d4..8fc3f6da7a2 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -186,7 +186,7 @@ function portableLaunchProvisionStep(): WorkflowStep { describe("portable profile systemctl fixture", () => { it( - "installs a mode-0700 shim that keeps the socket cold until the first client and preserves active status across try-restart (#9006)", + "installs a mode-0700 shim that preserves socket identity from cold activation through try-restart (#9006)", { timeout: 30_000 }, async () => { const scope = createFixture(); @@ -206,10 +206,15 @@ describe("portable profile systemctl fixture", () => { const activation = systemctl(scope, ["--user", "start", "podman.socket"]); expect(activation.status, String(activation.stderr)).toBe(0); - expect(fs.statSync(scope.socketPath).isSocket()).toBe(true); + const socketAuthority = fs.statSync(scope.socketPath); + expect(socketAuthority.isSocket()).toBe(true); expect(serviceStatus(scope)).toBe(3); - expect(await activateThroughSocket(scope.socketPath)).toBe(""); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); await waitForServiceStatus(scope, 0); + expect(fs.statSync(scope.socketPath)).toMatchObject({ + dev: socketAuthority.dev, + ino: socketAuthority.ino, + }); expect(serviceStatus(scope)).toBe(0); expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); @@ -220,6 +225,10 @@ describe("portable profile systemctl fixture", () => { expect(serviceStatus(scope)).toBe(0); expect(fs.readFileSync(servicePidFile, "utf8").trim()).not.toBe(firstPid); expect(pidIsActive(Number(firstPid))).toBe(false); + expect(fs.statSync(scope.socketPath)).toMatchObject({ + dev: socketAuthority.dev, + ino: socketAuthority.ino, + }); expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); } finally { await cleanFixture(scope); @@ -236,7 +245,8 @@ describe("portable profile systemctl fixture", () => { const installer = runInstallerOverride(scope); expect(installer.status, String(installer.stderr)).toBe(0); expect(installer.stdout).toContain(`DOCKER_HOST=unix://${scope.socketPath}`); - expect(fs.statSync(scope.socketPath).isSocket()).toBe(true); + const socketAuthority = fs.statSync(scope.socketPath); + expect(socketAuthority.isSocket()).toBe(true); expect(serviceStatus(scope)).toBe(3); expect( @@ -251,8 +261,12 @@ describe("portable profile systemctl fixture", () => { expect(serviceStatus(scope)).toBe(3); expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); expect(serviceStatus(scope)).toBe(3); - expect(await activateThroughSocket(scope.socketPath)).toBe(""); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); await waitForServiceStatus(scope, 0); + expect(fs.statSync(scope.socketPath)).toMatchObject({ + dev: socketAuthority.dev, + ino: socketAuthority.ino, + }); expect(serviceStatus(scope)).toBe(0); expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); } finally { From eeb329d85227a13d21372b614d6f37436878eed6 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 05:45:27 -0700 Subject: [PATCH 07/13] test(e2e): clean portable fixture processes Signed-off-by: Senthil Ravichandran --- .../fixtures/portable-profile-systemctl.ts | 67 +++++++++++++++++ .../portable-profile-rootless-linux.test.ts | 14 ++-- .../portable-profile-systemctl-shim.test.ts | 72 ++++++++++++------- 3 files changed, 118 insertions(+), 35 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl.ts b/test/e2e/fixtures/portable-profile-systemctl.ts index 6563b7a9cb6..11b17a337fe 100644 --- a/test/e2e/fixtures/portable-profile-systemctl.ts +++ b/test/e2e/fixtures/portable-profile-systemctl.ts @@ -9,9 +9,76 @@ const SYSTEMCTL_SHIM_SOURCE = fileURLToPath( new URL("./portable-profile-systemctl-shim.sh", import.meta.url), ); +const FIXTURE_PID_FILES = [ + "nemoclaw-podman-socket-activator.pid", + "nemoclaw-podman-service.pid", +] as const; +const FIXTURE_SOCKET_FILES = ["podman.sock", "nemoclaw-podman-service.sock"] as const; + +function readFixturePid(pidFile: string): number | undefined { + try { + const value = fs.readFileSync(pidFile, "utf8").trim(); + return /^[1-9][0-9]*$/.test(value) ? Number(value) : undefined; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +function fixtureProcessIsActive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } +} + +async function waitForFixtureProcessExit(pid: number): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (!fixtureProcessIsActive(pid)) return true; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return false; +} + +async function terminateFixtureProcess(pid: number): Promise { + try { + process.kill(pid, "SIGTERM"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return; + throw error; + } + if (await waitForFixtureProcessExit(pid)) return; + + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return; + throw error; + } + if (!(await waitForFixtureProcessExit(pid))) { + throw new Error(`Portable profile fixture process ${String(pid)} did not exit.`); + } +} + export function installPortableProfileSystemctlShim(binDir: string): string { const systemctl = path.join(binDir, "systemctl"); fs.copyFileSync(SYSTEMCTL_SHIM_SOURCE, systemctl); fs.chmodSync(systemctl, 0o700); return systemctl; } + +export async function cleanupPortableProfileSystemctlFixture(runtimeDir: string): Promise { + const pidFiles = FIXTURE_PID_FILES.map((name) => path.join(runtimeDir, name)); + const pids = pidFiles.map(readFixturePid).filter((pid): pid is number => pid !== undefined); + await Promise.all(pids.map(terminateFixtureProcess)); + + for (const artifact of [ + ...pidFiles, + ...FIXTURE_SOCKET_FILES.map((name) => path.join(runtimeDir, "podman", name)), + ]) { + fs.rmSync(artifact, { force: true }); + } +} diff --git a/test/e2e/live/portable-profile-rootless-linux.test.ts b/test/e2e/live/portable-profile-rootless-linux.test.ts index a2882163be8..bc334da29b8 100644 --- a/test/e2e/live/portable-profile-rootless-linux.test.ts +++ b/test/e2e/live/portable-profile-rootless-linux.test.ts @@ -14,7 +14,10 @@ import * as importedPortableHostPreparation from "../../../src/lib/onboard/exper import * as importedSandboxPrebuild from "../../../src/lib/onboard/sandbox-prebuild.ts"; import * as importedBuildContext from "../../../src/lib/sandbox/build-context.ts"; import { test } from "../fixtures/e2e-test.ts"; -import { installPortableProfileSystemctlShim } from "../fixtures/portable-profile-systemctl.ts"; +import { + cleanupPortableProfileSystemctlFixture, + installPortableProfileSystemctlShim, +} from "../fixtures/portable-profile-systemctl.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import { verifyPinnedPodmanGatewayStarts } from "./portable-profile-gateway-proof.ts"; @@ -259,15 +262,8 @@ async function main(progress: TestProgress): Promise { stdio: "ignore", timeout: 15_000, }); - const pidFile = path.join(runtimeDir, "nemoclaw-podman-service.pid"); - const pid = fs.existsSync(pidFile) - ? Number(fs.readFileSync(pidFile, "utf-8").trim()) - : Number.NaN; - const terminateService = Number.isInteger(pid) - ? () => process.kill(pid, "SIGTERM") - : () => undefined; - terminateService(); try { + await cleanupPortableProfileSystemctlFixture(runtimeDir); fs.rmSync(root, { recursive: true, force: true }); } catch (error) { console.warn(`Portable E2E temporary cleanup was incomplete: ${String(error)}`); diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index 8fc3f6da7a2..b8aa1abe70d 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -8,7 +8,10 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { installPortableProfileSystemctlShim } from "../fixtures/portable-profile-systemctl.ts"; +import { + cleanupPortableProfileSystemctlFixture, + installPortableProfileSystemctlShim, +} from "../fixtures/portable-profile-systemctl.ts"; import { readYaml, type Workflow, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; const INSTALLER_PAYLOAD = path.join(import.meta.dirname, "..", "..", "..", "scripts", "install.sh"); @@ -121,14 +124,6 @@ async function waitForServiceStatus(scope: FixtureScope, expected: number): Prom }); } -function readFixturePids(scope: FixtureScope): number[] { - return ["nemoclaw-podman-socket-activator.pid", "nemoclaw-podman-service.pid"] - .map((name) => path.join(scope.runtimeDir, name)) - .filter((pidFile) => fs.existsSync(pidFile)) - .map((pidFile) => Number(fs.readFileSync(pidFile, "utf8").trim())) - .filter((pid) => Number.isInteger(pid) && pid > 0); -} - function pidIsActive(pid: number): boolean { try { process.kill(pid, 0); @@ -139,24 +134,8 @@ function pidIsActive(pid: number): boolean { } } -async function waitForExit(pid: number): Promise { - await vi.waitFor(() => expect(pidIsActive(pid)).toBe(false), { - interval: 20, - timeout: 2_000, - }); -} - async function cleanFixture(scope: FixtureScope): Promise { - const pids = readFixturePids(scope); - for (const pid of pids) { - try { - process.kill(pid, "SIGTERM"); - } catch (error) { - expect((error as NodeJS.ErrnoException).code).toBe("ESRCH"); - } - } - await Promise.all(pids.map(waitForExit)); - expect(pids.every((pid) => !pidIsActive(pid))).toBe(true); + await cleanupPortableProfileSystemctlFixture(scope.runtimeDir); fs.rmSync(scope.directory, { force: true, recursive: true }); } @@ -275,6 +254,47 @@ describe("portable profile systemctl fixture", () => { }, ); + it( + "stops both fixture processes and removes both sockets during cleanup (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const backendSocketPath = path.join( + scope.runtimeDir, + "podman", + "nemoclaw-podman-service.sock", + ); + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + await waitForServiceStatus(scope, 0); + + const pids = [activatorPidFile, servicePidFile].map((pidFile) => + Number(fs.readFileSync(pidFile, "utf8").trim()), + ); + expect(pids.every(pidIsActive)).toBe(true); + expect(fs.statSync(scope.socketPath).isSocket()).toBe(true); + expect(fs.statSync(backendSocketPath).isSocket()).toBe(true); + + await cleanupPortableProfileSystemctlFixture(scope.runtimeDir); + + expect(pids.every((pid) => !pidIsActive(pid))).toBe(true); + for (const artifact of [ + activatorPidFile, + servicePidFile, + scope.socketPath, + backendSocketPath, + ]) { + expect(fs.existsSync(artifact), artifact).toBe(false); + } + } finally { + await cleanFixture(scope); + } + }, + ); + it("rejects malformed or extended user-service commands (#9006)", () => { const scope = createFixture(); try { From 90fbff5b6defa0e5f62c12add81e41ede256f8e9 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 06:07:19 -0700 Subject: [PATCH 08/13] test(e2e): fail closed on fixture cleanup Signed-off-by: Senthil Ravichandran --- .../fixtures/portable-profile-systemctl.ts | 14 ++++++++++- .../portable-profile-rootless-linux.test.ts | 9 ++----- .../portable-profile-systemctl-shim.test.ts | 24 +++++++++++++++++-- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl.ts b/test/e2e/fixtures/portable-profile-systemctl.ts index 11b17a337fe..6dbd3f55e47 100644 --- a/test/e2e/fixtures/portable-profile-systemctl.ts +++ b/test/e2e/fixtures/portable-profile-systemctl.ts @@ -18,7 +18,11 @@ const FIXTURE_SOCKET_FILES = ["podman.sock", "nemoclaw-podman-service.sock"] as function readFixturePid(pidFile: string): number | undefined { try { const value = fs.readFileSync(pidFile, "utf8").trim(); - return /^[1-9][0-9]*$/.test(value) ? Number(value) : undefined; + const pid = Number(value); + if (!/^[1-9][0-9]*$/.test(value) || !Number.isSafeInteger(pid)) { + throw new Error(`Portable profile fixture PID file ${pidFile} is invalid.`); + } + return pid; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; @@ -82,3 +86,11 @@ export async function cleanupPortableProfileSystemctlFixture(runtimeDir: string) fs.rmSync(artifact, { force: true }); } } + +export async function cleanupPortableProfileRootlessFixture( + runtimeDir: string, + root: string, +): Promise { + await cleanupPortableProfileSystemctlFixture(runtimeDir); + fs.rmSync(root, { force: true, recursive: true }); +} diff --git a/test/e2e/live/portable-profile-rootless-linux.test.ts b/test/e2e/live/portable-profile-rootless-linux.test.ts index bc334da29b8..91015fd2800 100644 --- a/test/e2e/live/portable-profile-rootless-linux.test.ts +++ b/test/e2e/live/portable-profile-rootless-linux.test.ts @@ -15,7 +15,7 @@ import * as importedSandboxPrebuild from "../../../src/lib/onboard/sandbox-prebu import * as importedBuildContext from "../../../src/lib/sandbox/build-context.ts"; import { test } from "../fixtures/e2e-test.ts"; import { - cleanupPortableProfileSystemctlFixture, + cleanupPortableProfileRootlessFixture, installPortableProfileSystemctlShim, } from "../fixtures/portable-profile-systemctl.ts"; import type { TestProgress } from "../fixtures/progress.ts"; @@ -262,12 +262,7 @@ async function main(progress: TestProgress): Promise { stdio: "ignore", timeout: 15_000, }); - try { - await cleanupPortableProfileSystemctlFixture(runtimeDir); - fs.rmSync(root, { recursive: true, force: true }); - } catch (error) { - console.warn(`Portable E2E temporary cleanup was incomplete: ${String(error)}`); - } + await cleanupPortableProfileRootlessFixture(runtimeDir, root); } } diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index b8aa1abe70d..0a59a761d09 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -9,6 +9,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { + cleanupPortableProfileRootlessFixture, cleanupPortableProfileSystemctlFixture, installPortableProfileSystemctlShim, } from "../fixtures/portable-profile-systemctl.ts"; @@ -135,8 +136,7 @@ function pidIsActive(pid: number): boolean { } async function cleanFixture(scope: FixtureScope): Promise { - await cleanupPortableProfileSystemctlFixture(scope.runtimeDir); - fs.rmSync(scope.directory, { force: true, recursive: true }); + await cleanupPortableProfileRootlessFixture(scope.runtimeDir, scope.directory); } function runInstallerOverride(scope: FixtureScope): ReturnType { @@ -295,6 +295,26 @@ describe("portable profile systemctl fixture", () => { }, ); + it.each([ + ["malformed PID text", "not-a-pid"], + ["a PID beyond Number.MAX_SAFE_INTEGER", `${Number.MAX_SAFE_INTEGER}0`], + ])("rejects %s without removing the rootless fixture (#9006)", async (_kind, invalidPid) => { + const scope = createFixture(); + const pidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + try { + fs.writeFileSync(pidFile, `${invalidPid}\n`, { mode: 0o600 }); + + await expect( + cleanupPortableProfileRootlessFixture(scope.runtimeDir, scope.directory), + ).rejects.toThrow(`Portable profile fixture PID file ${pidFile} is invalid.`); + expect(fs.existsSync(pidFile)).toBe(true); + expect(fs.existsSync(scope.directory)).toBe(true); + } finally { + fs.rmSync(pidFile, { force: true }); + await cleanFixture(scope); + } + }); + it("rejects malformed or extended user-service commands (#9006)", () => { const scope = createFixture(); try { From 3f3d2abb6233e178b0267e194c52465f0aee5138 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 06:38:45 -0700 Subject: [PATCH 09/13] ci(e2e): clean portable workflow fixture Signed-off-by: Senthil Ravichandran --- .github/workflows/portable-profile-e2e.yaml | 10 ++++----- .../portable-profile-systemctl-shim.test.ts | 21 +++++++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/portable-profile-e2e.yaml b/.github/workflows/portable-profile-e2e.yaml index 6dd5041f48d..5b011131e7a 100644 --- a/.github/workflows/portable-profile-e2e.yaml +++ b/.github/workflows/portable-profile-e2e.yaml @@ -145,10 +145,10 @@ jobs: export PATH="$shim_dir:$PATH" export XDG_RUNTIME_DIR="$runtime_dir" - systemctl --user start podman.socket printf '%s\n' "$shim_dir" >>"$GITHUB_PATH" printf 'XDG_RUNTIME_DIR=%s\n' "$runtime_dir" >>"$GITHUB_ENV" printf 'DOCKER_HOST=unix://%s/podman/podman.sock\n' "$runtime_dir" >>"$GITHUB_ENV" + systemctl --user start podman.socket podman --version docker --version docker --host "unix://$runtime_dir/podman/podman.sock" info @@ -175,7 +175,7 @@ jobs: shell: bash run: | podman system reset --force || true - pid_file="${XDG_RUNTIME_DIR}/nemoclaw-podman-service.pid" - if [[ -f "$pid_file" ]]; then - kill "$(<"$pid_file")" 2>/dev/null || true - fi + runtime_dir="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" + node --experimental-strip-types --no-warnings --input-type=module \ + --eval 'import { cleanupPortableProfileSystemctlFixture } from "./test/e2e/fixtures/portable-profile-systemctl.ts"; await cleanupPortableProfileSystemctlFixture(process.argv[1]);' \ + "$runtime_dir" diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index 0a59a761d09..fa7821912d4 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -154,10 +154,10 @@ function runInstallerOverride(scope: FixtureScope): ReturnType }); } -function portableLaunchProvisionStep(): WorkflowStep { +function portableLaunchStep(name: string): WorkflowStep { const workflow = readYaml(".github/workflows/portable-profile-e2e.yaml"); const step = workflow.jobs["portable-launch"]?.steps?.find( - (candidate) => candidate.name === "Provision restricted rootless Linux runtime", + (candidate) => candidate.name === name, ); expect(step).toBeDefined(); return step!; @@ -346,11 +346,24 @@ describe("portable profile systemctl fixture", () => { } }); - it("binds the portable-launch workflow to the shared systemctl fixture (#9006)", () => { - const provision = portableLaunchProvisionStep().run ?? ""; + it("binds portable-launch setup and always-run cleanup to the shared systemctl fixture (#9006)", () => { + const provision = portableLaunchStep("Provision restricted rootless Linux runtime").run ?? ""; expect(provision).toContain( 'install -m 700 test/e2e/fixtures/portable-profile-systemctl-shim.sh "$shim_dir/systemctl"', ); expect(provision).toContain("systemctl --user start podman.socket"); + const runtimeExportIndex = provision.indexOf("XDG_RUNTIME_DIR=%s"); + expect(runtimeExportIndex).toBeGreaterThanOrEqual(0); + expect(runtimeExportIndex).toBeLessThan( + provision.indexOf("systemctl --user start podman.socket"), + ); + + const cleanup = portableLaunchStep("Clean up portable runtime"); + expect(cleanup.if).toBe("always()"); + expect(cleanup.run).toContain('runtime_dir="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"'); + expect(cleanup.run).toContain( + 'import { cleanupPortableProfileSystemctlFixture } from "./test/e2e/fixtures/portable-profile-systemctl.ts"; await cleanupPortableProfileSystemctlFixture(process.argv[1]);', + ); + expect(cleanup.run).toContain('"$runtime_dir"'); }); }); From 6c1dcfddb32e3d01375e557019808cb9f586134c Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 07:17:31 -0700 Subject: [PATCH 10/13] fix(e2e): serialize portable fixture refresh Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.sh | 157 ++++++++++++---- .../portable-profile-systemctl-shim.test.ts | 167 +++++++++++++++++- 2 files changed, 288 insertions(+), 36 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index b01f2228301..215ae1f29a8 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -63,31 +63,30 @@ stop_runtime() { rm -f "$socket_path" "$backend_socket_path" } -wait_for_service() { +refresh_service() { + if ! service_is_active; then + return 0 + fi + + local previous_pid activator_pid + previous_pid="$(<"$service_pid_file")" + activator_pid="$(<"$activator_pid_file")" + kill -HUP "$activator_pid" + for ((attempt = 0; attempt < 100; attempt += 1)); do - if service_is_active; then - chmod 660 "$socket_path" + if service_is_active && [[ "$(<"$service_pid_file")" != "$previous_pid" ]]; then return 0 fi - if ! pid_is_active "$service_pid_file"; then + if ! pid_is_active "$activator_pid_file"; then break fi sleep 0.1 done - stop_service cat "$log_file" >&2 || true return 1 } -start_service() { - stop_service - install -d -m 700 "$service_dir" - nohup podman system service --time=0 "unix://$backend_socket_path" >>"$log_file" 2>&1 & - echo $! >"$service_pid_file" - wait_for_service -} - start_socket() { if socket_is_ready; then return 0 @@ -106,22 +105,52 @@ const net = require("node:net"); const [socketPath, backendSocketPath, servicePidFile, activatorPidFile] = process.argv.slice(2); const logFile = process.env.NEMOCLAW_PODMAN_LOG_FILE; -let activationPromise; +const refreshGate = process.env.NEMOCLAW_PODMAN_REFRESH_GATE; +let lifecycleTail = Promise.resolve(); function removeActivatorState() { fs.rmSync(activatorPidFile, { force: true }); } -function pidIsActive() { +function readServicePid() { + try { + const value = fs.readFileSync(servicePidFile, "utf8").trim(); + const pid = Number(value); + if (!/^[1-9][0-9]*$/.test(value) || !Number.isSafeInteger(pid)) { + throw new Error(`Portable profile fixture PID file ${servicePidFile} is invalid.`); + } + return pid; + } catch (error) { + if (error.code === "ENOENT") return undefined; + throw error; + } +} + +function processIsActive(pid) { try { - const pid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); process.kill(pid, 0); - return Number.isInteger(pid) && pid > 0; - } catch { + return true; + } catch (error) { + if (error.code !== "ESRCH") throw error; return false; } } +function signalProcess(pid, signal) { + try { + process.kill(pid, signal); + return true; + } catch (error) { + if (error.code !== "ESRCH") throw error; + return false; + } +} + +function pidIsActive() { + const pid = readServicePid(); + return pid !== undefined && processIsActive(pid); +} + function backendIsReady() { try { return pidIsActive() && fs.statSync(backendSocketPath).isSocket(); @@ -131,13 +160,42 @@ function backendIsReady() { } } -function serviceIsRunning(service) { - return service.exitCode === null && service.signalCode === null; +async function waitForProcessExit(pid) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (!processIsActive(pid)) return true; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return false; +} + +async function stopService() { + const pid = readServicePid(); + if (pid !== undefined && processIsActive(pid)) { + const termSent = signalProcess(pid, "SIGTERM"); + if (termSent && !(await waitForProcessExit(pid))) { + const killSent = signalProcess(pid, "SIGKILL"); + if (killSent && !(await waitForProcessExit(pid))) { + throw new Error(`Portable profile fixture process ${pid} did not exit.`); + } + } + } + fs.rmSync(servicePidFile, { force: true }); + fs.rmSync(backendSocketPath, { force: true }); +} + +async function waitForRefreshGate() { + if (!refreshGate) return; + fs.writeFileSync(`${refreshGate}.waiting`, `${process.pid}\n`, { mode: 0o600 }); + for (let attempt = 0; attempt < 100; attempt += 1) { + if (fs.existsSync(`${refreshGate}.release`)) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("Timed out waiting for the test to release the portable profile refresh gate."); } async function startService() { if (backendIsReady()) return; - fs.rmSync(backendSocketPath, { force: true }); + await stopService(); const output = fs.openSync(logFile, "a"); const service = spawn( "podman", @@ -151,25 +209,57 @@ async function startService() { for (let attempt = 0; attempt < 100; attempt += 1) { if (backendIsReady()) return; - if (!serviceIsRunning(service)) break; + if (!processIsActive(service.pid)) break; await new Promise((resolve) => setTimeout(resolve, 100)); } - if (serviceIsRunning(service)) service.kill("SIGTERM"); - fs.rmSync(servicePidFile, { force: true }); - fs.rmSync(backendSocketPath, { force: true }); + await stopService(); throw new Error("Podman service did not create its backend socket."); } +async function refreshService() { + await stopService(); + await waitForRefreshGate(); + await startService(); +} + +function runLifecycle(operation) { + const result = lifecycleTail.then(operation, operation); + lifecycleTail = result.catch(() => undefined); + return result; +} + +function connectBackend(client) { + return new Promise((resolve, reject) => { + const backend = net.createConnection(backendSocketPath); + const fail = (error) => { + client.off("close", clientClosed); + backend.destroy(); + reject(error); + }; + const clientClosed = () => fail(new Error("Portable profile client closed before proxying.")); + client.once("close", clientClosed); + backend.once("error", fail); + backend.once("connect", () => { + client.off("close", clientClosed); + backend.off("error", fail); + resolve(backend); + }); + }); +} + async function proxy(client) { - activationPromise ??= startService().finally(() => { - activationPromise = undefined; + const lifecycle = runLifecycle(async () => { + await startService(); + return connectBackend(client); }); - await activationPromise; - const backend = net.createConnection(backendSocketPath); - backend.once("connect", () => client.pipe(backend).pipe(client)); + if (refreshGate && fs.existsSync(`${refreshGate}.waiting`)) { + fs.writeFileSync(`${refreshGate}.client`, `${process.pid}\n`, { mode: 0o600 }); + } + const backend = await lifecycle; backend.once("error", () => client.destroy()); client.once("error", () => backend.destroy()); + client.pipe(backend).pipe(client); } const server = net.createServer((client) => { @@ -188,6 +278,9 @@ const stop = () => { }; process.on("SIGINT", stop); process.on("SIGTERM", stop); +process.on("SIGHUP", () => { + void runLifecycle(refreshService).catch((error) => console.error(error)); +}); NODE echo $! >"$activator_pid_file" @@ -218,9 +311,7 @@ if [[ "$#" -eq 3 && "$1" == "--user" && "$2" == "try-restart" && "$3" == "podman.service" ]]; then - if service_is_active; then - start_service - fi + refresh_service exit 0 fi diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index fa7821912d4..5ab3668e7e5 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import net from "node:net"; import path from "node:path"; @@ -57,13 +57,26 @@ if ( ) { process.exit(64); } +fs.appendFileSync(process.env.FAKE_PODMAN_PID_LOG, process.pid + "\\n"); const socketPath = args[3].slice("unix://".length); fs.rmSync(socketPath, { force: true }); +const sockets = new Set(); const server = net.createServer((socket) => { - socket.once("data", () => socket.end("ready")); + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + socket.once("data", (data) => { + if (data.toString() === "hold") { + socket.write("held"); + return; + } + socket.end("ready"); + }); }); server.listen(socketPath); -const stop = () => server.close(() => process.exit(0)); +const stop = () => { + for (const socket of sockets) socket.destroy(); + server.close(() => process.exit(0)); +}; process.on("SIGINT", stop); process.on("SIGTERM", stop); `, @@ -74,6 +87,7 @@ process.on("SIGTERM", stop); directory, env: { ...process.env, + FAKE_PODMAN_PID_LOG: path.join(directory, "podman-pids.log"), FAKE_PODMAN_SOCKET: socketPath, PATH: `${binDir}:${process.env.PATH ?? ""}`, XDG_RUNTIME_DIR: runtimeDir, @@ -92,6 +106,35 @@ function systemctl(scope: FixtureScope, args: string[]): ReturnType { + return new Promise((resolve, reject) => { + const child = spawn(scope.shim, args, { + env: scope.env, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`Timed out waiting for systemctl ${args.join(" ")}.`)); + }, 15_000); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + child.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once("close", (status) => { + clearTimeout(timeout); + resolve({ status, stderr }); + }); + }); +} + function serviceStatus(scope: FixtureScope): number | null { return systemctl(scope, ["--user", "is-active", "--quiet", "podman.service"]).status; } @@ -118,6 +161,27 @@ function activateThroughSocket(socketPath: string): Promise { }); } +function openHeldSocket(socketPath: string): Promise { + return new Promise((resolve, reject) => { + const client = net.createConnection(socketPath); + const timeout = setTimeout(() => { + client.destroy(); + reject(new Error("Timed out waiting for the held Podman client.")); + }, 15_000); + client.setEncoding("utf8"); + client.once("connect", () => client.write("hold")); + client.once("data", (chunk) => { + clearTimeout(timeout); + expect(chunk).toBe("held"); + resolve(client); + }); + client.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + }); +} + async function waitForServiceStatus(scope: FixtureScope, expected: number): Promise { await vi.waitFor(() => expect(serviceStatus(scope)).toBe(expected), { interval: 50, @@ -125,6 +189,13 @@ async function waitForServiceStatus(scope: FixtureScope, expected: number): Prom }); } +async function waitForPath(filePath: string): Promise { + await vi.waitFor(() => expect(fs.existsSync(filePath)).toBe(true), { + interval: 50, + timeout: 5_000, + }); +} + function pidIsActive(pid: number): boolean { try { process.kill(pid, 0); @@ -254,6 +325,96 @@ describe("portable profile systemctl fixture", () => { }, ); + it( + "serializes try-restart with a public-socket request and leaves only the recorded backend process active (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const refreshGate = path.join(scope.directory, "refresh-gate"); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + const backendSocketPath = path.join( + scope.runtimeDir, + "podman", + "nemoclaw-podman-service.sock", + ); + const pidLog = scope.env.FAKE_PODMAN_PID_LOG!; + scope.env.NEMOCLAW_PODMAN_REFRESH_GATE = refreshGate; + let backendPids: number[] = []; + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + const socketAuthority = fs.statSync(scope.socketPath); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + await waitForServiceStatus(scope, 0); + const previousPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + + const refresh = systemctlAsync(scope, ["--user", "try-restart", "podman.service"]); + await waitForPath(`${refreshGate}.waiting`); + const response = activateThroughSocket(scope.socketPath); + await waitForPath(`${refreshGate}.client`); + fs.writeFileSync(`${refreshGate}.release`, "release\n", { mode: 0o600 }); + + const [refreshResult, responseOutput] = await Promise.all([refresh, response]); + expect(refreshResult.status, refreshResult.stderr).toBe(0); + expect(responseOutput).toBe("ready"); + await waitForServiceStatus(scope, 0); + const recordedPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + backendPids = fs.readFileSync(pidLog, "utf8").trim().split("\n").map(Number); + expect(recordedPid).not.toBe(previousPid); + expect(pidIsActive(previousPid)).toBe(false); + expect(backendPids).toEqual([previousPid, recordedPid]); + expect(backendPids.filter(pidIsActive)).toEqual([recordedPid]); + expect(fs.statSync(scope.socketPath)).toMatchObject({ + dev: socketAuthority.dev, + ino: socketAuthority.ino, + }); + + await cleanupPortableProfileSystemctlFixture(scope.runtimeDir); + expect(backendPids.every((pid) => !pidIsActive(pid))).toBe(true); + for (const artifact of [ + activatorPidFile, + servicePidFile, + scope.socketPath, + backendSocketPath, + ]) { + expect(fs.existsSync(artifact), artifact).toBe(false); + } + } finally { + await cleanFixture(scope); + } + }, + ); + + it( + "refreshes the backend while an established public-socket client remains open (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + let heldClient: net.Socket | undefined; + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + await waitForServiceStatus(scope, 0); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const previousPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + + heldClient = await openHeldSocket(scope.socketPath); + expect(heldClient.destroyed).toBe(false); + const refresh = await systemctlAsync(scope, ["--user", "try-restart", "podman.service"]); + + expect(refresh.status, refresh.stderr).toBe(0); + await waitForServiceStatus(scope, 0); + const recordedPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + expect(recordedPid).not.toBe(previousPid); + expect(pidIsActive(previousPid)).toBe(false); + expect(pidIsActive(recordedPid)).toBe(true); + } finally { + heldClient?.destroy(); + await cleanFixture(scope); + } + }, + ); + it( "stops both fixture processes and removes both sockets during cleanup (#9006)", { timeout: 30_000 }, From 6de3e4d7d8d0027085372931dddd203ee9af0bef Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 09:32:00 -0700 Subject: [PATCH 11/13] test(e2e): bind portable fixture process identity Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.sh | 719 ++++++++++++++++-- .../fixtures/portable-profile-systemctl.ts | 160 +++- .../portable-profile-systemctl-shim.test.ts | 292 ++++++- 3 files changed, 1081 insertions(+), 90 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index 215ae1f29a8..9fff46838f5 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -11,73 +11,437 @@ backend_socket_path="${service_dir}/nemoclaw-podman-service.sock" activator_pid_file="${runtime_dir}/nemoclaw-podman-socket-activator.pid" service_pid_file="${runtime_dir}/nemoclaw-podman-service.pid" log_file="${runtime_dir}/nemoclaw-podman-service.log" +process_identity_env="NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID" +process_identity_failure_role="${NEMOCLAW_PODMAN_IDENTITY_FAILURE_ROLE:-}" +process_identity_failure_record="${NEMOCLAW_PODMAN_IDENTITY_FAILURE_RECORD:-}" + +process_start_time() { + local pid="$1" + if [[ -r "/proc/${pid}/stat" ]]; then + local stat fields + stat="$(<"/proc/${pid}/stat")" + stat="${stat##*) }" + read -r -a fields <<<"$stat" + [[ "${#fields[@]}" -gt 19 && "${fields[19]}" =~ ^[0-9]+$ ]] || return 1 + printf 'proc:%s\n' "${fields[19]}" + return 0 + fi + [[ ! -e /proc/self/stat ]] || return 1 + + local start_time + start_time="$(ps -o lstart= -p "$pid" 2>/dev/null)" || return 1 + start_time="${start_time#"${start_time%%[![:space:]]*}"}" + start_time="${start_time%"${start_time##*[![:space:]]}"}" + [[ -n "$start_time" ]] || return 1 + printf 'ps:%s\n' "$start_time" +} + +process_has_identity() { + local pid="$1" + local identity="$2" + local expected="${process_identity_env}=${identity}" + if [[ -r "/proc/${pid}/environ" ]]; then + local variable + while IFS= read -r -d '' variable; do + [[ "$variable" == "$expected" ]] && return 0 + done <"/proc/${pid}/environ" + return 1 + fi + [[ ! -e /proc/self/environ ]] || return 1 + + local command_line + command_line="$(ps eww -p "$pid" -o command= 2>/dev/null)" || return 1 + [[ " ${command_line} " == *" ${expected} "* ]] +} + +acquired_process_start_time="" + +acquire_process_identity() { + local pid="$1" + local identity="$2" + local current_start_time + for ((attempt = 0; attempt < 100; attempt += 1)); do + if current_start_time="$(process_start_time "$pid")" \ + && process_has_identity "$pid" "$identity"; then + acquired_process_start_time="$current_start_time" + return 0 + fi + kill -0 "$pid" 2>/dev/null || return 1 + sleep 0.05 + done + return 1 +} + +unrecorded_process_status() { + local pid="$1" + local identity="$2" + local start_time="$3" + if ! kill -0 "$pid" 2>/dev/null || process_is_zombie "$pid"; then + return 1 + fi + local current_start_time + if [[ -n "$start_time" ]]; then + if current_start_time="$(process_start_time "$pid")"; then + if [[ "$current_start_time" != "$start_time" ]]; then + echo "Portable profile fixture process ${pid} no longer matches its acquired start time." >&2 + return 2 + fi + elif kill -0 "$pid" 2>/dev/null; then + echo "Portable profile fixture could not revalidate the start time for process ${pid}." >&2 + return 2 + else + return 1 + fi + fi + if ! process_has_identity "$pid" "$identity"; then + kill -0 "$pid" 2>/dev/null || return 1 + echo "Portable profile fixture process ${pid} no longer matches its acquired identity." >&2 + return 2 + fi +} + +unrecorded_process_matches_acquired_start_time() { + local pid="$1" + local start_time="$2" + [[ -n "$start_time" ]] || return 2 + if ! kill -0 "$pid" 2>/dev/null || process_is_zombie "$pid"; then + return 1 + fi + local current_start_time + if current_start_time="$(process_start_time "$pid")"; then + : + elif kill -0 "$pid" 2>/dev/null; then + return 2 + else + return 1 + fi + [[ "$current_start_time" == "$start_time" ]] || return 2 +} + +unrecorded_process_matches_acquired_identity() { + local pid="$1" + local identity="$2" + local start_time="$3" + if [[ -n "$start_time" ]]; then + unrecorded_process_matches_acquired_start_time "$pid" "$start_time" + else + unrecorded_process_status "$pid" "$identity" "$start_time" + fi +} -pid_is_active() { +signal_unrecorded_process() { + local pid="$1" + local identity="$2" + local start_time="$3" + local signal="$4" + local status + if unrecorded_process_status "$pid" "$identity" "$start_time"; then + : + else + status=$? + return "$status" + fi + kill -"$signal" "$pid" 2>/dev/null || { + kill -0 "$pid" 2>/dev/null && return 2 + return 1 + } +} + +stop_unrecorded_process() { + local pid="$1" + local identity="$2" + local start_time="$3" + local status + if signal_unrecorded_process "$pid" "$identity" "$start_time" TERM; then + : + else + status=$? + [[ "$status" -eq 1 ]] && return 0 + return "$status" + fi + for ((attempt = 0; attempt < 100; attempt += 1)); do + if unrecorded_process_matches_acquired_identity "$pid" "$identity" "$start_time"; then + sleep 0.05 + continue + fi + status=$? + if [[ "$status" -eq 1 ]]; then + return 0 + fi + return "$status" + done + + if signal_unrecorded_process "$pid" "$identity" "$start_time" KILL; then + : + else + status=$? + [[ "$status" -eq 1 ]] && return 0 + return "$status" + fi + for ((attempt = 0; attempt < 100; attempt += 1)); do + if unrecorded_process_matches_acquired_identity "$pid" "$identity" "$start_time"; then + sleep 0.05 + continue + fi + status=$? + if [[ "$status" -eq 1 ]]; then + return 0 + fi + return "$status" + done + echo "Portable profile fixture process ${pid} did not exit." >&2 + return 1 +} + +process_is_zombie() { + local pid="$1" + if [[ -r "/proc/${pid}/stat" ]]; then + local stat fields + stat="$(<"/proc/${pid}/stat")" + stat="${stat##*) }" + read -r -a fields <<<"$stat" + [[ "${fields[0]:-}" == Z ]] + return + fi + [[ ! -e /proc/self/stat ]] || return 1 + + local process_state + process_state="$(ps -o stat= -p "$pid" 2>/dev/null)" || return 1 + [[ "$process_state" == Z* ]] +} + +recorded_pid="" +recorded_start_time="" +recorded_identity="" + +pid_is_safe_integer_text() { + local pid="$1" + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1 + [[ "${#pid}" -lt 16 || + ("${#pid}" -eq 16 && "$pid" -lt 9007199254740992) ]] +} + +read_pid_record() { local pid_file="$1" + local role="$2" [[ -f "$pid_file" ]] || return 1 - local pid - pid="$(<"$pid_file")" - [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1 - kill -0 "$pid" 2>/dev/null + local value extra + value="$(<"$pid_file")" + IFS=$'\t' read -r recorded_pid recorded_start_time recorded_identity extra <<<"$value" + if ! pid_is_safe_integer_text "$recorded_pid" \ + || [[ -n "${extra:-}" || + "$value" != "${recorded_pid}"$'\t'"${recorded_start_time}"$'\t'"${recorded_identity}" || + ! "$recorded_start_time" =~ ^(proc:[0-9]+|ps:.+)$ || + ! "$recorded_identity" =~ ^${role}:[0-9a-f]{32}$ ]]; then + echo "Portable profile fixture PID file ${pid_file} is invalid." >&2 + return 2 + fi +} + +recorded_process_status() { + local pid_file="$1" + local role="$2" + local status + if read_pid_record "$pid_file" "$role"; then + : + else + status=$? + return "$status" + fi + if ! kill -0 "$recorded_pid" 2>/dev/null; then + return 1 + fi + process_is_zombie "$recorded_pid" && return 1 + + local current_start_time + if current_start_time="$(process_start_time "$recorded_pid")"; then + : + elif kill -0 "$recorded_pid" 2>/dev/null; then + echo "Portable profile fixture PID file ${pid_file} cannot verify process ${recorded_pid}." >&2 + return 2 + else + return 1 + fi + if [[ "$current_start_time" != "$recorded_start_time" ]] \ + || ! process_has_identity "$recorded_pid" "$recorded_identity"; then + echo "Portable profile fixture PID file ${pid_file} does not match process ${recorded_pid}." >&2 + return 2 + fi +} + +recorded_process_has_recorded_start_time() { + local pid_file="$1" + local role="$2" + local status + if read_pid_record "$pid_file" "$role"; then + : + else + status=$? + return "$status" + fi + if ! kill -0 "$recorded_pid" 2>/dev/null || process_is_zombie "$recorded_pid"; then + return 1 + fi + + local current_start_time + if current_start_time="$(process_start_time "$recorded_pid")"; then + : + elif kill -0 "$recorded_pid" 2>/dev/null; then + echo "Portable profile fixture PID file ${pid_file} cannot verify process ${recorded_pid}." >&2 + return 2 + else + return 1 + fi + if [[ "$current_start_time" != "$recorded_start_time" ]]; then + echo "Portable profile fixture PID file ${pid_file} does not match process ${recorded_pid}." >&2 + return 2 + fi +} + +signal_recorded_process() { + local pid_file="$1" + local role="$2" + local signal="$3" + local status + if recorded_process_status "$pid_file" "$role"; then + : + else + status=$? + return "$status" + fi + kill -"$signal" "$recorded_pid" 2>/dev/null || { + kill -0 "$recorded_pid" 2>/dev/null && return 2 + return 1 + } +} + +recorded_process_is_active() { + recorded_process_status "$1" "$2" } service_is_active() { - pid_is_active "$activator_pid_file" \ - && pid_is_active "$service_pid_file" \ - && [[ -S "$socket_path" ]] \ - && [[ -S "$backend_socket_path" ]] + local status + if recorded_process_is_active "$activator_pid_file" activator; then + : + else + status=$? + return "$status" + fi + if recorded_process_is_active "$service_pid_file" service; then + : + else + status=$? + return "$status" + fi + [[ -S "$socket_path" ]] && [[ -S "$backend_socket_path" ]] } socket_is_ready() { - [[ -S "$socket_path" ]] && pid_is_active "$activator_pid_file" + [[ -S "$socket_path" ]] || return 1 + recorded_process_is_active "$activator_pid_file" activator } -stop_pid() { +stop_recorded_process() { local pid_file="$1" + local role="$2" [[ -f "$pid_file" ]] || return 0 - local pid - pid="$(<"$pid_file")" - if [[ "$pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$pid" 2>/dev/null; then - kill "$pid" 2>/dev/null || true - for ((attempt = 0; attempt < 100; attempt += 1)); do - if ! kill -0 "$pid" 2>/dev/null; then - break - fi + local status pid + if recorded_process_status "$pid_file" "$role"; then + pid="$recorded_pid" + else + status=$? + if [[ "$status" -eq 1 ]]; then + rm -f "$pid_file" + return 0 + fi + return "$status" + fi + + if signal_recorded_process "$pid_file" "$role" TERM; then + : + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" + fi + for ((attempt = 0; attempt < 100; attempt += 1)); do + if recorded_process_has_recorded_start_time "$pid_file" "$role"; then sleep 0.05 - done - if kill -0 "$pid" 2>/dev/null; then - kill -KILL "$pid" 2>/dev/null || true + continue + fi + status=$? + if [[ "$status" -eq 1 ]]; then + rm -f "$pid_file" + return 0 fi + return "$status" + done + + if signal_recorded_process "$pid_file" "$role" KILL; then + : + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" fi - rm -f "$pid_file" + for ((attempt = 0; attempt < 100; attempt += 1)); do + if recorded_process_has_recorded_start_time "$pid_file" "$role"; then + sleep 0.05 + continue + fi + status=$? + if [[ "$status" -eq 1 ]]; then + rm -f "$pid_file" + return 0 + fi + return "$status" + done + echo "Portable profile fixture process ${pid} did not exit." >&2 + return 1 } stop_service() { - stop_pid "$service_pid_file" + stop_recorded_process "$service_pid_file" service rm -f "$backend_socket_path" } stop_runtime() { stop_service - stop_pid "$activator_pid_file" + stop_recorded_process "$activator_pid_file" activator rm -f "$socket_path" "$backend_socket_path" } refresh_service() { - if ! service_is_active; then - return 0 + local status + if service_is_active; then + : + else + status=$? + [[ "$status" -eq 1 ]] && return 0 + return "$status" fi - local previous_pid activator_pid - previous_pid="$(<"$service_pid_file")" - activator_pid="$(<"$activator_pid_file")" - kill -HUP "$activator_pid" + local previous_record + previous_record="$(<"$service_pid_file")" + signal_recorded_process "$activator_pid_file" activator HUP for ((attempt = 0; attempt < 100; attempt += 1)); do - if service_is_active && [[ "$(<"$service_pid_file")" != "$previous_pid" ]]; then - return 0 + if [[ -f "$service_pid_file" && "$(<"$service_pid_file")" != "$previous_record" ]]; then + if service_is_active; then + return 0 + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" + fi + elif recorded_process_has_recorded_start_time "$service_pid_file" service; then + : + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" fi - if ! pid_is_active "$activator_pid_file"; then + if recorded_process_is_active "$activator_pid_file" activator; then + : + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" break fi sleep 0.1 @@ -88,44 +452,72 @@ refresh_service() { } start_socket() { + local status if socket_is_ready; then return 0 + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" fi stop_runtime install -d -m 700 "$service_dir" NEMOCLAW_PODMAN_LOG_FILE="$log_file" export NEMOCLAW_PODMAN_LOG_FILE - nohup node - "$socket_path" "$backend_socket_path" "$service_pid_file" \ + local activator_identity activator_pid + activator_identity="activator:$(node -e 'process.stdout.write(require("node:crypto").randomBytes(16).toString("hex"))')" + NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID="$activator_identity" nohup node - "$socket_path" "$backend_socket_path" "$service_pid_file" \ "$activator_pid_file" \ >>"$log_file" 2>&1 <<'NODE' & -const { spawn } = require("node:child_process"); +const { spawn, spawnSync } = require("node:child_process"); +const { randomBytes } = require("node:crypto"); const fs = require("node:fs"); const net = require("node:net"); const [socketPath, backendSocketPath, servicePidFile, activatorPidFile] = process.argv.slice(2); const logFile = process.env.NEMOCLAW_PODMAN_LOG_FILE; const refreshGate = process.env.NEMOCLAW_PODMAN_REFRESH_GATE; +const processIdentityEnv = "NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID"; +const processIdentityFailureRole = process.env.NEMOCLAW_PODMAN_IDENTITY_FAILURE_ROLE; +const processIdentityFailureRecord = process.env.NEMOCLAW_PODMAN_IDENTITY_FAILURE_RECORD; +const processQueryTimeoutMs = 5000; let lifecycleTail = Promise.resolve(); function removeActivatorState() { fs.rmSync(activatorPidFile, { force: true }); } -function readServicePid() { +function readProcessRecord(pidFile, role) { try { - const value = fs.readFileSync(servicePidFile, "utf8").trim(); - const pid = Number(value); - if (!/^[1-9][0-9]*$/.test(value) || !Number.isSafeInteger(pid)) { - throw new Error(`Portable profile fixture PID file ${servicePidFile} is invalid.`); + const value = fs.readFileSync(pidFile, "utf8").trim(); + const [pidText, startTime, identity, ...extra] = value.split("\t"); + const pid = Number(pidText); + if ( + extra.length !== 0 || + !/^[1-9][0-9]*$/.test(pidText || "") || + !Number.isSafeInteger(pid) || + !/^(?:proc:[0-9]+|ps:.+)$/.test(startTime || "") || + !new RegExp(`^${role}:[0-9a-f]{32}$`).test(identity || "") + ) { + throw new Error(`Portable profile fixture PID file ${pidFile} is invalid.`); } - return pid; + return { identity, pid, pidFile, startTime }; } catch (error) { if (error.code === "ENOENT") return undefined; throw error; } } +function writeProcessRecord(pidFile, processIdentity) { + const temporaryPidFile = `${pidFile}.${process.pid}.tmp`; + fs.writeFileSync( + temporaryPidFile, + `${processIdentity.pid}\t${processIdentity.startTime}\t${processIdentity.identity}\n`, + { mode: 0o600 }, + ); + fs.renameSync(temporaryPidFile, pidFile); +} + function processIsActive(pid) { try { process.kill(pid, 0); @@ -136,7 +528,92 @@ function processIsActive(pid) { } } -function signalProcess(pid, signal) { +function processIsZombie(pid) { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + return stat.slice(stat.lastIndexOf(") ") + 2).trim().split(/\s+/)[0] === "Z"; + } catch (error) { + if (error.code !== "ENOENT") throw error; + if (fs.existsSync("/proc/self/stat")) return false; + } + + const result = spawnSync("ps", ["-o", "stat=", "-p", String(pid)], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: processQueryTimeoutMs, + }); + return result.status === 0 && result.stdout.trim().startsWith("Z"); +} + +function readProcessStartTime(pid) { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(") ") + 2).trim().split(/\s+/); + const startTime = fields[19]; + if (!startTime || !/^[0-9]+$/.test(startTime)) { + throw new Error(`Portable profile fixture process ${pid} has invalid /proc stat data.`); + } + return `proc:${startTime}`; + } catch (error) { + if (error.code !== "ENOENT") throw error; + if (fs.existsSync("/proc/self/stat")) return undefined; + } + + const result = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: processQueryTimeoutMs, + }); + const startTime = result.status === 0 ? result.stdout.trim().replace(/\s+/g, " ") : ""; + return startTime ? `ps:${startTime}` : undefined; +} + +function processHasIdentity(pid, identity) { + const expected = `${processIdentityEnv}=${identity}`; + try { + return fs.readFileSync(`/proc/${pid}/environ`, "utf8").split("\0").includes(expected); + } catch (error) { + if (error.code !== "ENOENT") throw error; + if (fs.existsSync("/proc/self/environ")) return false; + } + + const result = spawnSync("ps", ["eww", "-p", String(pid), "-o", "command="], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: processQueryTimeoutMs, + }); + return result.status === 0 && result.stdout.split(/\s+/).includes(expected); +} + +function recordedProcessIsActive(processIdentity) { + if (!processIsActive(processIdentity.pid)) return false; + if (processIsZombie(processIdentity.pid)) return false; + if ( + readProcessStartTime(processIdentity.pid) !== processIdentity.startTime || + !processHasIdentity(processIdentity.pid, processIdentity.identity) + ) { + if (!processIsActive(processIdentity.pid)) return false; + throw new Error( + `Portable profile fixture PID file ${processIdentity.pidFile} does not match process ${processIdentity.pid}.`, + ); + } + return true; +} + +function unrecordedProcessIsActive(pid, identity) { + if (!processIsActive(pid)) return false; + if (processIsZombie(pid)) return false; + if (!processHasIdentity(pid, identity)) { + if (!processIsActive(pid)) return false; + throw new Error( + `Portable profile fixture process ${pid} no longer matches its acquired identity.`, + ); + } + return true; +} + +function signalUnrecordedProcess(pid, identity, signal) { + if (!unrecordedProcessIsActive(pid, identity)) return false; try { process.kill(pid, signal); return true; @@ -146,39 +623,97 @@ function signalProcess(pid, signal) { } } -function pidIsActive() { - const pid = readServicePid(); - return pid !== undefined && processIsActive(pid); +async function acquireProcessIdentity(pid, identity, pidFile) { + for (let attempt = 0; attempt < 100; attempt += 1) { + const startTime = readProcessStartTime(pid); + if (startTime && processHasIdentity(pid, identity)) { + return { identity, pid, pidFile, startTime }; + } + if (!processIsActive(pid)) return undefined; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return undefined; +} + +function processHasRecordedStartTime(processIdentity) { + if (!processIsActive(processIdentity.pid)) return false; + if (processIsZombie(processIdentity.pid)) return false; + const startTime = readProcessStartTime(processIdentity.pid); + if (!startTime && !processIsActive(processIdentity.pid)) return false; + if (startTime !== processIdentity.startTime) { + throw new Error( + `Portable profile fixture PID file ${processIdentity.pidFile} does not match process ${processIdentity.pid}.`, + ); + } + return true; +} + +function signalProcess(processIdentity, signal) { + if (!recordedProcessIsActive(processIdentity)) return false; + try { + process.kill(processIdentity.pid, signal); + return true; + } catch (error) { + if (error.code !== "ESRCH") throw error; + return false; + } +} + +function serviceIsActive() { + const processIdentity = readProcessRecord(servicePidFile, "service"); + return processIdentity !== undefined && recordedProcessIsActive(processIdentity); } function backendIsReady() { try { - return pidIsActive() && fs.statSync(backendSocketPath).isSocket(); + return serviceIsActive() && fs.statSync(backendSocketPath).isSocket(); } catch (error) { if (error.code !== "ENOENT") throw error; return false; } } -async function waitForProcessExit(pid) { +async function waitForProcessExit(processIdentity) { for (let attempt = 0; attempt < 100; attempt += 1) { - if (!processIsActive(pid)) return true; + if (!processHasRecordedStartTime(processIdentity)) return true; await new Promise((resolve) => setTimeout(resolve, 50)); } return false; } -async function stopService() { - const pid = readServicePid(); - if (pid !== undefined && processIsActive(pid)) { - const termSent = signalProcess(pid, "SIGTERM"); - if (termSent && !(await waitForProcessExit(pid))) { - const killSent = signalProcess(pid, "SIGKILL"); - if (killSent && !(await waitForProcessExit(pid))) { - throw new Error(`Portable profile fixture process ${pid} did not exit.`); - } +async function terminateProcessIdentity(processIdentity) { + if (!recordedProcessIsActive(processIdentity)) return; + const termSent = signalProcess(processIdentity, "SIGTERM"); + if (termSent && !(await waitForProcessExit(processIdentity))) { + const killSent = signalProcess(processIdentity, "SIGKILL"); + if (killSent && !(await waitForProcessExit(processIdentity))) { + throw new Error(`Portable profile fixture process ${processIdentity.pid} did not exit.`); } } +} + +async function waitForUnrecordedProcessExit(pid, identity) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (!unrecordedProcessIsActive(pid, identity)) return true; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return false; +} + +async function terminateUnrecordedProcess(pid, identity) { + if (!unrecordedProcessIsActive(pid, identity)) return; + const termSent = signalUnrecordedProcess(pid, identity, "SIGTERM"); + if (termSent && !(await waitForUnrecordedProcessExit(pid, identity))) { + const killSent = signalUnrecordedProcess(pid, identity, "SIGKILL"); + if (killSent && !(await waitForUnrecordedProcessExit(pid, identity))) { + throw new Error(`Portable profile fixture process ${pid} did not exit.`); + } + } +} + +async function stopService() { + const processIdentity = readProcessRecord(servicePidFile, "service"); + if (processIdentity !== undefined) await terminateProcessIdentity(processIdentity); fs.rmSync(servicePidFile, { force: true }); fs.rmSync(backendSocketPath, { force: true }); } @@ -197,19 +732,47 @@ async function startService() { if (backendIsReady()) return; await stopService(); const output = fs.openSync(logFile, "a"); + const identity = `service:${randomBytes(16).toString("hex")}`; const service = spawn( "podman", ["system", "service", "--time=0", `unix://${backendSocketPath}`], - { detached: true, stdio: ["ignore", output, output] }, + { + detached: true, + env: { ...process.env, [processIdentityEnv]: identity }, + stdio: ["ignore", output, output], + }, ); fs.closeSync(output); if (!service.pid) throw new Error("Podman service did not report a process ID."); - fs.writeFileSync(servicePidFile, `${service.pid}\n`, { mode: 0o600 }); + let processIdentity; + try { + processIdentity = await acquireProcessIdentity(service.pid, identity, servicePidFile); + } catch (error) { + await terminateUnrecordedProcess(service.pid, identity); + throw error; + } + if (!processIdentity) { + await terminateUnrecordedProcess(service.pid, identity); + throw new Error( + `Portable profile fixture could not create the process identity record for service ${service.pid}.`, + ); + } + if (processIdentityFailureRole === "service") { + if (processIdentityFailureRecord) { + writeProcessRecord(processIdentityFailureRecord, processIdentity); + } + await terminateProcessIdentity(processIdentity); + fs.rmSync(backendSocketPath, { force: true }); + throw new Error( + `Portable profile fixture could not create the process identity record for service ${service.pid}.`, + ); + } + writeProcessRecord(servicePidFile, processIdentity); service.unref(); for (let attempt = 0; attempt < 100; attempt += 1) { if (backendIsReady()) return; - if (!processIsActive(service.pid)) break; + if (!recordedProcessIsActive(processIdentity)) break; await new Promise((resolve) => setTimeout(resolve, 100)); } @@ -282,13 +845,40 @@ process.on("SIGHUP", () => { void runLifecycle(refreshService).catch((error) => console.error(error)); }); NODE - echo $! >"$activator_pid_file" + activator_pid=$! + if acquire_process_identity "$activator_pid" "$activator_identity"; then + if [[ "$process_identity_failure_role" == activator ]]; then + if [[ -n "$process_identity_failure_record" ]]; then + printf '%s\t%s\t%s\n' "$activator_pid" "$acquired_process_start_time" "$activator_identity" \ + >"$process_identity_failure_record" + chmod 600 "$process_identity_failure_record" + fi + stop_unrecorded_process "$activator_pid" "$activator_identity" "$acquired_process_start_time" || true + echo "Portable profile fixture could not create the process identity record for activator ${activator_pid}." >&2 + cat "$log_file" >&2 || true + return 1 + fi + local activator_pid_file_tmp="${activator_pid_file}.$$.tmp" + printf '%s\t%s\t%s\n' "$activator_pid" "$acquired_process_start_time" "$activator_identity" \ + >"$activator_pid_file_tmp" + chmod 600 "$activator_pid_file_tmp" + mv "$activator_pid_file_tmp" "$activator_pid_file" + else + stop_unrecorded_process "$activator_pid" "$activator_identity" "" || true + echo "Portable profile fixture could not create the process identity record for activator ${activator_pid}." >&2 + cat "$log_file" >&2 || true + return 1 + fi for ((attempt = 0; attempt < 100; attempt += 1)); do if socket_is_ready; then return 0 fi - if ! pid_is_active "$activator_pid_file"; then + if recorded_process_is_active "$activator_pid_file" activator; then + : + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" break fi sleep 0.1 @@ -322,6 +912,9 @@ if [[ "$#" -eq 4 && "$4" == "podman.service" ]]; then if service_is_active; then exit 0 + else + status=$? + [[ "$status" -eq 1 ]] || exit "$status" fi exit 3 fi diff --git a/test/e2e/fixtures/portable-profile-systemctl.ts b/test/e2e/fixtures/portable-profile-systemctl.ts index 6dbd3f55e47..9d29c4f80e9 100644 --- a/test/e2e/fixtures/portable-profile-systemctl.ts +++ b/test/e2e/fixtures/portable-profile-systemctl.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,27 +10,46 @@ const SYSTEMCTL_SHIM_SOURCE = fileURLToPath( new URL("./portable-profile-systemctl-shim.sh", import.meta.url), ); +const FIXTURE_PROCESS_ID_ENV = "NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID"; +const PROCESS_QUERY_TIMEOUT_MS = 5_000; const FIXTURE_PID_FILES = [ - "nemoclaw-podman-socket-activator.pid", - "nemoclaw-podman-service.pid", + ["nemoclaw-podman-socket-activator.pid", "activator"], + ["nemoclaw-podman-service.pid", "service"], ] as const; const FIXTURE_SOCKET_FILES = ["podman.sock", "nemoclaw-podman-service.sock"] as const; -function readFixturePid(pidFile: string): number | undefined { +interface FixtureProcessIdentity { + readonly identity: string; + readonly pid: number; + readonly pidFile: string; + readonly startTime: string; +} + +function readFixtureProcessIdentity( + pidFile: string, + role: (typeof FIXTURE_PID_FILES)[number][1], +): FixtureProcessIdentity | undefined { try { const value = fs.readFileSync(pidFile, "utf8").trim(); - const pid = Number(value); - if (!/^[1-9][0-9]*$/.test(value) || !Number.isSafeInteger(pid)) { + const [pidText, startTime, identity, ...extra] = value.split("\t"); + const pid = Number(pidText); + if ( + extra.length !== 0 || + !/^[1-9][0-9]*$/.test(pidText ?? "") || + !Number.isSafeInteger(pid) || + !/^(?:proc:[0-9]+|ps:.+)$/.test(startTime ?? "") || + !new RegExp(`^${role}:[0-9a-f]{32}$`).test(identity ?? "") + ) { throw new Error(`Portable profile fixture PID file ${pidFile} is invalid.`); } - return pid; + return { identity, pid, pidFile, startTime }; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; } } -function fixtureProcessIsActive(pid: number): boolean { +function processIsActive(pid: number): boolean { try { process.kill(pid, 0); return true; @@ -39,31 +59,132 @@ function fixtureProcessIsActive(pid: number): boolean { } } -async function waitForFixtureProcessExit(pid: number): Promise { +function processIsZombie(pid: number): boolean { + try { + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf8"); + return ( + stat + .slice(stat.lastIndexOf(") ") + 2) + .trim() + .split(/\s+/)[0] === "Z" + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (fs.existsSync("/proc/self/stat")) return false; + } + + const result = spawnSync("ps", ["-o", "stat=", "-p", String(pid)], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: PROCESS_QUERY_TIMEOUT_MS, + }); + return result.status === 0 && result.stdout.trim().startsWith("Z"); +} + +function readProcessStartTime(pid: number): string | undefined { + try { + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf8"); + const fields = stat + .slice(stat.lastIndexOf(") ") + 2) + .trim() + .split(/\s+/); + const startTime = fields[19]; + if (!startTime || !/^[0-9]+$/.test(startTime)) { + throw new Error( + `Portable profile fixture process ${String(pid)} has invalid /proc stat data.`, + ); + } + return `proc:${startTime}`; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (fs.existsSync("/proc/self/stat")) return undefined; + } + + const result = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: PROCESS_QUERY_TIMEOUT_MS, + }); + const startTime = result.status === 0 ? result.stdout.trim().replace(/\s+/g, " ") : ""; + return startTime ? `ps:${startTime}` : undefined; +} + +function processHasIdentity(pid: number, identity: string): boolean { + const expected = `${FIXTURE_PROCESS_ID_ENV}=${identity}`; + try { + const environment = fs.readFileSync(`/proc/${String(pid)}/environ`, "utf8").split("\0"); + return environment.includes(expected); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (fs.existsSync("/proc/self/environ")) return false; + } + + const result = spawnSync("ps", ["eww", "-p", String(pid), "-o", "command="], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: PROCESS_QUERY_TIMEOUT_MS, + }); + return result.status === 0 && result.stdout.split(/\s+/).includes(expected); +} + +function fixtureProcessIsActive(processIdentity: FixtureProcessIdentity): boolean { + if (!processIsActive(processIdentity.pid)) return false; + if (processIsZombie(processIdentity.pid)) return false; + if ( + readProcessStartTime(processIdentity.pid) !== processIdentity.startTime || + !processHasIdentity(processIdentity.pid, processIdentity.identity) + ) { + throw new Error( + `Portable profile fixture PID file ${processIdentity.pidFile} does not match process ${String(processIdentity.pid)}.`, + ); + } + return true; +} + +function fixtureProcessHasRecordedStartTime(processIdentity: FixtureProcessIdentity): boolean { + if (!processIsActive(processIdentity.pid)) return false; + if (processIsZombie(processIdentity.pid)) return false; + const startTime = readProcessStartTime(processIdentity.pid); + if (!startTime && !processIsActive(processIdentity.pid)) return false; + if (startTime !== processIdentity.startTime) { + throw new Error( + `Portable profile fixture PID file ${processIdentity.pidFile} does not match process ${String(processIdentity.pid)}.`, + ); + } + return true; +} + +async function waitForFixtureProcessExit( + processIdentity: FixtureProcessIdentity, +): Promise { for (let attempt = 0; attempt < 100; attempt += 1) { - if (!fixtureProcessIsActive(pid)) return true; + if (!fixtureProcessHasRecordedStartTime(processIdentity)) return true; await new Promise((resolve) => setTimeout(resolve, 50)); } return false; } -async function terminateFixtureProcess(pid: number): Promise { +async function terminateFixtureProcess(processIdentity: FixtureProcessIdentity): Promise { + if (!fixtureProcessIsActive(processIdentity)) return; try { - process.kill(pid, "SIGTERM"); + process.kill(processIdentity.pid, "SIGTERM"); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return; throw error; } - if (await waitForFixtureProcessExit(pid)) return; + if (await waitForFixtureProcessExit(processIdentity)) return; + if (!fixtureProcessIsActive(processIdentity)) return; try { - process.kill(pid, "SIGKILL"); + process.kill(processIdentity.pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return; throw error; } - if (!(await waitForFixtureProcessExit(pid))) { - throw new Error(`Portable profile fixture process ${String(pid)} did not exit.`); + if (!(await waitForFixtureProcessExit(processIdentity))) { + throw new Error( + `Portable profile fixture process ${String(processIdentity.pid)} did not exit.`, + ); } } @@ -75,9 +196,12 @@ export function installPortableProfileSystemctlShim(binDir: string): string { } export async function cleanupPortableProfileSystemctlFixture(runtimeDir: string): Promise { - const pidFiles = FIXTURE_PID_FILES.map((name) => path.join(runtimeDir, name)); - const pids = pidFiles.map(readFixturePid).filter((pid): pid is number => pid !== undefined); - await Promise.all(pids.map(terminateFixtureProcess)); + const pidFiles = FIXTURE_PID_FILES.map(([name]) => path.join(runtimeDir, name)); + const processIdentities = FIXTURE_PID_FILES.map(([name, role]) => + readFixtureProcessIdentity(path.join(runtimeDir, name), role), + ).filter((identity): identity is FixtureProcessIdentity => identity !== undefined); + for (const processIdentity of processIdentities) fixtureProcessIsActive(processIdentity); + await Promise.all(processIdentities.map(terminateFixtureProcess)); for (const artifact of [ ...pidFiles, diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index 5ab3668e7e5..f094a6b4b99 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -26,6 +26,13 @@ interface FixtureScope { readonly socketPath: string; } +interface FixtureProcessRecord { + readonly identity: string; + readonly pid: number; + readonly startTime: string; + readonly value: string; +} + function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o700 }); } @@ -102,6 +109,7 @@ function systemctl(scope: FixtureScope, args: string[]): ReturnType { }); } +async function waitForFileText(filePath: string, text: string): Promise { + await vi.waitFor(() => expect(fs.readFileSync(filePath, "utf8")).toContain(text), { + interval: 50, + timeout: 5_000, + }); +} + function pidIsActive(pid: number): boolean { try { process.kill(pid, 0); @@ -206,6 +221,40 @@ function pidIsActive(pid: number): boolean { } } +function expectProcessActive(pid: number): void { + expect(pidIsActive(pid)).toBe(true); +} + +function readFixtureProcessRecord(pidFile: string): FixtureProcessRecord { + const value = fs.readFileSync(pidFile, "utf8").trim(); + const [pidText, startTime, identity] = value.split("\t"); + return { identity, pid: Number(pidText), startTime, value }; +} + +function replaceRecordedPid(record: FixtureProcessRecord, pid: number): string { + return `${String(pid)}\t${record.startTime}\t${record.identity}\n`; +} + +function spawnUnrelatedProcess(): ReturnType { + const child = spawn(process.execPath, ["-e", "setInterval(() => undefined, 1000)"], { + stdio: "ignore", + }); + expect(child.pid).toBeDefined(); + return child; +} + +async function stopUnrelatedProcess(child: ReturnType | undefined): Promise { + if (!child?.pid || child.exitCode !== null) return; + if (pidIsActive(child.pid)) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timeout = setTimeout(resolve, 5_000); + child.once("close", () => { + clearTimeout(timeout); + resolve(); + }); + }); +} + async function cleanFixture(scope: FixtureScope): Promise { await cleanupPortableProfileRootlessFixture(scope.runtimeDir, scope.directory); } @@ -221,6 +270,7 @@ function runInstallerOverride(scope: FixtureScope): ReturnType return spawnSync("bash", ["-c", script], { encoding: "utf8", env: scope.env, + killSignal: "SIGKILL", timeout: 15_000, }); } @@ -269,12 +319,12 @@ describe("portable profile systemctl fixture", () => { expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); - const firstPid = fs.readFileSync(servicePidFile, "utf8").trim(); + const firstProcess = readFixtureProcessRecord(servicePidFile); const refresh = systemctl(scope, ["--user", "try-restart", "podman.service"]); expect(refresh.status, String(refresh.stderr)).toBe(0); expect(serviceStatus(scope)).toBe(0); - expect(fs.readFileSync(servicePidFile, "utf8").trim()).not.toBe(firstPid); - expect(pidIsActive(Number(firstPid))).toBe(false); + expect(readFixtureProcessRecord(servicePidFile).value).not.toBe(firstProcess.value); + expect(pidIsActive(firstProcess.pid)).toBe(false); expect(fs.statSync(scope.socketPath)).toMatchObject({ dev: socketAuthority.dev, ino: socketAuthority.ino, @@ -346,7 +396,7 @@ describe("portable profile systemctl fixture", () => { const socketAuthority = fs.statSync(scope.socketPath); expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); await waitForServiceStatus(scope, 0); - const previousPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + const previousPid = readFixtureProcessRecord(servicePidFile).pid; const refresh = systemctlAsync(scope, ["--user", "try-restart", "podman.service"]); await waitForPath(`${refreshGate}.waiting`); @@ -358,7 +408,7 @@ describe("portable profile systemctl fixture", () => { expect(refreshResult.status, refreshResult.stderr).toBe(0); expect(responseOutput).toBe("ready"); await waitForServiceStatus(scope, 0); - const recordedPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + const recordedPid = readFixtureProcessRecord(servicePidFile).pid; backendPids = fs.readFileSync(pidLog, "utf8").trim().split("\n").map(Number); expect(recordedPid).not.toBe(previousPid); expect(pidIsActive(previousPid)).toBe(false); @@ -396,7 +446,7 @@ describe("portable profile systemctl fixture", () => { expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); await waitForServiceStatus(scope, 0); const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); - const previousPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + const previousPid = readFixtureProcessRecord(servicePidFile).pid; heldClient = await openHeldSocket(scope.socketPath); expect(heldClient.destroyed).toBe(false); @@ -404,7 +454,7 @@ describe("portable profile systemctl fixture", () => { expect(refresh.status, refresh.stderr).toBe(0); await waitForServiceStatus(scope, 0); - const recordedPid = Number(fs.readFileSync(servicePidFile, "utf8").trim()); + const recordedPid = readFixtureProcessRecord(servicePidFile).pid; expect(recordedPid).not.toBe(previousPid); expect(pidIsActive(previousPid)).toBe(false); expect(pidIsActive(recordedPid)).toBe(true); @@ -432,8 +482,8 @@ describe("portable profile systemctl fixture", () => { expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); await waitForServiceStatus(scope, 0); - const pids = [activatorPidFile, servicePidFile].map((pidFile) => - Number(fs.readFileSync(pidFile, "utf8").trim()), + const pids = [activatorPidFile, servicePidFile].map( + (pidFile) => readFixtureProcessRecord(pidFile).pid, ); expect(pids.every(pidIsActive)).toBe(true); expect(fs.statSync(scope.socketPath).isSocket()).toBe(true); @@ -456,6 +506,230 @@ describe("portable profile systemctl fixture", () => { }, ); + it( + "stops the owned activator when it cannot create the process identity record (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + const failureRecord = path.join(scope.runtimeDir, "activator-identity-failure.record"); + scope.env.NEMOCLAW_PODMAN_IDENTITY_FAILURE_ROLE = "activator"; + scope.env.NEMOCLAW_PODMAN_IDENTITY_FAILURE_RECORD = failureRecord; + try { + const start = systemctl(scope, ["--user", "start", "podman.socket"]); + expect(start.status).not.toBe(0); + expect(start.stderr).toContain( + "Portable profile fixture could not create the process identity record for activator", + ); + const processRecord = readFixtureProcessRecord(failureRecord); + expect(pidIsActive(processRecord.pid)).toBe(false); + expect(fs.existsSync(activatorPidFile)).toBe(false); + expect(fs.existsSync(scope.socketPath)).toBe(false); + } finally { + await cleanFixture(scope); + } + }, + ); + + it( + "stops the owned backend when it cannot create the process identity record (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const backendSocketPath = path.join( + scope.runtimeDir, + "podman", + "nemoclaw-podman-service.sock", + ); + const failureRecord = path.join(scope.runtimeDir, "service-identity-failure.record"); + scope.env.NEMOCLAW_PODMAN_IDENTITY_FAILURE_ROLE = "service"; + scope.env.NEMOCLAW_PODMAN_IDENTITY_FAILURE_RECORD = failureRecord; + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe(""); + await waitForPath(failureRecord); + const processRecord = readFixtureProcessRecord(failureRecord); + expect(pidIsActive(processRecord.pid)).toBe(false); + expect(fs.existsSync(servicePidFile)).toBe(false); + expect(fs.existsSync(backendSocketPath)).toBe(false); + } finally { + await cleanFixture(scope); + } + }, + ); + + it( + "rejects a reused activator PID during shared fixture cleanup without signaling the unrelated process (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + let originalRecord: FixtureProcessRecord | undefined; + let unrelated: ReturnType | undefined; + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + originalRecord = readFixtureProcessRecord(activatorPidFile); + unrelated = spawnUnrelatedProcess(); + await vi.waitFor(() => expect(pidIsActive(unrelated!.pid!)).toBe(true)); + fs.writeFileSync(activatorPidFile, replaceRecordedPid(originalRecord, unrelated.pid!), { + mode: 0o600, + }); + + await expect(cleanupPortableProfileSystemctlFixture(scope.runtimeDir)).rejects.toThrow( + `Portable profile fixture PID file ${activatorPidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + expect(fs.existsSync(activatorPidFile)).toBe(true); + expect(fs.existsSync(scope.socketPath)).toBe(true); + expect(fs.existsSync(scope.directory)).toBe(true); + } finally { + if (originalRecord) fs.writeFileSync(activatorPidFile, `${originalRecord.value}\n`); + await stopUnrelatedProcess(unrelated); + await cleanFixture(scope); + } + }, + ); + + it( + "rejects a reused activator PID during socket start without signaling the unrelated process (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + const unrelated = spawnUnrelatedProcess(); + try { + await vi.waitFor(() => expect(pidIsActive(unrelated.pid!)).toBe(true)); + const staleRecord = `${String(unrelated.pid)}\tproc:1\tactivator:${"0".repeat(32)}\n`; + fs.writeFileSync(activatorPidFile, staleRecord, { mode: 0o600 }); + + const start = systemctl(scope, ["--user", "start", "podman.socket"]); + expect(start.status).not.toBe(0); + expect(start.stderr).toContain( + `Portable profile fixture PID file ${activatorPidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + expect(fs.readFileSync(activatorPidFile, "utf8")).toBe(staleRecord); + expect(fs.existsSync(scope.socketPath)).toBe(false); + } finally { + fs.rmSync(activatorPidFile, { force: true }); + await stopUnrelatedProcess(unrelated); + await cleanFixture(scope); + } + }, + ); + + it( + "rejects a reused activator PID during try-restart without signaling the unrelated process (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + let originalRecord: FixtureProcessRecord | undefined; + let unrelated: ReturnType | undefined; + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + await waitForServiceStatus(scope, 0); + const servicePid = readFixtureProcessRecord(servicePidFile).pid; + originalRecord = readFixtureProcessRecord(activatorPidFile); + unrelated = spawnUnrelatedProcess(); + await vi.waitFor(() => expect(pidIsActive(unrelated!.pid!)).toBe(true)); + fs.writeFileSync(activatorPidFile, replaceRecordedPid(originalRecord, unrelated.pid!), { + mode: 0o600, + }); + + const refresh = systemctl(scope, ["--user", "try-restart", "podman.service"]); + expect(refresh.status).not.toBe(0); + expect(refresh.stderr).toContain( + `Portable profile fixture PID file ${activatorPidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + expect(pidIsActive(servicePid)).toBe(true); + expect(readFixtureProcessRecord(servicePidFile).pid).toBe(servicePid); + } finally { + if (originalRecord) fs.writeFileSync(activatorPidFile, `${originalRecord.value}\n`); + await stopUnrelatedProcess(unrelated); + await cleanFixture(scope); + } + }, + ); + + it( + "rejects a reused backend PID during socket reset without signaling the unrelated process (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const unrelated = spawnUnrelatedProcess(); + try { + await vi.waitFor(() => expect(pidIsActive(unrelated.pid!)).toBe(true)); + const staleRecord = `${String(unrelated.pid)}\tproc:1\tservice:${"0".repeat(32)}\n`; + fs.writeFileSync(servicePidFile, staleRecord, { mode: 0o600 }); + + const start = systemctl(scope, ["--user", "start", "podman.socket"]); + expect(start.status).not.toBe(0); + expect(start.stderr).toContain( + `Portable profile fixture PID file ${servicePidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + expect(fs.readFileSync(servicePidFile, "utf8")).toBe(staleRecord); + expect(fs.existsSync(scope.socketPath)).toBe(false); + } finally { + fs.rmSync(servicePidFile, { force: true }); + await stopUnrelatedProcess(unrelated); + await cleanFixture(scope); + } + }, + ); + + it( + "rejects a reused backend PID during status and activator refresh without signaling the unrelated process (#9006)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const activatorPidFile = path.join(scope.runtimeDir, "nemoclaw-podman-socket-activator.pid"); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const logFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.log"); + let originalRecord: FixtureProcessRecord | undefined; + let unrelated: ReturnType | undefined; + try { + expect(systemctl(scope, ["--user", "start", "podman.socket"]).status).toBe(0); + expect(await activateThroughSocket(scope.socketPath)).toBe("ready"); + await waitForServiceStatus(scope, 0); + originalRecord = readFixtureProcessRecord(servicePidFile); + const activatorPid = readFixtureProcessRecord(activatorPidFile).pid; + unrelated = spawnUnrelatedProcess(); + await vi.waitFor(() => expect(pidIsActive(unrelated!.pid!)).toBe(true)); + fs.writeFileSync(servicePidFile, replaceRecordedPid(originalRecord, unrelated.pid!), { + mode: 0o600, + }); + + expect(serviceStatus(scope)).not.toBe(0); + expect(pidIsActive(unrelated.pid!)).toBe(true); + const refresh = systemctl(scope, ["--user", "try-restart", "podman.service"]); + expect(refresh.status).not.toBe(0); + expect(refresh.stderr).toContain( + `Portable profile fixture PID file ${servicePidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + process.kill(activatorPid, "SIGHUP"); + await waitForFileText( + logFile, + `Portable profile fixture PID file ${servicePidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + expectProcessActive(originalRecord.pid); + expect(fs.existsSync(servicePidFile)).toBe(true); + } finally { + if (originalRecord) fs.writeFileSync(servicePidFile, `${originalRecord.value}\n`); + await stopUnrelatedProcess(unrelated); + await cleanFixture(scope); + } + }, + ); + it.each([ ["malformed PID text", "not-a-pid"], ["a PID beyond Number.MAX_SAFE_INTEGER", `${Number.MAX_SAFE_INTEGER}0`], From ab3613a21117e28b5ad14fc09819a82c07e840f0 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 10:08:06 -0700 Subject: [PATCH 12/13] test(e2e): keep portable fixture teardown linear Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.test.ts | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index f094a6b4b99..3e373cd7bd8 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -244,15 +244,33 @@ function spawnUnrelatedProcess(): ReturnType { } async function stopUnrelatedProcess(child: ReturnType | undefined): Promise { - if (!child?.pid || child.exitCode !== null) return; - if (pidIsActive(child.pid)) child.kill("SIGKILL"); - await new Promise((resolve) => { - const timeout = setTimeout(resolve, 5_000); - child.once("close", () => { - clearTimeout(timeout); - resolve(); - }); - }); + await Promise.all( + [child] + .filter( + (candidate): candidate is ReturnType => + candidate?.pid !== undefined && candidate.exitCode === null, + ) + .map( + (candidate) => + new Promise((resolve) => { + const timeout = setTimeout(resolve, 5_000); + candidate.once("close", () => { + clearTimeout(timeout); + resolve(); + }); + candidate.kill("SIGKILL"); + }), + ), + ); +} + +function restoreFixtureProcessRecord( + pidFile: string, + record: FixtureProcessRecord | undefined, +): void { + [record] + .filter((candidate): candidate is FixtureProcessRecord => candidate !== undefined) + .forEach((candidate) => fs.writeFileSync(pidFile, `${candidate.value}\n`)); } async function cleanFixture(scope: FixtureScope): Promise { @@ -584,7 +602,7 @@ describe("portable profile systemctl fixture", () => { expect(fs.existsSync(scope.socketPath)).toBe(true); expect(fs.existsSync(scope.directory)).toBe(true); } finally { - if (originalRecord) fs.writeFileSync(activatorPidFile, `${originalRecord.value}\n`); + restoreFixtureProcessRecord(activatorPidFile, originalRecord); await stopUnrelatedProcess(unrelated); await cleanFixture(scope); } @@ -649,7 +667,7 @@ describe("portable profile systemctl fixture", () => { expect(pidIsActive(servicePid)).toBe(true); expect(readFixtureProcessRecord(servicePidFile).pid).toBe(servicePid); } finally { - if (originalRecord) fs.writeFileSync(activatorPidFile, `${originalRecord.value}\n`); + restoreFixtureProcessRecord(activatorPidFile, originalRecord); await stopUnrelatedProcess(unrelated); await cleanFixture(scope); } @@ -723,7 +741,7 @@ describe("portable profile systemctl fixture", () => { expectProcessActive(originalRecord.pid); expect(fs.existsSync(servicePidFile)).toBe(true); } finally { - if (originalRecord) fs.writeFileSync(servicePidFile, `${originalRecord.value}\n`); + restoreFixtureProcessRecord(servicePidFile, originalRecord); await stopUnrelatedProcess(unrelated); await cleanFixture(scope); } From 4e252fbe444f438180c12821836537da303650a0 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 15 Aug 2026 11:23:27 -0700 Subject: [PATCH 13/13] fix(e2e): stabilize portable process identity checks Signed-off-by: Senthil Ravichandran --- .../portable-profile-systemctl-shim.sh | 17 ++++-- .../fixtures/portable-profile-systemctl.ts | 1 + .../portable-profile-systemctl-shim.test.ts | 59 +++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index 9fff46838f5..08c22c021d7 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -15,6 +15,14 @@ process_identity_env="NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID" process_identity_failure_role="${NEMOCLAW_PODMAN_IDENTITY_FAILURE_ROLE:-}" process_identity_failure_record="${NEMOCLAW_PODMAN_IDENTITY_FAILURE_RECORD:-}" +format_ps_start_time() { + local start_time="$1" + local -a start_time_fields + read -r -a start_time_fields <<<"$start_time" + [[ "${#start_time_fields[@]}" -gt 0 ]] || return 1 + printf 'ps:%s\n' "${start_time_fields[*]}" +} + process_start_time() { local pid="$1" if [[ -r "/proc/${pid}/stat" ]]; then @@ -30,10 +38,7 @@ process_start_time() { local start_time start_time="$(ps -o lstart= -p "$pid" 2>/dev/null)" || return 1 - start_time="${start_time#"${start_time%%[![:space:]]*}"}" - start_time="${start_time%"${start_time##*[![:space:]]}"}" - [[ -n "$start_time" ]] || return 1 - printf 'ps:%s\n' "$start_time" + format_ps_start_time "$start_time" } process_has_identity() { @@ -889,6 +894,10 @@ NODE return 1 } +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return 0 +fi + if [[ "$#" -eq 4 && "$1" == "--user" && "$2" == "set-environment" && diff --git a/test/e2e/fixtures/portable-profile-systemctl.ts b/test/e2e/fixtures/portable-profile-systemctl.ts index 9d29c4f80e9..a37bc82148d 100644 --- a/test/e2e/fixtures/portable-profile-systemctl.ts +++ b/test/e2e/fixtures/portable-profile-systemctl.ts @@ -134,6 +134,7 @@ function fixtureProcessIsActive(processIdentity: FixtureProcessIdentity): boolea readProcessStartTime(processIdentity.pid) !== processIdentity.startTime || !processHasIdentity(processIdentity.pid, processIdentity.identity) ) { + if (!processIsActive(processIdentity.pid)) return false; throw new Error( `Portable profile fixture PID file ${processIdentity.pidFile} does not match process ${String(processIdentity.pid)}.`, ); diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index 3e373cd7bd8..d05f4369916 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -114,6 +114,25 @@ function systemctl(scope: FixtureScope, args: string[]): ReturnType { + return spawnSync( + "bash", + [ + "-c", + 'source "$1"\nformat_ps_start_time "$2"', + "portable-profile-ps-start-time", + scope.shim, + startTime, + ], + { + encoding: "utf8", + env: scope.env, + killSignal: "SIGKILL", + timeout: 15_000, + }, + ); +} + function systemctlAsync( scope: FixtureScope, args: string[], @@ -303,6 +322,46 @@ function portableLaunchStep(name: string): WorkflowStep { } describe("portable profile systemctl fixture", () => { + it("normalizes irregular ps fallback spacing to one process-start-time identity (#9006)", () => { + const scope = createFixture(); + try { + const result = formatPsStartTime(scope, " Fri Aug 8 12:34:56 2026 "); + expect(result.status, String(result.stderr)).toBe(0); + expect(result.stdout).toBe("ps:Fri Aug 8 12:34:56 2026\n"); + } finally { + fs.rmSync(scope.directory, { force: true, recursive: true }); + } + }); + + it("treats a process that exits during shared cleanup identity revalidation as inactive (#9006)", async () => { + const scope = createFixture(); + const servicePidFile = path.join(scope.runtimeDir, "nemoclaw-podman-service.pid"); + const exitedPid = Number.MAX_SAFE_INTEGER; + const kill = vi.spyOn(process, "kill"); + kill.mockImplementationOnce((_pid, signal) => { + expect(signal).toBe(0); + return true; + }); + kill.mockImplementation((_pid, signal) => { + expect(signal).toBe(0); + throw Object.assign(new Error("process exited"), { code: "ESRCH" }); + }); + fs.writeFileSync(servicePidFile, `${String(exitedPid)}\tproc:1\tservice:${"0".repeat(32)}\n`, { + mode: 0o600, + }); + + try { + await expect( + cleanupPortableProfileSystemctlFixture(scope.runtimeDir), + ).resolves.toBeUndefined(); + expect(kill).toHaveBeenCalledTimes(3); + expect(fs.existsSync(servicePidFile)).toBe(false); + } finally { + kill.mockRestore(); + fs.rmSync(scope.directory, { force: true, recursive: true }); + } + }); + it( "installs a mode-0700 shim that preserves socket identity from cold activation through try-restart (#9006)", { timeout: 30_000 },