diff --git a/test/e2e/fixtures/portable-profile-systemctl-shim.sh b/test/e2e/fixtures/portable-profile-systemctl-shim.sh index 08c22c021d7..ce00184635a 100755 --- a/test/e2e/fixtures/portable-profile-systemctl-shim.sh +++ b/test/e2e/fixtures/portable-profile-systemctl-shim.sh @@ -5,12 +5,58 @@ set -euo pipefail runtime_dir="${XDG_RUNTIME_DIR:?}" +home_dir="${HOME:?}" +config_home="${XDG_CONFIG_HOME:-}" +[[ "$config_home" == /* ]] || config_home="${home_dir}/.config" +state_home="${XDG_STATE_HOME:-}" +[[ "$state_home" == /* ]] || state_home="${home_dir}/.local/state" +bin_home="${XDG_BIN_HOME:-}" +[[ "$bin_home" == /* ]] || bin_home="${home_dir}/.local/bin" service_dir="${runtime_dir}/podman" socket_path="${service_dir}/podman.sock" backend_socket_path="${service_dir}/nemoclaw-podman-service.sock" activator_pid_file="${runtime_dir}/nemoclaw-podman-socket-activator.pid" service_pid_file="${runtime_dir}/nemoclaw-podman-service.pid" log_file="${runtime_dir}/nemoclaw-podman-service.log" +gateway_service_name="nemoclaw-openshell-gateway" +gateway_unit_path="${config_home}/systemd/user/${gateway_service_name}.service" +gateway_binary_path="${bin_home}/openshell-gateway" +gateway_env_file="${config_home}/openshell/gateway.env" +gateway_tls_dir="${state_home}/openshell/tls" +gateway_state_dir="${state_home}/openshell/gateway" +gateway_pid_file="${runtime_dir}/nemoclaw-openshell-gateway.pid" +gateway_launch_pid_file="${runtime_dir}/nemoclaw-openshell-gateway-launch.pid" +gateway_log_file="${runtime_dir}/nemoclaw-openshell-gateway.log" +gateway_environment_keys=( + CONTAINERS_CONF + DOCKER_HOST + OPENSHELL_DRIVERS + OPENSHELL_BIND_ADDRESS + OPENSHELL_SERVER_PORT + OPENSHELL_DISABLE_TLS + OPENSHELL_DISABLE_GATEWAY_AUTH + OPENSHELL_LOCAL_TLS_DIR + OPENSHELL_DB_URL + OPENSHELL_GRPC_ENDPOINT + OPENSHELL_SSH_GATEWAY_HOST + OPENSHELL_SSH_GATEWAY_PORT + OPENSHELL_DOCKER_NETWORK_NAME + OPENSHELL_DOCKER_SUPERVISOR_IMAGE + OPENSHELL_DOCKER_SUPERVISOR_BIN + OPENSHELL_PODMAN_SOCKET + OPENSHELL_GATEWAY_CONFIG + OPENSHELL_VM_DRIVER_STATE_DIR + OPENSHELL_DRIVER_DIR + NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE + NETAVARK_FW +) +gateway_fixture_environment_keys=( + FAKE_GATEWAY_CERT_MARKER + FAKE_GATEWAY_COMMAND_LOG +) +gateway_process_environment=() +gateway_launch_start_time="" process_identity_env="NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID" process_identity_failure_role="${NEMOCLAW_PODMAN_IDENTITY_FAILURE_ROLE:-}" process_identity_failure_record="${NEMOCLAW_PODMAN_IDENTITY_FAILURE_RECORD:-}" @@ -414,6 +460,286 @@ stop_runtime() { rm -f "$socket_path" "$backend_socket_path" } +validate_gateway_unit() { + if [[ ! -f "$gateway_unit_path" || -L "$gateway_unit_path" || ! -r "$gateway_unit_path" ]]; then + echo "Portable profile fixture requires the managed gateway user service at ${gateway_unit_path}." >&2 + return 1 + fi + if [[ "$(grep -Fxc '# NEMOCLAW_MANAGED_OPENSHELL_GATEWAY=1' "$gateway_unit_path" || true)" -ne 1 ]]; then + echo "Portable profile fixture rejected the foreign gateway user service at ${gateway_unit_path}." >&2 + return 1 + fi + if [[ "$(grep -Fxc "ExecStart=${gateway_binary_path}" "$gateway_unit_path" || true)" -ne 1 ]] \ + || [[ "$(grep -Fxc "ExecStartPre=${gateway_binary_path} generate-certs --output-dir \${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal" "$gateway_unit_path" || true)" -ne 1 ]] \ + || [[ "$(grep -Fxc 'StateDirectory=openshell/gateway' "$gateway_unit_path" || true)" -ne 1 ]] \ + || [[ "$(grep -Fxc 'Environment=OPENSHELL_LOCAL_TLS_DIR=%S/openshell/tls' "$gateway_unit_path" || true)" -ne 1 ]] \ + || [[ "$(grep -Fxc 'EnvironmentFile=-%E/openshell/gateway.env' "$gateway_unit_path" || true)" -ne 1 ]] \ + || [[ ! -x "$gateway_binary_path" || -L "$gateway_binary_path" ]]; then + echo "Portable profile fixture rejected the gateway user service identity at ${gateway_unit_path}." >&2 + return 1 + fi +} + +load_gateway_environment() { + local key + for key in "${gateway_environment_keys[@]}"; do + unset "$key" + done + export OPENSHELL_LOCAL_TLS_DIR="$gateway_tls_dir" + [[ -e "$gateway_env_file" || -L "$gateway_env_file" ]] || return 0 + if [[ ! -f "$gateway_env_file" || -L "$gateway_env_file" || ! -r "$gateway_env_file" ]]; then + echo "Portable profile fixture rejected the gateway environment file at ${gateway_env_file}." >&2 + return 1 + fi + + local line value managed_key managed_key_candidate + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -n "$line" && "$line" != \#* ]] || continue + [[ "$line" == *=* ]] || { + echo "Portable profile fixture rejected an invalid gateway environment assignment." >&2 + return 1 + } + key="${line%%=*}" + value="${line#*=}" + managed_key=false + for managed_key_candidate in "${gateway_environment_keys[@]}"; do + if [[ "$key" == "$managed_key_candidate" ]]; then + managed_key=true + break + fi + done + if [[ "$managed_key" != true ]]; then + echo "Portable profile fixture rejected gateway environment key ${key}." >&2 + return 1 + fi + if [[ "$value" == \'* || "$value" == *\' ]]; then + if [[ "$value" != \'*\' || "${#value}" -lt 2 ]]; then + echo "Portable profile fixture rejected an invalid gateway environment value for ${key}." >&2 + return 1 + fi + value="${value:1:${#value}-2}" + fi + export "${key}=${value}" + done <"$gateway_env_file" +} + +build_gateway_process_environment() { + gateway_process_environment=( + "HOME=${home_dir}" + "PATH=/usr/local/bin:/usr/bin:/bin" + "XDG_BIN_HOME=${bin_home}" + "XDG_CONFIG_HOME=${config_home}" + "XDG_RUNTIME_DIR=${runtime_dir}" + "XDG_STATE_HOME=${state_home}" + ) + local key + for key in "${gateway_environment_keys[@]}" "${gateway_fixture_environment_keys[@]}"; do + if declare -p "$key" >/dev/null 2>&1; then + gateway_process_environment+=("${key}=${!key}") + fi + done +} + +gateway_service_is_active() { + recorded_process_is_active "$gateway_pid_file" gateway +} + +stop_gateway_service() { + stop_recorded_process "$gateway_pid_file" gateway +} + +stop_gateway_launch() { + stop_recorded_process "$gateway_launch_pid_file" gateway +} + +wait_for_gateway_launch_record() { + local pid="$1" + local identity="$2" + local status + for ((attempt = 0; attempt < 100; attempt += 1)); do + if read_pid_record "$gateway_launch_pid_file" gateway; then + if [[ "$recorded_pid" != "$pid" || "$recorded_identity" != "$identity" ]]; then + echo "Portable profile fixture gateway launch record does not match the launched process." >&2 + return 2 + fi + gateway_launch_start_time="$recorded_start_time" + return 0 + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" + fi + kill -0 "$pid" 2>/dev/null || return 1 + sleep 0.05 + done + return 2 +} + +stop_gateway_without_launch_record() { + local pid="$1" + local identity="$2" + if [[ "${NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_UNRECORDED_CLEANUP_FAILURE:-}" == "1" ]]; then + return 2 + fi + stop_unrecorded_process "$pid" "$identity" "" +} + +fail_recorded_gateway_start() { + local cleanup_status + if [[ "${NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_CLEANUP_FAILURE:-}" == "1" ]]; then + cleanup_status=2 + elif stop_gateway_launch; then + cleanup_status=0 + else + cleanup_status=$? + fi + echo "Portable profile fixture could not create the gateway process identity record." >&2 + if [[ "$cleanup_status" -ne 0 ]]; then + echo "Portable profile fixture could not stop the gateway launch process." >&2 + return "$cleanup_status" + fi + return 1 +} + +start_gateway_service() { + validate_gateway_unit + stop_gateway_launch + load_gateway_environment + build_gateway_process_environment + install -d -m 700 "$OPENSHELL_LOCAL_TLS_DIR" "$gateway_state_dir" + install -m 600 /dev/null "$gateway_log_file" + if ! env -i "${gateway_process_environment[@]}" "$gateway_binary_path" generate-certs \ + --output-dir "$OPENSHELL_LOCAL_TLS_DIR" \ + --server-san host.openshell.internal >>"$gateway_log_file" 2>&1; then + echo "Portable profile fixture could not generate gateway certificates." >&2 + return 1 + fi + + local cleanup_status failure_status gateway_drift_identity gateway_identity gateway_pid + gateway_identity="gateway:$(node -e 'process.stdout.write(require("node:crypto").randomBytes(16).toString("hex"))')" + # shellcheck disable=SC2016 # Positional parameters and variables expand inside the launch wrapper. + env -i "${gateway_process_environment[@]}" \ + "NEMOCLAW_PORTABLE_PROFILE_PROCESS_ID=${gateway_identity}" nohup "$BASH" -c ' + set -euo pipefail + gateway_binary_path="$1" + gateway_launch_pid_file="$2" + gateway_identity="$3" + inject_record_failure="$4" + local_start_time="" + if [[ -r "/proc/$$/stat" ]]; then + stat="$(<"/proc/$$/stat")" + stat="${stat##*) }" + read -r -a fields <<<"$stat" + [[ "${#fields[@]}" -gt 19 && "${fields[19]}" =~ ^[0-9]+$ ]] + local_start_time="proc:${fields[19]}" + else + [[ ! -e /proc/self/stat ]] + local_start_time="$(ps -o lstart= -p "$$")" + read -r -a fields <<<"$local_start_time" + [[ "${#fields[@]}" -gt 0 ]] + local_start_time="ps:${fields[*]}" + fi + if [[ "$inject_record_failure" == "1" ]]; then + printf "Portable profile fixture injected gateway launch-record failure for process %s.\n" "$$" + exit 73 + fi + launch_pid_file_tmp="${gateway_launch_pid_file}.$$.tmp" + trap '\''rm -f "$launch_pid_file_tmp"'\'' EXIT + printf "%s\t%s\t%s\n" "$$" "$local_start_time" "$gateway_identity" \ + >"$launch_pid_file_tmp" + chmod 600 "$launch_pid_file_tmp" + mv "$launch_pid_file_tmp" "$gateway_launch_pid_file" + trap - EXIT + exec "$gateway_binary_path" + ' portable-profile-gateway-launch "$gateway_binary_path" "$gateway_launch_pid_file" \ + "$gateway_identity" "${NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_LAUNCH_RECORD_FAILURE:-0}" \ + >>"$gateway_log_file" 2>&1 &2 + if [[ "$cleanup_status" -ne 0 ]]; then + echo "Portable profile fixture could not complete gateway launch cleanup." >&2 + return "$cleanup_status" + fi + return "$failure_status" + fi + if acquire_process_identity "$gateway_pid" "$gateway_identity" \ + && [[ "$acquired_process_start_time" == "$gateway_launch_start_time" ]]; then + if [[ "${NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_RECORD_FAILURE:-}" == "1" ]]; then + if fail_recorded_gateway_start; then + failure_status=1 + else + failure_status=$? + fi + return "$failure_status" + fi + mv "$gateway_launch_pid_file" "$gateway_pid_file" + if [[ "${NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_RECORD_DRIFT:-}" == "1" ]]; then + cp "$gateway_pid_file" "${gateway_pid_file}.before-validation" + gateway_drift_identity="gateway:00000000000000000000000000000000" + if [[ "$gateway_drift_identity" == "$gateway_identity" ]]; then + gateway_drift_identity="gateway:11111111111111111111111111111111" + fi + printf '%s\t%s\t%s\n' "$gateway_pid" "$acquired_process_start_time" \ + "$gateway_drift_identity" \ + >"$gateway_pid_file" + fi + else + if fail_recorded_gateway_start; then + failure_status=1 + else + failure_status=$? + fi + return "$failure_status" + fi + + local gateway_status + if gateway_service_is_active; then + return 0 + else + gateway_status=$? + fi + if [[ "$gateway_status" -eq 1 ]]; then + rm -f "$gateway_pid_file" + fi + echo "Portable profile fixture gateway process did not remain active." >&2 + return 1 +} + +restart_gateway_service() { + validate_gateway_unit + stop_gateway_launch + stop_gateway_service + start_gateway_service +} + +print_gateway_identity() { + validate_gateway_unit + printf 'FragmentPath=%s\n' "$gateway_unit_path" + printf 'ExecStart={ path=%s ; argv[]=%s ; }\n' "$gateway_binary_path" "$gateway_binary_path" +} + +print_active_gateway_identity() { + print_gateway_identity + local status gateway_pid=0 active_state=inactive + if gateway_service_is_active; then + gateway_pid="$recorded_pid" + active_state=active + else + status=$? + [[ "$status" -eq 1 ]] || return "$status" + fi + printf 'ActiveState=%s\n' "$active_state" + printf 'MainPID=%s\n' "$gateway_pid" +} + refresh_service() { local status if service_is_active; then @@ -906,6 +1232,74 @@ if [[ "$#" -eq 4 && exit 0 fi +if [[ "$#" -eq 2 && + "$1" == "--user" && + "$2" == "daemon-reload" ]]; then + validate_gateway_unit + exit 0 +fi + +if [[ "$#" -eq 5 && + "$1" == "--user" && + "$2" == "show" && + "$3" == "$gateway_service_name" && + "$4" == "--property=FragmentPath" && + "$5" == "--property=ExecStart" ]]; then + print_gateway_identity + exit 0 +fi + +if [[ "$#" -eq 7 && + "$1" == "--user" && + "$2" == "show" && + "$3" == "$gateway_service_name" && + "$4" == "--property=FragmentPath" && + "$5" == "--property=ExecStart" && + "$6" == "--property=ActiveState" && + "$7" == "--property=MainPID" ]]; then + print_active_gateway_identity + exit 0 +fi + +if [[ "$#" -eq 3 && + "$1" == "--user" && + "$2" == "stop" && + "$3" == "$gateway_service_name" ]]; then + validate_gateway_unit + stop_gateway_service + exit 0 +fi + +if [[ "$#" -eq 3 && + "$1" == "--user" && + "$2" == "enable" && + "$3" == "$gateway_service_name" ]]; then + validate_gateway_unit + exit 0 +fi + +if [[ "$#" -eq 3 && + "$1" == "--user" && + "$2" == "restart" && + "$3" == "$gateway_service_name" ]]; then + restart_gateway_service + exit 0 +fi + +if [[ "$#" -eq 4 && + "$1" == "--user" && + "$2" == "is-active" && + "$3" == "--quiet" && + "$4" == "$gateway_service_name" ]]; then + if gateway_service_is_active; then + exit 0 + else + status=$? + [[ "$status" -eq 1 ]] || exit "$status" + fi + exit 3 +fi + if [[ "$#" -eq 3 && "$1" == "--user" && "$2" == "try-restart" && diff --git a/test/e2e/fixtures/portable-profile-systemctl.ts b/test/e2e/fixtures/portable-profile-systemctl.ts index a37bc82148d..5feb697a482 100644 --- a/test/e2e/fixtures/portable-profile-systemctl.ts +++ b/test/e2e/fixtures/portable-profile-systemctl.ts @@ -15,6 +15,8 @@ const PROCESS_QUERY_TIMEOUT_MS = 5_000; const FIXTURE_PID_FILES = [ ["nemoclaw-podman-socket-activator.pid", "activator"], ["nemoclaw-podman-service.pid", "service"], + ["nemoclaw-openshell-gateway-launch.pid", "gateway"], + ["nemoclaw-openshell-gateway.pid", "gateway"], ] as const; const FIXTURE_SOCKET_FILES = ["podman.sock", "nemoclaw-podman-service.sock"] as const; diff --git a/test/e2e/support/portable-profile-systemctl-shim.test.ts b/test/e2e/support/portable-profile-systemctl-shim.test.ts index d05f4369916..0fa8c15de2d 100644 --- a/test/e2e/support/portable-profile-systemctl-shim.test.ts +++ b/test/e2e/support/portable-profile-systemctl-shim.test.ts @@ -16,11 +16,16 @@ import { import { readYaml, type Workflow, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; const INSTALLER_PAYLOAD = path.join(import.meta.dirname, "..", "..", "..", "scripts", "install.sh"); - interface FixtureScope { readonly binDir: string; readonly directory: string; readonly env: NodeJS.ProcessEnv; + readonly gatewayBin: string; + readonly gatewayCommandLog: string; + readonly gatewayPidFile: string; + readonly gatewayTlsDir: string; + readonly gatewayUnitPath: string; + readonly homeDir: string; readonly runtimeDir: string; readonly shim: string; readonly socketPath: string; @@ -37,13 +42,40 @@ function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o700 }); } +function gatewayServiceUnit(gatewayBin: string): string { + return `# NEMOCLAW_MANAGED_OPENSHELL_GATEWAY=1 +[Service] +StateDirectory=openshell/gateway +Environment=OPENSHELL_LOCAL_TLS_DIR=%S/openshell/tls +EnvironmentFile=-%E/openshell/gateway.env +ExecStartPre=${gatewayBin} generate-certs --output-dir \${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal +ExecStart=${gatewayBin} +`; +} + function createFixture(): FixtureScope { const directory = fs.mkdtempSync("/tmp/portable-systemctl-shim-"); const binDir = path.join(directory, "bin"); + const homeDir = path.join(directory, "home"); + const binHome = path.join(homeDir, ".local", "bin"); + const configHome = path.join(homeDir, ".config"); + const stateHome = path.join(homeDir, ".local", "state"); const runtimeDir = path.join(directory, "runtime"); const socketPath = path.join(runtimeDir, "podman", "podman.sock"); + const gatewayBin = path.join(binHome, "openshell-gateway"); + const gatewayCommandLog = path.join(directory, "gateway-commands.jsonl"); + const gatewayPidFile = path.join(runtimeDir, "nemoclaw-openshell-gateway.pid"); + const gatewayTlsDir = path.join(stateHome, "nemoclaw", "openshell-docker-gateway", "tls"); + const gatewayUnitPath = path.join( + configHome, + "systemd", + "user", + "nemoclaw-openshell-gateway.service", + ); fs.mkdirSync(binDir); fs.mkdirSync(runtimeDir); + fs.mkdirSync(path.dirname(gatewayBin), { recursive: true, mode: 0o700 }); + fs.mkdirSync(path.dirname(gatewayUnitPath), { recursive: true, mode: 0o700 }); const shim = installPortableProfileSystemctlShim(binDir); writeExecutable( path.join(binDir, "podman"), @@ -89,16 +121,92 @@ process.on("SIGTERM", stop); `, ); writeExecutable(path.join(binDir, "docker"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable( + gatewayBin, + `#!${process.execPath} +const fs = require("node:fs"); +const args = process.argv.slice(2); +const record = (value) => fs.appendFileSync( + process.env.FAKE_GATEWAY_COMMAND_LOG, + JSON.stringify(value) + "\\n", +); +if ( + args.length === 5 && + args[0] === "generate-certs" && + args[1] === "--output-dir" && + args[3] === "--server-san" && + args[4] === "host.openshell.internal" +) { + if (fs.existsSync(process.env.FAKE_GATEWAY_CERT_MARKER + ".fail")) { + console.error("test-only gateway certificate diagnostic"); + process.exit(70); + } + fs.mkdirSync(args[2], { recursive: true, mode: 0o700 }); + fs.writeFileSync(process.env.FAKE_GATEWAY_CERT_MARKER, "generated\\n", { mode: 0o600 }); + record({ + args, + kind: "generate-certs", + nvidiaInferenceApiKey: process.env.NVIDIA_INFERENCE_API_KEY ?? null, + path: process.env.PATH, + tls: process.env.OPENSHELL_LOCAL_TLS_DIR, + }); + process.exit(0); +} +if (args.length !== 0) process.exit(64); +record({ + bindAddress: process.env.OPENSHELL_BIND_ADDRESS ?? null, + bindMounts: process.env.NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS ?? null, + disableGatewayAuth: process.env.OPENSHELL_DISABLE_GATEWAY_AUTH ?? null, + disableTls: process.env.OPENSHELL_DISABLE_TLS ?? null, + dockerHost: process.env.DOCKER_HOST, + drivers: process.env.OPENSHELL_DRIVERS, + kind: "serve", + nvidiaInferenceApiKey: process.env.NVIDIA_INFERENCE_API_KEY ?? null, + path: process.env.PATH, + pid: process.pid, + tls: process.env.OPENSHELL_LOCAL_TLS_DIR, +}); +const stop = () => process.exit(0); +process.on("SIGINT", stop); +process.on("SIGTERM", stop); +setInterval(() => undefined, 1000); +`, + ); + fs.writeFileSync(gatewayUnitPath, gatewayServiceUnit(gatewayBin), { mode: 0o600 }); + const gatewayEnvFile = path.join(homeDir, ".config", "openshell", "gateway.env"); + fs.mkdirSync(path.dirname(gatewayEnvFile), { recursive: true, mode: 0o700 }); + fs.writeFileSync( + gatewayEnvFile, + `DOCKER_HOST='unix://${socketPath}'\nOPENSHELL_DRIVERS=podman\nOPENSHELL_LOCAL_TLS_DIR=${gatewayTlsDir}\n`, + { mode: 0o600 }, + ); return { binDir, directory, env: { ...process.env, + FAKE_GATEWAY_CERT_MARKER: path.join(directory, "gateway-cert.marker"), + FAKE_GATEWAY_COMMAND_LOG: gatewayCommandLog, FAKE_PODMAN_PID_LOG: path.join(directory, "podman-pids.log"), FAKE_PODMAN_SOCKET: socketPath, + HOME: homeDir, + NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS: "1", + NVIDIA_INFERENCE_API_KEY: "test-only-hostile-inherited-key", + OPENSHELL_BIND_ADDRESS: "127.0.0.1", + OPENSHELL_DISABLE_GATEWAY_AUTH: "1", + OPENSHELL_DISABLE_TLS: "1", PATH: `${binDir}:${process.env.PATH ?? ""}`, + XDG_BIN_HOME: binHome, + XDG_CONFIG_HOME: configHome, XDG_RUNTIME_DIR: runtimeDir, + XDG_STATE_HOME: stateHome, }, + gatewayBin, + gatewayCommandLog, + gatewayPidFile, + gatewayTlsDir, + gatewayUnitPath, + homeDir, runtimeDir, shim, socketPath, @@ -166,6 +274,10 @@ function serviceStatus(scope: FixtureScope): number | null { return systemctl(scope, ["--user", "is-active", "--quiet", "podman.service"]).status; } +function gatewayStatus(scope: FixtureScope): number | null { + return systemctl(scope, ["--user", "is-active", "--quiet", "nemoclaw-openshell-gateway"]).status; +} + function activateThroughSocket(socketPath: string): Promise { return new Promise((resolve, reject) => { const client = net.createConnection(socketPath); @@ -452,6 +564,332 @@ describe("portable profile systemctl fixture", () => { }, ); + it( + "runs the managed gateway user-service sequence and cleanup through the fixture (#9208)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + try { + expect(gatewayStatus(scope)).toBe(3); + expect(systemctl(scope, ["--user", "daemon-reload"]).status).toBe(0); + + const identity = systemctl(scope, [ + "--user", + "show", + "nemoclaw-openshell-gateway", + "--property=FragmentPath", + "--property=ExecStart", + ]); + expect(identity.status, String(identity.stderr)).toBe(0); + expect(identity.stdout).toBe( + `FragmentPath=${scope.gatewayUnitPath}\nExecStart={ path=${scope.gatewayBin} ; argv[]=${scope.gatewayBin} ; }\n`, + ); + expect(systemctl(scope, ["--user", "stop", "nemoclaw-openshell-gateway"]).status).toBe(0); + expect(systemctl(scope, ["--user", "enable", "nemoclaw-openshell-gateway"]).status).toBe(0); + + const restart = systemctl(scope, ["--user", "restart", "nemoclaw-openshell-gateway"]); + expect(restart.status, String(restart.stderr)).toBe(0); + expect(gatewayStatus(scope)).toBe(0); + const gatewayProcess = readFixtureProcessRecord(scope.gatewayPidFile); + expectProcessActive(gatewayProcess.pid); + + const activeIdentity = systemctl(scope, [ + "--user", + "show", + "nemoclaw-openshell-gateway", + "--property=FragmentPath", + "--property=ExecStart", + "--property=ActiveState", + "--property=MainPID", + ]); + expect(activeIdentity.status, String(activeIdentity.stderr)).toBe(0); + expect(activeIdentity.stdout).toContain("ActiveState=active\n"); + expect(activeIdentity.stdout).toContain(`MainPID=${String(gatewayProcess.pid)}\n`); + + const commands = fs + .readFileSync(scope.gatewayCommandLog, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(commands.map((command) => command.kind)).toEqual(["generate-certs", "serve"]); + expect(commands[0]).toMatchObject({ + args: [ + "generate-certs", + "--output-dir", + scope.gatewayTlsDir, + "--server-san", + "host.openshell.internal", + ], + nvidiaInferenceApiKey: null, + path: "/usr/local/bin:/usr/bin:/bin", + tls: scope.gatewayTlsDir, + }); + expect(commands[1]).toMatchObject({ + bindAddress: null, + bindMounts: null, + disableGatewayAuth: null, + disableTls: null, + dockerHost: `unix://${scope.socketPath}`, + drivers: "podman", + nvidiaInferenceApiKey: null, + path: "/usr/local/bin:/usr/bin:/bin", + tls: scope.gatewayTlsDir, + }); + expect(fs.readFileSync(scope.env.FAKE_GATEWAY_CERT_MARKER!, "utf8")).toBe("generated\n"); + + await cleanupPortableProfileSystemctlFixture(scope.runtimeDir); + expect(pidIsActive(gatewayProcess.pid)).toBe(false); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(false); + } finally { + await cleanFixture(scope); + } + }, + ); + + it("does not emit gateway child output when certificate generation fails (#9208)", async () => { + const scope = createFixture(); + try { + fs.writeFileSync(`${scope.env.FAKE_GATEWAY_CERT_MARKER!}.fail`, "fail\n", { + mode: 0o600, + }); + const restart = systemctl(scope, ["--user", "restart", "nemoclaw-openshell-gateway"]); + expect(restart.status).toBe(1); + expect(String(restart.stderr)).toContain( + "Portable profile fixture could not generate gateway certificates.", + ); + expect(String(restart.stderr)).not.toContain("test-only gateway certificate diagnostic"); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(false); + expect( + fs.statSync(path.join(scope.runtimeDir, "nemoclaw-openshell-gateway.log")).mode & 0o777, + ).toBe(0o600); + } finally { + fs.rmSync(`${scope.env.FAKE_GATEWAY_CERT_MARKER!}.fail`, { force: true }); + await cleanFixture(scope); + } + }); + + it( + "preserves a launch record when initial gateway cleanup fails (#9208)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + try { + const restart = systemctl( + { + ...scope, + env: { + ...scope.env, + NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_CLEANUP_FAILURE: "1", + NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_RECORD_FAILURE: "1", + }, + }, + ["--user", "restart", "nemoclaw-openshell-gateway"], + ); + expect(restart.status).toBe(2); + expect(String(restart.stderr)).toContain( + "Portable profile fixture could not create the gateway process identity record.", + ); + expect(String(restart.stderr)).toContain( + "Portable profile fixture could not stop the gateway launch process.", + ); + const gatewayLaunchPidFile = path.join( + scope.runtimeDir, + "nemoclaw-openshell-gateway-launch.pid", + ); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(false); + expect(fs.existsSync(gatewayLaunchPidFile)).toBe(true); + const commands = fs + .readFileSync(scope.gatewayCommandLog, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(commands.map((command) => command.kind)).toEqual(["generate-certs", "serve"]); + const gatewayPid = commands[1]!.pid as number; + expect(readFixtureProcessRecord(gatewayLaunchPidFile).pid).toBe(gatewayPid); + expect(pidIsActive(gatewayPid)).toBe(true); + await cleanupPortableProfileSystemctlFixture(scope.runtimeDir); + expect(pidIsActive(gatewayPid)).toBe(false); + expect(fs.existsSync(gatewayLaunchPidFile)).toBe(false); + } finally { + await cleanFixture(scope); + } + }, + ); + + it( + "does not leave a gateway launch process when launch-record publication and cleanup fail (#9208)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + try { + const restart = systemctl( + { + ...scope, + env: { + ...scope.env, + NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_LAUNCH_RECORD_FAILURE: "1", + NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_UNRECORDED_CLEANUP_FAILURE: "1", + }, + }, + ["--user", "restart", "nemoclaw-openshell-gateway"], + ); + expect(restart.status).toBe(2); + expect(String(restart.stderr)).toContain( + "Portable profile fixture could not create the gateway launch identity record.", + ); + expect(String(restart.stderr)).toContain( + "Portable profile fixture could not complete gateway launch cleanup.", + ); + const gatewayLaunchPidFile = path.join( + scope.runtimeDir, + "nemoclaw-openshell-gateway-launch.pid", + ); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(false); + expect(fs.existsSync(gatewayLaunchPidFile)).toBe(false); + const gatewayLog = fs.readFileSync( + path.join(scope.runtimeDir, "nemoclaw-openshell-gateway.log"), + "utf8", + ); + const launchedPid = /injected gateway launch-record failure for process ([0-9]+)/.exec( + gatewayLog, + ); + expect(launchedPid).not.toBeNull(); + const gatewayPid = Number(launchedPid![1]); + await vi.waitFor(() => expect(pidIsActive(gatewayPid)).toBe(false)); + const commands = fs + .readFileSync(scope.gatewayCommandLog, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(commands.map((command) => command.kind)).toEqual(["generate-certs"]); + } finally { + await cleanFixture(scope); + } + }, + ); + + it( + "isolates the managed gateway user service from ambient XDG homes (#9208)", + { timeout: 30_000 }, + async () => { + const ambientRoot = fs.mkdtempSync("/tmp/portable-systemctl-ambient-"); + vi.stubEnv("XDG_BIN_HOME", path.join(ambientRoot, "bin")); + vi.stubEnv("XDG_CONFIG_HOME", path.join(ambientRoot, "config")); + vi.stubEnv("XDG_RUNTIME_DIR", path.join(ambientRoot, "runtime")); + vi.stubEnv("XDG_STATE_HOME", path.join(ambientRoot, "state")); + const scope = createFixture(); + try { + expect(scope.env.XDG_BIN_HOME).not.toBe(path.join(ambientRoot, "bin")); + expect(scope.env.XDG_CONFIG_HOME).not.toBe(path.join(ambientRoot, "config")); + expect(scope.env.XDG_RUNTIME_DIR).not.toBe(path.join(ambientRoot, "runtime")); + expect(scope.env.XDG_STATE_HOME).not.toBe(path.join(ambientRoot, "state")); + const reload = systemctl(scope, ["--user", "daemon-reload"]); + expect(reload.status, String(reload.stderr)).toBe(0); + const restart = systemctl(scope, ["--user", "restart", "nemoclaw-openshell-gateway"]); + expect(restart.status, String(restart.stderr)).toBe(0); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(true); + expect(fs.readdirSync(ambientRoot)).toEqual([]); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(ambientRoot, { force: true, recursive: true }); + await cleanFixture(scope); + } + }, + ); + + it( + "preserves the gateway PID record when startup identity validation fails (#9208)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + const originalRecordPath = `${scope.gatewayPidFile}.before-validation`; + try { + const result = systemctl( + { + ...scope, + env: { + ...scope.env, + NEMOCLAW_PORTABLE_PROFILE_TEST_GATEWAY_RECORD_DRIFT: "1", + }, + }, + ["--user", "restart", "nemoclaw-openshell-gateway"], + ); + fs.accessSync(originalRecordPath, fs.constants.R_OK); + expect(result.status).toBe(1); + expect(String(result.stderr)).toContain("does not match process"); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(true); + expect(fs.existsSync(originalRecordPath)).toBe(true); + const driftedRecord = readFixtureProcessRecord(scope.gatewayPidFile); + expect(pidIsActive(driftedRecord.pid)).toBe(true); + } finally { + try { + fs.copyFileSync(originalRecordPath, scope.gatewayPidFile); + fs.chmodSync(scope.gatewayPidFile, 0o600); + } finally { + fs.rmSync(originalRecordPath, { force: true }); + await cleanFixture(scope); + } + } + }, + ); + + it( + "rejects a reused gateway PID during shared cleanup without signaling the unrelated process (#9208)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + let originalRecord: FixtureProcessRecord | undefined; + let unrelated: ReturnType | undefined; + try { + expect(systemctl(scope, ["--user", "restart", "nemoclaw-openshell-gateway"]).status).toBe( + 0, + ); + originalRecord = readFixtureProcessRecord(scope.gatewayPidFile); + unrelated = spawnUnrelatedProcess(); + await vi.waitFor(() => expect(pidIsActive(unrelated!.pid!)).toBe(true)); + fs.writeFileSync(scope.gatewayPidFile, replaceRecordedPid(originalRecord, unrelated.pid!), { + mode: 0o600, + }); + + await expect(cleanupPortableProfileSystemctlFixture(scope.runtimeDir)).rejects.toThrow( + `Portable profile fixture PID file ${scope.gatewayPidFile} does not match process ${String(unrelated.pid)}.`, + ); + expect(pidIsActive(unrelated.pid!)).toBe(true); + expect(fs.existsSync(scope.gatewayPidFile)).toBe(true); + expect(fs.existsSync(scope.directory)).toBe(true); + } finally { + restoreFixtureProcessRecord(scope.gatewayPidFile, originalRecord); + await stopUnrelatedProcess(unrelated); + await cleanFixture(scope); + } + }, + ); + + it( + "rejects gateway unit drift before restarting the managed process (#9208)", + { timeout: 30_000 }, + async () => { + const scope = createFixture(); + try { + const start = systemctl(scope, ["--user", "restart", "nemoclaw-openshell-gateway"]); + expect(start.status, String(start.stderr)).toBe(0); + const gatewayProcess = readFixtureProcessRecord(scope.gatewayPidFile); + expectProcessActive(gatewayProcess.pid); + fs.writeFileSync(scope.gatewayUnitPath, "[Service]\nExecStart=/tmp/foreign\n", { + mode: 0o600, + }); + + const restart = systemctl(scope, ["--user", "restart", "nemoclaw-openshell-gateway"]); + expect(restart.status).not.toBe(0); + expect(restart.stderr).toContain("rejected the foreign gateway user service"); + expectProcessActive(gatewayProcess.pid); + expect(readFixtureProcessRecord(scope.gatewayPidFile).pid).toBe(gatewayProcess.pid); + } finally { + await cleanFixture(scope); + } + }, + ); + it( "serializes try-restart with a public-socket request and leaves only the recorded backend process active (#9006)", { timeout: 30_000 }, @@ -858,6 +1296,26 @@ describe("portable profile systemctl fixture", () => { } }); + it("rejects malformed or extended gateway user-service commands (#9208)", () => { + const scope = createFixture(); + try { + const driftedCommands = [ + ["--user", "daemon-reload", "trailing"], + ["--user", "show", "nemoclaw-openshell-gateway", "--property=ExecStart"], + ["--user", "restart", "nemoclaw-openshell-gateway", "trailing"], + ["--user", "enable", "--now", "nemoclaw-openshell-gateway"], + ["--user", "is-active", "nemoclaw-openshell-gateway", "--quiet"], + ]; + for (const args of driftedCommands) { + const result = systemctl(scope, args); + expect(result.status, args.join(" ")).toBe(64); + expect(result.stderr).toContain("unexpected user-service command:"); + } + } finally { + fs.rmSync(scope.directory, { force: true, recursive: true }); + } + }); + it("binds portable-launch setup and always-run cleanup to the shared systemctl fixture (#9006)", () => { const provision = portableLaunchStep("Provision restricted rootless Linux runtime").run ?? ""; expect(provision).toContain(