diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 77396cbc6d2..9d51b1b26c4 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -26,7 +26,7 @@ "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 34, - "src/lib/state/registry.ts": 102, + "src/lib/state/registry.ts": 99, "src/lib/state/state-root.ts": 23, "src/lib/subprocess-env.ts": 23, "src/lib/validation.ts": 25 diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index ed2d68fbe4e..b69f2d59054 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -39,7 +39,8 @@ import { type TrustedLocalBaseImageOverride, } from "../../sandbox-base-image"; import * as shields from "../../shields"; -import * as registry from "../../state/registry"; +import type { SandboxEntry } from "../../state/registry"; +import { load as loadRegistry } from "../../state/registry/persistence"; import * as sandboxState from "../../state/sandbox"; import * as userManagedFilesProbe from "../../state/user-managed-files-probe"; import { @@ -49,11 +50,11 @@ import { } from "./gateway-state"; import { openRebuildShieldsWindow, type RebuildShieldsWindow } from "./rebuild-shields"; -export type RebuildSandboxEntry = registry.SandboxEntry & { agents?: unknown[] }; +export type RebuildSandboxEntry = SandboxEntry & { agents?: unknown[] }; export type RebuildLiveState = { staleRecovery: boolean; - staleRegistrySnapshot: ReturnType | null; + staleRegistrySnapshot: ReturnType | null; }; export type RebuildAgentBaseImageOptions = { @@ -224,7 +225,7 @@ export async function resolveRebuildLiveState( ); return { staleRecovery: true, - staleRegistrySnapshot: JSON.parse(JSON.stringify(registry.load())), + staleRegistrySnapshot: JSON.parse(JSON.stringify(loadRegistry())), }; } diff --git a/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts b/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts index 07b2eb50540..0f0b1880eb2 100644 --- a/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts +++ b/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts @@ -8,6 +8,7 @@ import * as openshellRuntime from "../../adapters/openshell/runtime"; import * as gatewayRuntime from "../../gateway-runtime-action"; import * as dockerDriverRecovery from "../../onboard/docker-driver-sandbox-recovery"; import * as registry from "../../state/registry"; +import * as registryPersistence from "../../state/registry/persistence"; import { type RebuildSandboxEntry, resolveRebuildLiveState } from "./rebuild-flow-helpers"; import { checkRebuildGatewaySchemaPreflight, @@ -84,7 +85,7 @@ describe("rebuild gateway drift preflight", () => { .spyOn(dockerDriverRecovery, "recoverDockerDriverSandbox") .mockReturnValue({ recovered: false, via: null }); vi.spyOn(registry, "getSandbox").mockReturnValue(makeSandboxEntry() as never); - vi.spyOn(registry, "load").mockReturnValue({ + vi.spyOn(registryPersistence, "load").mockReturnValue({ sandboxes: { alpha: makeSandboxEntry() }, } as never); errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -138,7 +139,7 @@ describe("rebuild gateway drift preflight", () => { const entry = makeSandboxEntry(recordedGateway, recordedPort); const registrySnapshot = { sandboxes: { alpha: entry } }; vi.mocked(registry.getSandbox).mockReturnValue(entry as never); - vi.mocked(registry.load).mockReturnValue(registrySnapshot as never); + vi.mocked(registryPersistence.load).mockReturnValue(registrySnapshot as never); captureOpenshellSpy .mockReturnValueOnce({ status: 0, output: "" }) .mockReturnValueOnce({ status: 1, output: "Error: × Not Found: sandbox not found" }); @@ -165,7 +166,7 @@ describe("rebuild gateway drift preflight", () => { expect.anything(), ); expect(recoverDockerDriverSandboxSpy).toHaveBeenCalledWith("alpha"); - expect(registry.load).toHaveBeenCalledOnce(); + expect(registryPersistence.load).toHaveBeenCalledOnce(); expect(logSpy.mock.calls.flat().join("\n")).toContain("absent from the live OpenShell gateway"); expect(behaviorLog.mock.calls.flat().join("\n")).toContain("Stale-sandbox recovery"); }); @@ -180,7 +181,7 @@ describe("rebuild gateway drift preflight", () => { const entry = makeSandboxEntry(gatewayName, gatewayPort); const registrySnapshot = { sandboxes: { alpha: entry } }; vi.mocked(registry.getSandbox).mockReturnValue(entry as never); - vi.mocked(registry.load).mockReturnValue(registrySnapshot as never); + vi.mocked(registryPersistence.load).mockReturnValue(registrySnapshot as never); captureOpenshellSpy .mockReturnValueOnce({ status: 1, @@ -220,7 +221,7 @@ describe("rebuild gateway drift preflight", () => { ); expect(getNamedGatewayLifecycleStateSpy).not.toHaveBeenCalled(); expect(recoverDockerDriverSandboxSpy).toHaveBeenCalledWith("alpha"); - expect(registry.load).toHaveBeenCalledOnce(); + expect(registryPersistence.load).toHaveBeenCalledOnce(); expect(logSpy.mock.calls.flat().join("\n")).toContain("absent from the live OpenShell gateway"); expect(behaviorLog.mock.calls.flat().join("\n")).toContain("Stale-sandbox recovery"); }); @@ -245,7 +246,7 @@ describe("rebuild gateway drift preflight", () => { expect(captureOpenshellSpy).toHaveBeenCalledWith(["sandbox", "list"]); expect(getNamedGatewayLifecycleStateSpy).not.toHaveBeenCalled(); expect(recoverDockerDriverSandboxSpy).not.toHaveBeenCalled(); - expect(registry.load).not.toHaveBeenCalled(); + expect(registryPersistence.load).not.toHaveBeenCalled(); expect(errorSpy.mock.calls.flat().join("\n")).toContain("Failed to query running sandboxes"); }); }); diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index f28d95f83cd..1f973340629 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -10,7 +10,7 @@ import { hydrateCredentialEnv } from "../../onboard/credential-env"; import { DOCKER_GPU_PATCH_NETWORK_ENV } from "../../onboard/docker-gpu-patch"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import * as onboardSession from "../../state/onboard-session"; -import * as registry from "../../state/registry"; +import { load as loadRegistry } from "../../state/registry/persistence"; import { normalizeRebuildTargetPolicyPresets, runRebuildBackupPhase } from "./rebuild-backup-phase"; import { buildRefreshMutableOpenClawConfigHashCommand } from "./rebuild-config-hash"; import { DCODE_AGENT_NAME } from "./rebuild-dcode-target"; @@ -126,7 +126,7 @@ async function rebuildSandboxUnlocked( try { if (blockRebuildOnPendingBaselineTransition(sandboxEntry, sandboxName, bail)) return; let recoveryRegistrySnapshot = preparedBackupRecovery - ? JSON.parse(JSON.stringify(registry.load())) + ? JSON.parse(JSON.stringify(loadRegistry())) : liveState.staleRegistrySnapshot; const registryRollback = createRebuildRegistryRollback({ sandboxName, diff --git a/src/lib/actions/sandbox/rebuild-prepared-recovery.ts b/src/lib/actions/sandbox/rebuild-prepared-recovery.ts index a2ae73b104d..0ca28003e40 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-recovery.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-recovery.ts @@ -4,7 +4,8 @@ import { isDeepStrictEqual } from "node:util"; import { RD as _RD, R } from "../../cli/terminal-style"; -import * as registry from "../../state/registry"; +import type { SandboxRegistry } from "../../state/registry"; +import { load as loadRegistry } from "../../state/registry/persistence"; import * as sandboxState from "../../state/sandbox"; import type { RebuildBail } from "./rebuild-credential-preflight"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; @@ -89,16 +90,16 @@ export function revalidatePreparedRecoveryBeforeDelete( sandboxName: string, initialEntry: RebuildSandboxEntry, candidate: sandboxState.RebuildManifest | null, - registrySnapshot: registry.SandboxRegistry | null, + registrySnapshot: SandboxRegistry | null, allowLegacyManagedImageRecovery: boolean, bail: RebuildBail, ): { manifest: sandboxState.RebuildManifest | null; - registrySnapshot: registry.SandboxRegistry | null; + registrySnapshot: SandboxRegistry | null; } { if (!candidate) return { manifest: null, registrySnapshot }; - const refreshedRegistrySnapshot = registry.load(); + const refreshedRegistrySnapshot = loadRegistry(); const currentEntry = refreshedRegistrySnapshot.sandboxes[sandboxName]; if (!currentEntry) { return failPreparedRecoveryPreDelete( diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index a2a9d763ba9..e77ba03c8ab 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -1,41 +1,30 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import path from "node:path"; import { isDeepStrictEqual } from "node:util"; -import { isErrnoException } from "../core/errno"; -import { isObjectRecord } from "../core/json-types"; -import { GATEWAY_PORT } from "../core/ports"; import type { InferenceSelection } from "../inference/selection"; import { inferenceSelectionRegistryFields, normalizeInferenceSelection, } from "../inference/selection"; import { normalizeToolDisclosure, type ToolDisclosure } from "../tool-disclosure"; -import { ensureConfigDir, readConfigFile, writeConfigFile } from "./config-io"; import { applyAddExtraProvider, applyRemoveExtraProvider, isValidExtraProviderName, - normalizeExtraProviders, readExtraProviders, } from "./extra-providers"; import type { OpenClawImagePluginInstall } from "./openclaw-plugin-restore"; -import { - normalizeSandboxMcpState, - type SandboxMcpState, - serializeSandboxMcpStateForDisk, -} from "./registry-mcp"; +import { normalizeSandboxMcpState, type SandboxMcpState } from "./registry-mcp"; import type { SandboxMessagingState } from "./registry-messaging"; import { normalizeBaselineExclusions, normalizeBaselineExclusionTransition, - parseSandboxRegistryEntries, retainedDefaultSandbox, } from "./registry-normalization"; import * as reversibleRemoval from "./registry-reversible-removal"; -import { nemoclawStateRoot } from "./state-root"; +import { withLock } from "./registry/lock"; +import { load, save } from "./registry/persistence"; export { getSandboxEntryDisplayInference, @@ -53,12 +42,26 @@ import { cloneSandboxMessagingState, getConfiguredMessagingChannels as getRegistryConfiguredMessagingChannels, getDisabledChannels as getRegistryDisabledChannels, - serializeSandboxMessagingStateForDisk, setChannelDisabled as setRegistryChannelDisabled, } from "./registry-messaging"; export type { McpBridgeEntry, SandboxMcpState } from "./registry-mcp"; +export { + acquireLock, + classifyExistingLock, + LOCK_DIR, + LOCK_MAX_RETRIES, + LOCK_OWNER, + LOCK_RETRY_MS, + LOCK_STALE_MS, + releaseLock, + type RegistryLockDecision, + withLock, +} from "./registry/lock"; + +export { load, REGISTRY_FILE, save } from "./registry/persistence"; + export { getConfiguredMessagingChannelsFromEntry, getDisabledMessagingChannelsFromEntry, @@ -208,334 +211,6 @@ export interface SandboxRegistry { export type SandboxRemovalReceipt = reversibleRemoval.RegistryRemovalReceipt; -export const REGISTRY_FILE = path.join( - nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT), - "sandboxes.json", -); -export const LOCK_DIR = `${REGISTRY_FILE}.lock`; -export const LOCK_OWNER = path.join(LOCK_DIR, "owner"); -export const LOCK_STALE_MS = 10_000; -export const LOCK_RETRY_MS = 100; -export const LOCK_MAX_RETRIES = 120; -/** kill(pid, 0) liveness probe. EPERM means the pid exists but is owned by - * another user, which still counts as alive. */ -function isProcessAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (error) { - return isErrnoException(error) && error.code === "EPERM"; - } -} - -/** Wall-clock start time (ms since epoch) of `pid` from /proc, or null when it - * cannot be read (process gone, or a non-Linux host without /proc). Mirrors the - * onboard-session lock's recycle check. */ -function readProcessStartMs(pid: number): number | null { - try { - const statText = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); - const btimeLine = fs - .readFileSync("/proc/stat", "utf8") - .split("\n") - .find((line) => line.startsWith("btime ")); - const bootSeconds = btimeLine ? Number(btimeLine.trim().split(/\s+/)[1]) : NaN; - const closeParen = statText.lastIndexOf(")"); - if (!Number.isFinite(bootSeconds) || closeParen < 0) return null; - const fieldsAfterComm = statText - .slice(closeParen + 2) - .trim() - .split(/\s+/); - const startTicks = Number(fieldsAfterComm[19]); - if (!Number.isFinite(startTicks)) return null; - // /proc//stat starttime is in USER_HZ ticks (100 on supported hosts). - const clockTicksPerSecond = 100; - return (bootSeconds + startTicks / clockTicksPerSecond) * 1000; - } catch { - return null; - } -} - -export type RegistryLockDecision = "break" | "wait"; - -/** - * Decide whether an existing registry lock should be broken (stale) or waited - * on. Exported for tests. - * - * The PID-recycle wedge this guards against: a holder that crashes without - * releasing leaves `LOCK_DIR` + the owner pid behind. If that pid is later - * reused by an unrelated live process, `kill(pid, 0)` succeeds, so a - * liveness-only check treats the lock as held forever and every registry write - * wedges (retries exhausted -> "Failed to acquire lock"). When the owner looks - * alive we therefore also confirm it started BEFORE it took the lock: a process - * whose /proc start time is after the lock's mtime is a recycled pid, so the - * lock is stale. When the owner pid or its start time cannot be read (missing - * owner file, non-Linux host), fall back to breaking the lock once it is older - * than a registry op could legitimately take. - */ -export function classifyExistingLock(opts: { - ownerPid: number | null; - ownerAlive: boolean; - processStartMs: number | null; - lockMtimeMs: number; - nowMs: number; - staleMs: number; -}): RegistryLockDecision { - const ageMs = opts.nowMs - opts.lockMtimeMs; - if (opts.ownerPid === null) { - // Owner file missing or unreadable: decide on age alone. - return ageMs > opts.staleMs ? "break" : "wait"; - } - if (!opts.ownerAlive) { - return "break"; - } - if (opts.processStartMs !== null && opts.processStartMs > opts.lockMtimeMs + 1000) { - // Live pid that started after the lock was taken -> the pid was recycled. - return "break"; - } - // Live original holder (or start time unknown): only break once the lock is - // clearly older than a registry op could take, which also covers hosts where - // recycle cannot be detected directly. - return ageMs > opts.staleMs ? "break" : "wait"; -} - -/** Acquire an advisory lock using mkdir (atomic on POSIX). */ -export function acquireLock(): void { - ensureConfigDir(path.dirname(REGISTRY_FILE)); - const sleepBuf = new Int32Array(new SharedArrayBuffer(4)); - for (let i = 0; i < LOCK_MAX_RETRIES; i++) { - try { - fs.mkdirSync(LOCK_DIR); - const ownerTmp = `${LOCK_OWNER}.tmp.${process.pid}`; - try { - fs.writeFileSync(ownerTmp, String(process.pid), { mode: 0o600 }); - fs.renameSync(ownerTmp, LOCK_OWNER); - } catch (ownerErr) { - try { - fs.unlinkSync(ownerTmp); - } catch { - /* best effort */ - } - try { - fs.unlinkSync(LOCK_OWNER); - } catch { - /* best effort */ - } - try { - fs.rmdirSync(LOCK_DIR); - } catch { - /* best effort */ - } - throw ownerErr; - } - return; - } catch (error) { - if (!isErrnoException(error) || error.code !== "EEXIST") { - throw error; - } - let lockStat: fs.Stats; - try { - lockStat = fs.statSync(LOCK_DIR); - } catch { - // Lock dir vanished between the failed mkdir and this stat: another - // waiter released it, so retry immediately. - continue; - } - let ownerPid: number | null = null; - try { - const parsed = Number.parseInt(fs.readFileSync(LOCK_OWNER, "utf-8").trim(), 10); - ownerPid = Number.isFinite(parsed) && parsed > 0 ? parsed : null; - } catch { - ownerPid = null; - } - const ownerAlive = ownerPid !== null ? isProcessAlive(ownerPid) : false; - const processStartMs = ownerPid !== null && ownerAlive ? readProcessStartMs(ownerPid) : null; - const decision = classifyExistingLock({ - ownerPid, - ownerAlive, - processStartMs, - lockMtimeMs: lockStat.mtimeMs, - nowMs: Date.now(), - staleMs: LOCK_STALE_MS, - }); - if (decision === "break") { - // Only break the lock if it is provably the same one we classified. - // Re-stat LOCK_DIR and require the inode + mtime to be unchanged (a - // replacement lock is a fresh mkdir, hence a new inode) and, when the - // owner pid was readable, that it still matches. Any stat/read failure - // means the identity cannot be proven, so the lock is left alone rather - // than risk clobbering an in-flight replacement that exists as LOCK_DIR - // before its owner file has been written. - let stillSameLock = false; - try { - const currentStat = fs.statSync(LOCK_DIR); - stillSameLock = - currentStat.ino === lockStat.ino && currentStat.mtimeMs === lockStat.mtimeMs; - if (stillSameLock && ownerPid !== null) { - const recheck = Number.parseInt(fs.readFileSync(LOCK_OWNER, "utf-8").trim(), 10); - stillSameLock = recheck === ownerPid; - } - } catch { - stillSameLock = false; - } - if (stillSameLock) { - fs.rmSync(LOCK_DIR, { recursive: true, force: true }); - continue; - } - } - Atomics.wait(sleepBuf, 0, 0, LOCK_RETRY_MS); - } - } - throw new Error(`Failed to acquire lock on ${REGISTRY_FILE} after ${LOCK_MAX_RETRIES} retries`); -} - -export function releaseLock(): void { - try { - fs.unlinkSync(LOCK_OWNER); - } catch (error) { - if (!isErrnoException(error) || error.code !== "ENOENT") { - throw error; - } - } - try { - fs.rmSync(LOCK_DIR, { recursive: true, force: true }); - } catch (error) { - if (!isErrnoException(error) || error.code !== "ENOENT") { - throw error; - } - } -} - -export function withLock(fn: () => T): T { - acquireLock(); - try { - return fn(); - } finally { - releaseLock(); - } -} - -export function load(): SandboxRegistry { - return normalizeRegistry( - readConfigFile(REGISTRY_FILE, { sandboxes: {}, defaultSandbox: null }), - ); -} - -export function save(data: SandboxRegistry): void { - writeConfigFile(REGISTRY_FILE, serializeRegistryForDisk(data)); -} - -function normalizeRegistry(value: unknown): SandboxRegistry { - const data = isObjectRecord(value) ? value : {}; - const extraProviders = normalizeExtraProviders(data.extraProviders); - const sandboxes = Object.fromEntries( - parseSandboxRegistryEntries(data.sandboxes).map(([name, entry]) => [ - name, - normalizeSandboxEntryForRuntime(entry), - ]), - ); - const base: SandboxRegistry = { - // Preserve a stale string pointer at read time so diagnostics can explain - // which sandbox disappeared. Mutation paths repair it before persistence. - defaultSandbox: typeof data.defaultSandbox === "string" ? data.defaultSandbox : null, - defaultSelectionRevision: reversibleRemoval.normalizeDefaultSelectionRevision( - data.defaultSelectionRevision, - ), - sandboxes, - }; - if (extraProviders) base.extraProviders = extraProviders; - return base; -} - -function serializeRegistryForDisk(data: SandboxRegistry): SandboxRegistry { - const extraProviders = normalizeExtraProviders(data.extraProviders); - const sandboxes = Object.fromEntries( - Object.entries(data.sandboxes).map(([name, entry]) => [ - name, - serializeSandboxEntryForDisk(entry), - ]), - ); - const defaultSandbox = retainedDefaultSandbox(data.defaultSandbox, sandboxes); - const currentDefaultSelectionRevision = reversibleRemoval.normalizeDefaultSelectionRevision( - data.defaultSelectionRevision, - ); - const base: SandboxRegistry = { - defaultSandbox, - defaultSelectionRevision: - defaultSandbox === data.defaultSandbox - ? currentDefaultSelectionRevision - : reversibleRemoval.incrementDefaultSelectionRevision(currentDefaultSelectionRevision), - sandboxes, - }; - if (extraProviders) base.extraProviders = extraProviders; - return base; -} - -function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { - const messaging = cloneSandboxMessagingState(entry.messaging); - const mcp = normalizeSandboxMcpState(entry.mcp); - const baselineExclusions = normalizeBaselineExclusions(entry.baselineExclusions); - const baselineExclusionTransition = normalizeBaselineExclusionTransition( - entry.baselineExclusionTransition, - ); - const { - messaging: _messaging, - mcp: _mcp, - baselineExclusions: _baselineExclusions, - baselineExclusionTransition: _baselineExclusionTransition, - ...rest - } = entry; - return { - ...rest, - ...(messaging ? { messaging } : {}), - ...(mcp ? { mcp } : {}), - ...(baselineExclusions ? { baselineExclusions } : {}), - ...(baselineExclusionTransition ? { baselineExclusionTransition } : {}), - }; -} - -/** - * Prepare a sandbox entry for persistence: canonicalize a no-dashboard port to - * null, normalize messaging state, and drop transient #5714 display-only - * markers plus legacy provider credential hashes that must never reach - * sandboxes.json. - */ -function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { - // Defensively drop non-durable recovery markers and legacy - // providerCredentialHashes so they can never reach sandboxes.json even if a - // caller force-passed them through updateSandbox(). - const { - recoveredFromGateway: _recovered, - livePhase: _phase, - providerCredentialHashes: _legacyProviderCredentialHashes, - ...durable - } = entry as SandboxEntry & { - recoveredFromGateway?: boolean; - livePhase?: string | null; - providerCredentialHashes?: unknown; - }; - const messaging = serializeSandboxMessagingStateForDisk(durable.messaging); - const mcp = serializeSandboxMcpStateForDisk(durable.mcp); - const baselineExclusions = normalizeBaselineExclusions(durable.baselineExclusions); - const baselineExclusionTransition = normalizeBaselineExclusionTransition( - durable.baselineExclusionTransition, - ); - const { - messaging: _messaging, - mcp: _mcp, - baselineExclusions: _baselineExclusions, - baselineExclusionTransition: _baselineExclusionTransition, - ...rest - } = durable; - return { - ...rest, - ...(rest.dashboardPort === 0 ? { dashboardPort: null } : {}), - ...(messaging ? { messaging } : {}), - ...(mcp ? { mcp } : {}), - ...(baselineExclusions ? { baselineExclusions } : {}), - ...(baselineExclusionTransition ? { baselineExclusionTransition } : {}), - }; -} - export function getSandbox(name: string): SandboxEntry | null { const data = load(); return data.sandboxes[name] || null; diff --git a/src/lib/state/registry/lock.ts b/src/lib/state/registry/lock.ts new file mode 100644 index 00000000000..37e1a19e57d --- /dev/null +++ b/src/lib/state/registry/lock.ts @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { isErrnoException } from "../../core/errno"; +import { ensureConfigDir } from "../config-io"; +import { REGISTRY_FILE } from "./persistence"; + +export const LOCK_DIR = `${REGISTRY_FILE}.lock`; +export const LOCK_OWNER = path.join(LOCK_DIR, "owner"); +export const LOCK_STALE_MS = 10_000; +export const LOCK_RETRY_MS = 100; +export const LOCK_MAX_RETRIES = 120; +/** kill(pid, 0) liveness probe. EPERM means the pid exists but is owned by + * another user, which still counts as alive. */ +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return isErrnoException(error) && error.code === "EPERM"; + } +} + +/** Wall-clock start time (ms since epoch) of `pid` from /proc, or null when it + * cannot be read (process gone, or a non-Linux host without /proc). Mirrors the + * onboard-session lock's recycle check. */ +function readProcessStartMs(pid: number): number | null { + try { + const statText = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const btimeLine = fs + .readFileSync("/proc/stat", "utf8") + .split("\n") + .find((line) => line.startsWith("btime ")); + const bootSeconds = btimeLine ? Number(btimeLine.trim().split(/\s+/)[1]) : NaN; + const closeParen = statText.lastIndexOf(")"); + if (!Number.isFinite(bootSeconds) || closeParen < 0) return null; + const fieldsAfterComm = statText + .slice(closeParen + 2) + .trim() + .split(/\s+/); + const startTicks = Number(fieldsAfterComm[19]); + if (!Number.isFinite(startTicks)) return null; + // /proc//stat starttime is in USER_HZ ticks (100 on supported hosts). + const clockTicksPerSecond = 100; + return (bootSeconds + startTicks / clockTicksPerSecond) * 1000; + } catch { + return null; + } +} + +export type RegistryLockDecision = "break" | "wait"; + +/** + * Decide whether an existing registry lock should be broken (stale) or waited + * on. Exported for tests. + * + * The PID-recycle wedge this guards against: a holder that crashes without + * releasing leaves `LOCK_DIR` + the owner pid behind. If that pid is later + * reused by an unrelated live process, `kill(pid, 0)` succeeds, so a + * liveness-only check treats the lock as held forever and every registry write + * wedges (retries exhausted -> "Failed to acquire lock"). When the owner looks + * alive we therefore also confirm it started BEFORE it took the lock: a process + * whose /proc start time is after the lock's mtime is a recycled pid, so the + * lock is stale. When the owner pid or its start time cannot be read (missing + * owner file, non-Linux host), fall back to breaking the lock once it is older + * than a registry op could legitimately take. + */ +export function classifyExistingLock(opts: { + ownerPid: number | null; + ownerAlive: boolean; + processStartMs: number | null; + lockMtimeMs: number; + nowMs: number; + staleMs: number; +}): RegistryLockDecision { + const ageMs = opts.nowMs - opts.lockMtimeMs; + if (opts.ownerPid === null) { + // Owner file missing or unreadable: decide on age alone. + return ageMs > opts.staleMs ? "break" : "wait"; + } + if (!opts.ownerAlive) { + return "break"; + } + if (opts.processStartMs !== null && opts.processStartMs > opts.lockMtimeMs + 1000) { + // Live pid that started after the lock was taken -> the pid was recycled. + return "break"; + } + // Live original holder (or start time unknown): only break once the lock is + // clearly older than a registry op could take, which also covers hosts where + // recycle cannot be detected directly. + return ageMs > opts.staleMs ? "break" : "wait"; +} + +/** Acquire an advisory lock using mkdir (atomic on POSIX). */ +export function acquireLock(): void { + ensureConfigDir(path.dirname(REGISTRY_FILE)); + const sleepBuf = new Int32Array(new SharedArrayBuffer(4)); + for (let i = 0; i < LOCK_MAX_RETRIES; i++) { + try { + fs.mkdirSync(LOCK_DIR); + const ownerTmp = `${LOCK_OWNER}.tmp.${process.pid}`; + try { + fs.writeFileSync(ownerTmp, String(process.pid), { mode: 0o600 }); + fs.renameSync(ownerTmp, LOCK_OWNER); + } catch (ownerErr) { + try { + fs.unlinkSync(ownerTmp); + } catch { + /* best effort */ + } + try { + fs.unlinkSync(LOCK_OWNER); + } catch { + /* best effort */ + } + try { + fs.rmdirSync(LOCK_DIR); + } catch { + /* best effort */ + } + throw ownerErr; + } + return; + } catch (error) { + if (!isErrnoException(error) || error.code !== "EEXIST") { + throw error; + } + let lockStat: fs.Stats; + try { + lockStat = fs.statSync(LOCK_DIR); + } catch { + // Lock dir vanished between the failed mkdir and this stat: another + // waiter released it, so retry immediately. + continue; + } + let ownerPid: number | null = null; + try { + const parsed = Number.parseInt(fs.readFileSync(LOCK_OWNER, "utf-8").trim(), 10); + ownerPid = Number.isFinite(parsed) && parsed > 0 ? parsed : null; + } catch { + ownerPid = null; + } + const ownerAlive = ownerPid !== null ? isProcessAlive(ownerPid) : false; + const processStartMs = ownerPid !== null && ownerAlive ? readProcessStartMs(ownerPid) : null; + const decision = classifyExistingLock({ + ownerPid, + ownerAlive, + processStartMs, + lockMtimeMs: lockStat.mtimeMs, + nowMs: Date.now(), + staleMs: LOCK_STALE_MS, + }); + if (decision === "break") { + // Only break the lock if it is provably the same one we classified. + // Re-stat LOCK_DIR and require the inode + mtime to be unchanged (a + // replacement lock is a fresh mkdir, hence a new inode) and, when the + // owner pid was readable, that it still matches. Any stat/read failure + // means the identity cannot be proven, so the lock is left alone rather + // than risk clobbering an in-flight replacement that exists as LOCK_DIR + // before its owner file has been written. + let stillSameLock = false; + try { + const currentStat = fs.statSync(LOCK_DIR); + stillSameLock = + currentStat.ino === lockStat.ino && currentStat.mtimeMs === lockStat.mtimeMs; + if (stillSameLock && ownerPid !== null) { + const recheck = Number.parseInt(fs.readFileSync(LOCK_OWNER, "utf-8").trim(), 10); + stillSameLock = recheck === ownerPid; + } + } catch { + stillSameLock = false; + } + if (stillSameLock) { + fs.rmSync(LOCK_DIR, { recursive: true, force: true }); + continue; + } + } + Atomics.wait(sleepBuf, 0, 0, LOCK_RETRY_MS); + } + } + throw new Error(`Failed to acquire lock on ${REGISTRY_FILE} after ${LOCK_MAX_RETRIES} retries`); +} + +export function releaseLock(): void { + try { + fs.unlinkSync(LOCK_OWNER); + } catch (error) { + if (!isErrnoException(error) || error.code !== "ENOENT") { + throw error; + } + } + try { + fs.rmSync(LOCK_DIR, { recursive: true, force: true }); + } catch (error) { + if (!isErrnoException(error) || error.code !== "ENOENT") { + throw error; + } + } +} + +export function withLock(fn: () => T): T { + acquireLock(); + try { + return fn(); + } finally { + releaseLock(); + } +} diff --git a/src/lib/state/registry/persistence.ts b/src/lib/state/registry/persistence.ts new file mode 100644 index 00000000000..804fbe12048 --- /dev/null +++ b/src/lib/state/registry/persistence.ts @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; +import { isObjectRecord } from "../../core/json-types"; +import { GATEWAY_PORT } from "../../core/ports"; +import { readConfigFile, writeConfigFile } from "../config-io"; +import { normalizeExtraProviders } from "../extra-providers"; +import { normalizeSandboxMcpState, serializeSandboxMcpStateForDisk } from "../registry-mcp"; +import { + cloneSandboxMessagingState, + serializeSandboxMessagingStateForDisk, +} from "../registry-messaging"; +import { + normalizeBaselineExclusions, + normalizeBaselineExclusionTransition, + parseSandboxRegistryEntries, + retainedDefaultSandbox, +} from "../registry-normalization"; +import * as reversibleRemoval from "../registry-reversible-removal"; +import type { SandboxEntry, SandboxRegistry } from "../registry"; +import { nemoclawStateRoot } from "../state-root"; + +export const REGISTRY_FILE = path.join( + nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT), + "sandboxes.json", +); +export function load(): SandboxRegistry { + return normalizeRegistry( + readConfigFile(REGISTRY_FILE, { sandboxes: {}, defaultSandbox: null }), + ); +} + +export function save(data: SandboxRegistry): void { + writeConfigFile(REGISTRY_FILE, serializeRegistryForDisk(data)); +} + +function normalizeRegistry(value: unknown): SandboxRegistry { + const data = isObjectRecord(value) ? value : {}; + const extraProviders = normalizeExtraProviders(data.extraProviders); + const sandboxes = Object.fromEntries( + parseSandboxRegistryEntries(data.sandboxes).map(([name, entry]) => [ + name, + normalizeSandboxEntryForRuntime(entry), + ]), + ); + const base: SandboxRegistry = { + // Preserve a stale string pointer at read time so diagnostics can explain + // which sandbox disappeared. Mutation paths repair it before persistence. + defaultSandbox: typeof data.defaultSandbox === "string" ? data.defaultSandbox : null, + defaultSelectionRevision: reversibleRemoval.normalizeDefaultSelectionRevision( + data.defaultSelectionRevision, + ), + sandboxes, + }; + if (extraProviders) base.extraProviders = extraProviders; + return base; +} + +function serializeRegistryForDisk(data: SandboxRegistry): SandboxRegistry { + const extraProviders = normalizeExtraProviders(data.extraProviders); + const sandboxes = Object.fromEntries( + Object.entries(data.sandboxes).map(([name, entry]) => [ + name, + serializeSandboxEntryForDisk(entry), + ]), + ); + const defaultSandbox = retainedDefaultSandbox(data.defaultSandbox, sandboxes); + const currentDefaultSelectionRevision = reversibleRemoval.normalizeDefaultSelectionRevision( + data.defaultSelectionRevision, + ); + const base: SandboxRegistry = { + defaultSandbox, + defaultSelectionRevision: + defaultSandbox === data.defaultSandbox + ? currentDefaultSelectionRevision + : reversibleRemoval.incrementDefaultSelectionRevision(currentDefaultSelectionRevision), + sandboxes, + }; + if (extraProviders) base.extraProviders = extraProviders; + return base; +} + +function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { + const messaging = cloneSandboxMessagingState(entry.messaging); + const mcp = normalizeSandboxMcpState(entry.mcp); + const baselineExclusions = normalizeBaselineExclusions(entry.baselineExclusions); + const baselineExclusionTransition = normalizeBaselineExclusionTransition( + entry.baselineExclusionTransition, + ); + const { + messaging: _messaging, + mcp: _mcp, + baselineExclusions: _baselineExclusions, + baselineExclusionTransition: _baselineExclusionTransition, + ...rest + } = entry; + return { + ...rest, + ...(messaging ? { messaging } : {}), + ...(mcp ? { mcp } : {}), + ...(baselineExclusions ? { baselineExclusions } : {}), + ...(baselineExclusionTransition ? { baselineExclusionTransition } : {}), + }; +} + +/** + * Prepare a sandbox entry for persistence: canonicalize a no-dashboard port to + * null, normalize messaging state, and drop transient #5714 display-only + * markers plus legacy provider credential hashes that must never reach + * sandboxes.json. + */ +function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { + // Defensively drop non-durable recovery markers and legacy + // providerCredentialHashes so they can never reach sandboxes.json even if a + // caller force-passed them through updateSandbox(). + const { + recoveredFromGateway: _recovered, + livePhase: _phase, + providerCredentialHashes: _legacyProviderCredentialHashes, + ...durable + } = entry as SandboxEntry & { + recoveredFromGateway?: boolean; + livePhase?: string | null; + providerCredentialHashes?: unknown; + }; + const messaging = serializeSandboxMessagingStateForDisk(durable.messaging); + const mcp = serializeSandboxMcpStateForDisk(durable.mcp); + const baselineExclusions = normalizeBaselineExclusions(durable.baselineExclusions); + const baselineExclusionTransition = normalizeBaselineExclusionTransition( + durable.baselineExclusionTransition, + ); + const { + messaging: _messaging, + mcp: _mcp, + baselineExclusions: _baselineExclusions, + baselineExclusionTransition: _baselineExclusionTransition, + ...rest + } = durable; + return { + ...rest, + ...(rest.dashboardPort === 0 ? { dashboardPort: null } : {}), + ...(messaging ? { messaging } : {}), + ...(mcp ? { mcp } : {}), + ...(baselineExclusions ? { baselineExclusions } : {}), + ...(baselineExclusionTransition ? { baselineExclusionTransition } : {}), + }; +} diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 3600c49f0f1..ef2846b26c6 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -35,6 +35,7 @@ const { rebuildOnboardDependencies } = requireDist("./rebuild-onboard-dependenci const onboardCredentialEnv = requireDist("../../onboard/credential-env.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const registry = requireDist("../../state/registry.js"); +const registryPersistence = requireDist("../../state/registry/persistence.js"); const sandboxState = requireDist("../../state/sandbox.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); const sandboxVersion = requireDist("../../sandbox/version.js"); @@ -426,7 +427,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): ) as never; }); let registryLoadCount = 0; - vi.spyOn(registry, "load").mockImplementation(() => { + vi.spyOn(registryPersistence, "load").mockImplementation(() => { const isPreDeleteRead = registryLoadCount > 0; registryLoadCount++; return { diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index 672441e2353..20aab708d39 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -40,6 +40,7 @@ const onboardCredentialEnv = requireDist("../../onboard/credential-env.js"); const hermesProviderAuth = requireDist("../../hermes-provider-auth.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const registry = requireDist("../../state/registry.js"); +const registryPersistence = requireDist("../../state/registry/persistence.js"); const sandboxState = requireDist("../../state/sandbox.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); const sandboxVersion = requireDist("../../sandbox/version.js"); @@ -243,7 +244,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): return true; }); let registryLoadCount = 0; - vi.spyOn(registry, "load").mockImplementation(() => { + vi.spyOn(registryPersistence, "load").mockImplementation(() => { const isPreDeleteRead = registryLoadCount > 0; registryLoadCount++; const defaultSandbox = isPreDeleteRead ? preDeleteDefaultSandbox : initialDefaultSandbox;