diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference-acceleration.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference-acceleration.test.ts index 27264e35b11..78ac6a6faff 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference-acceleration.test.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference-acceleration.test.ts @@ -32,6 +32,7 @@ function operationRuntime( engine: harness.engine, env: harness.env, acceleration: harness.operationAcceleration, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, @@ -51,6 +52,7 @@ describe("Podman host-local inference acceleration authority", () => { engine: harness.engine, env: harness.env, acceleration: "tpu" as never, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, @@ -69,6 +71,7 @@ describe("Podman host-local inference acceleration authority", () => { createPodmanHostLocalInferenceRuntime({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, authority, @@ -191,6 +194,7 @@ describe("Podman host-local inference acceleration authority", () => { const validationRuntime = createPodmanHostLocalInferenceRuntime({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, authority: qualifyPodmanInferenceAuthority(harness.engine), diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference-cleanup-settlement.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference-cleanup-settlement.test.ts new file mode 100644 index 00000000000..da932281fbe --- /dev/null +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference-cleanup-settlement.test.ts @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createPodmanHostLocalInferenceTestHarness } from "../../../../test/helpers/podman-host-local-inference-test-harness"; +import type { HostLocalInferenceRuntime } from "./host-local-inference"; +import { createPodmanHostLocalInferenceOperation } from "./podman-host-local-inference"; + +function operationRuntime( + harness: ReturnType, +): HostLocalInferenceRuntime { + const operation = createPodmanHostLocalInferenceOperation({ + engine: harness.engine, + env: harness.env, + acceleration: harness.operationAcceleration, + probeCleanupTiming: harness.probeCleanupTiming, + authorityStore: harness.authorityStore, + routeAuthorityStore: harness.routeAuthorityStore, + onFailureEvidence: harness.onFailureEvidence, + redactSensitive: harness.redactSensitive, + }); + return ( + operation.managedRuntime ?? + (() => { + throw new Error("test operation lacks managed runtime"); + })() + ); +} + +describe("Podman inference disposable-probe cleanup settlement", () => { + it("retries a timed-out pre-remove inspection before removing the same exact probe", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeCleanupInspectTimeoutsRemaining = 1; + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(2); + }); + + it("does not remove a probe that reached exact ID and name absence before cleanup", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeDisappearBeforeCleanupCount = 1; + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + }); + + it("settles delayed exact ID and name absence after one removal", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeRemovalIdObservationsRemaining = 1; + harness.state.probeRemovalNameObservationsRemaining = 2; + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + const sleeps = harness.events + .filter((event) => event.startsWith("probe-cleanup:sleep ")) + .map((event) => Number(event.slice("probe-cleanup:sleep ".length))); + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + expect(sleeps.length).toBeGreaterThan(0); + expect(sleeps.every((delay) => delay === 1_000)).toBe(true); + expect(sleeps.reduce((total, delay) => total + delay, 0)).toBeLessThanOrEqual(30_000); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(2); + }); + + it("accepts a nonzero removal result only after delayed exact absence", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeRemoveLostAcknowledgement = true; + harness.state.probeRemovalIdObservationsRemaining = 1; + harness.state.probeRemovalNameObservationsRemaining = 1; + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(2); + expect( + harness.failures.some(({ message }) => message.includes("probe removal returned exit 125")), + ).toBe(true); + }); + + it("accepts a timed-out removal result only after delayed exact absence", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeRemoveTimeout = true; + harness.state.probeRemovalIdObservationsRemaining = 1; + harness.state.probeRemovalNameObservationsRemaining = 1; + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(2); + expect( + harness.failures.some(({ message }) => message.includes("probe removal returned exit 1")), + ).toBe(true); + }); + + it("stops after one removal when the exact probe remains through the deadline", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeRemoveLeavesContainer = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + + const sleeps = harness.events + .filter((event) => event.startsWith("probe-cleanup:sleep ")) + .map((event) => Number(event.slice("probe-cleanup:sleep ".length))); + expect(sleeps.reduce((total, delay) => total + delay, 0)).toBe(30_000); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + expect(harness.probe()).not.toBeNull(); + }); + + it("accepts exact absence and final currentness at the settlement deadline", () => { + const clock = [0, 30_000, 30_000, 30_000, 30_000, 30_000]; + const harness = createPodmanHostLocalInferenceTestHarness({ + probeCleanupTiming: { + now: () => clock.shift() ?? 30_000, + sleep: () => undefined, + }, + }); + + const prepared = operationRuntime(harness).startManaged(harness.input, harness.writer); + + expect(prepared.receipt.service).toBe("nim"); + expect(harness.probe()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(2); + }); + + it("rejects absence first observed after the settlement deadline", () => { + const clock = [0, 30_001]; + const harness = createPodmanHostLocalInferenceTestHarness({ + probeCleanupTiming: { + now: () => clock.shift() ?? 30_001, + sleep: () => undefined, + }, + }); + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + expect(harness.probe()).toBeNull(); + expect(harness.container()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + }); + + it("rejects final authority currentness that completes after the settlement deadline", () => { + const clock = [0, 30_000, 30_001]; + const harness = createPodmanHostLocalInferenceTestHarness({ + probeCleanupTiming: { + now: () => clock.shift() ?? 30_001, + sleep: () => undefined, + }, + }); + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + expect(harness.probe()).toBeNull(); + expect(harness.container()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + }); + + it("rejects probe label drift after removal without a second mutation", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeRemovalIdObservationsRemaining = 1; + harness.state.probeCleanupLabelDriftAfterRemoval = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + expect(harness.probe()).not.toBeNull(); + expect(harness.container()).toBeNull(); + }); + + it("rejects probe spec drift after removal without a second mutation", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeRemovalIdObservationsRemaining = 1; + harness.state.probeCleanupSpecDriftAfterRemoval = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + expect(harness.probe()).not.toBeNull(); + expect(harness.container()).toBeNull(); + }); + + it("rejects a terminal existence-read failure without removal or settlement sleep", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeCleanupExistenceFailure = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup lost exact identity", + ); + expect(harness.events.some((event) => event === `podman:rm --force ${"c".repeat(64)}`)).toBe( + false, + ); + expect(harness.events.some((event) => event.startsWith("probe-cleanup:sleep "))).toBe(false); + expect(harness.probe()).not.toBeNull(); + }); + + it("rejects a terminal inspect-read failure without removal or settlement sleep", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeCleanupInspectFailure = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup lost exact identity", + ); + expect(harness.events.some((event) => event === `podman:rm --force ${"c".repeat(64)}`)).toBe( + false, + ); + expect(harness.events.some((event) => event.startsWith("probe-cleanup:sleep "))).toBe(false); + expect(harness.probe()).not.toBeNull(); + }); + + it("rejects a malformed inspection without removal or settlement sleep", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeCleanupMalformedInspection = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup lost exact identity", + ); + expect(harness.events.some((event) => event === `podman:rm --force ${"c".repeat(64)}`)).toBe( + false, + ); + expect(harness.events.some((event) => event.startsWith("probe-cleanup:sleep "))).toBe(false); + expect(harness.probe()).not.toBeNull(); + }); + + it("rejects an ambiguous name lookup without removal or settlement sleep", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeCleanupAmbiguousLookup = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup lost exact identity", + ); + expect(harness.events.some((event) => event === `podman:rm --force ${"c".repeat(64)}`)).toBe( + false, + ); + expect(harness.events.some((event) => event.startsWith("probe-cleanup:sleep "))).toBe(false); + expect(harness.probe()).not.toBeNull(); + }); + + it("rejects network-authority drift before probe removal", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeNetworkDriftBeforeRemoval = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "Podman inference network identity or name changed after qualification.", + ); + expect(harness.events.some((event) => event === `podman:rm --force ${"c".repeat(64)}`)).toBe( + false, + ); + expect(harness.probe()).not.toBeNull(); + expect(harness.container()).toBeNull(); + }); + + it("restores the parent after network-authority drift during cleanup settlement", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeNetworkDriftAfterRemoval = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + expect(harness.probe()).toBeNull(); + expect(harness.container()).toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + }); + + it("reports restoration failure after engine drift during cleanup settlement", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeEngineDriftAfterRemoval = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "Exact prior-runtime restoration also failed", + ); + expect(harness.probe()).toBeNull(); + expect(harness.container()).not.toBeNull(); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + }); + + it("reports restoration failure after engine drift before probe removal", () => { + const harness = createPodmanHostLocalInferenceTestHarness(); + harness.state.probeEngineDriftBeforeRemoval = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "Exact prior-runtime restoration also failed", + ); + expect(harness.events.some((event) => event === `podman:rm --force ${"c".repeat(64)}`)).toBe( + false, + ); + expect(harness.probe()).not.toBeNull(); + expect(harness.container()).not.toBeNull(); + }); + + it("rejects an invalid cleanup clock after the single removal", () => { + const harness = createPodmanHostLocalInferenceTestHarness({ + probeCleanupTiming: { now: () => Number.NaN, sleep: () => undefined }, + }); + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + expect(harness.container()).toBeNull(); + }); + + it("rejects a backward cleanup clock during settlement", () => { + const clock = [1_000, 1_000, 999]; + const harness = createPodmanHostLocalInferenceTestHarness({ + probeCleanupTiming: { + now: () => clock.shift() ?? 999, + sleep: () => undefined, + }, + }); + harness.state.probeRemoveLeavesContainer = true; + + expect(() => operationRuntime(harness).startManaged(harness.input, harness.writer)).toThrow( + "probe cleanup is indeterminate", + ); + expect( + harness.events.filter((event) => event === `podman:rm --force ${"c".repeat(64)}`), + ).toHaveLength(1); + expect(harness.probe()).not.toBeNull(); + }); +}); diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference-ollama.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference-ollama.test.ts index 7b786a30bcf..2fccdbf62c8 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference-ollama.test.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference-ollama.test.ts @@ -33,6 +33,7 @@ function managedOllamaFixture( engine: harness.engine, env: harness.env, acceleration: harness.operationAcceleration, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, ...(options.externalNetwork === false diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts index 7e82bbcb28a..1027fa64155 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference-probe-inspect.test.ts @@ -14,6 +14,7 @@ function operationRuntime( engine: harness.engine, env: harness.env, acceleration: harness.operationAcceleration, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference-published-resume.test.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference-published-resume.test.ts index ee535cb8e16..110802479a3 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference-published-resume.test.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference-published-resume.test.ts @@ -11,6 +11,7 @@ function runtimeFor(harness: ReturnType { const operation = createPodmanHostLocalInferenceOperation({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, redactSensitive: harness.redactSensitive, }); harness.events.length = 0; - expect(() => operation.engine.capture(["run", "--privileged"])).toThrow( "provider-owned lifecycle", ); @@ -353,6 +354,7 @@ describe("Podman host-local inference lifecycle", () => { const operation = createPodmanHostLocalInferenceOperation({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, @@ -796,6 +798,7 @@ describe("Podman host-local inference lifecycle", () => { const runtime = createPodmanHostLocalInferenceRuntime({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, authority, @@ -857,6 +860,7 @@ describe("Podman host-local inference lifecycle", () => { const runtime = createPodmanHostLocalInferenceRuntime({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, authority, @@ -930,6 +934,7 @@ describe("Podman host-local inference lifecycle", () => { const operation = createPodmanHostLocalInferenceOperation({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, @@ -939,7 +944,6 @@ describe("Podman host-local inference lifecycle", () => { harness.seedManaged("stopped", false, operation.bindingSha256); harness.state.startLostAcknowledgement = true; harness.events.length = 0; - const prepared = runtime.recoverManaged!(harness.input, harness.writer); expect(harness.failures.at(-1)).toMatchObject({ phase: "start" }); const startIndex = harness.events.findIndex((event) => event.startsWith("podman:start ")); @@ -1273,6 +1277,7 @@ describe("Podman host-local inference lifecycle", () => { const operation = createPodmanHostLocalInferenceOperation({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: { load: () => null, record: (authority) => ({ ...authority, engineId: "docker" }), @@ -1282,7 +1287,6 @@ describe("Podman host-local inference lifecycle", () => { redactSensitive: harness.redactSensitive, }); harness.events.length = 0; - expect(() => operation.managedRuntime?.startManaged(harness.input, harness.writer)).toThrow( "does not match persisted authority", ); @@ -1458,6 +1462,7 @@ describe("Podman host-local inference lifecycle", () => { const missingRuntime = createPodmanHostLocalInferenceRuntime({ engine: missing.engine, env: {}, + probeCleanupTiming: missing.probeCleanupTiming, authorityStore: missing.authorityStore, routeAuthorityStore: missing.routeAuthorityStore, authority: qualifyPodmanInferenceAuthority(missing.engine), @@ -1468,7 +1473,6 @@ describe("Podman host-local inference lifecycle", () => { "requires environment 'NGC_API_KEY'", ); expect(missing.container()).toBeNull(); - const vllm = createPodmanHostLocalInferenceTestHarness({ service: "vllm" }); const vllmRuntime = operationRuntime(vllm); expect(() => @@ -1479,11 +1483,11 @@ describe("Podman host-local inference lifecycle", () => { it("requires a qualified injected redactor before any provider call", () => { const harness = createPodmanHostLocalInferenceTestHarness(); - expect(() => createPodmanHostLocalInferenceOperation({ engine: harness.engine, env: harness.env, + probeCleanupTiming: harness.probeCleanupTiming, authorityStore: harness.authorityStore, routeAuthorityStore: harness.routeAuthorityStore, onFailureEvidence: harness.onFailureEvidence, diff --git a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts index fef12c6d157..b623e1347a9 100644 --- a/src/lib/onboard/runtime-provider/podman-host-local-inference.ts +++ b/src/lib/onboard/runtime-provider/podman-host-local-inference.ts @@ -105,6 +105,9 @@ const RESERVED_NETWORK_NAMES = new Set([ const AT_REST_STATES = new Set(["configured", "created", "dead", "exited", "stopped"]); const PROBE_TIMEOUT_MS = 30_000; const POST_CREATE_PROBE_INSPECT_MAX_ATTEMPTS = 3; +const PROBE_CLEANUP_SETTLEMENT_TIMEOUT_MS = 30_000; +const PROBE_CLEANUP_SETTLEMENT_INTERVAL_MS = 1_000; +const PROBE_CLEANUP_SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4)); const INFERENCE_PROBE_TIMEOUT_MS = 150_000; const READY_PROBE_TIMEOUT_MS = 240_000; const PROBE_CURL_MAX_TIME_SECONDS = 20; @@ -119,6 +122,11 @@ const SECRET_ENVIRONMENT_BY_SERVICE = Object.freeze({ vllm: new Set(), }); +export interface PodmanProbeCleanupTiming { + readonly now?: () => number; + readonly sleep?: (milliseconds: number) => void; +} + export interface PodmanHostLocalInferenceRuntimeOptions { readonly engine: PodmanContainerEngine; /** Exact operation input environment; values remain memory-only. */ @@ -134,6 +142,7 @@ export interface PodmanHostLocalInferenceRuntimeOptions { readonly serializedReceipt: string; readonly assertForwardAuthority: () => void; }; + readonly probeCleanupTiming?: PodmanProbeCleanupTiming; readonly externalNetwork?: PodmanExternalInferenceNetworkAuthority; /** Immutable accepted acceleration scope for this one operation. */ readonly operationAcceleration?: HostLocalOllamaAccelerationAuthority; @@ -153,6 +162,7 @@ export interface PodmanHostLocalInferenceOperationOptions { readonly authority?: PodmanInferenceAuthorityReceipt; readonly authorityQualification?: PodmanInferenceQualificationOptions; readonly hermesPortablePublishedEngineAuthority?: PodmanHostLocalInferenceRuntimeOptions["hermesPortablePublishedEngineAuthority"]; + readonly probeCleanupTiming?: PodmanProbeCleanupTiming; readonly authorityStore: PersistedEngineAuthorityStore; readonly routeAuthorityStore: HostLocalInferenceRouteAuthorityStore; readonly onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void; @@ -1234,17 +1244,7 @@ function inspectContainer(engine: ContainerEngine, runtimeId: string): ManagedCo }); } -function inspectProbeContainer( - engine: ContainerEngine, - runtimeId: string, - maxAttempts = 1, -): ProbeContainer { - const args = ["container", "inspect", runtimeId] as const; - let result = engine.capture(args, PROBE_TIMEOUT_MS); - for (let attempt = 1; attempt < maxAttempts && commandTimedOut(result); attempt += 1) { - result = engine.capture(args, PROBE_TIMEOUT_MS); - } - const output = requireSuccess("probe container inspection", result); +function parseProbeContainerInspection(output: string): ProbeContainer { let parsed: unknown; try { parsed = JSON.parse(output); @@ -1310,6 +1310,22 @@ function inspectProbeContainer( }); } +function inspectProbeContainer( + engine: ContainerEngine, + runtimeId: string, + maxAttempts = 1, + beforeAttempt: () => void = () => undefined, +): ProbeContainer { + const args = ["container", "inspect", runtimeId] as const; + beforeAttempt(); + let result = engine.capture(args, PROBE_TIMEOUT_MS); + for (let attempt = 1; attempt < maxAttempts && commandTimedOut(result); attempt += 1) { + beforeAttempt(); + result = engine.capture(args, PROBE_TIMEOUT_MS); + } + return parseProbeContainerInspection(requireSuccess("probe container inspection", result)); +} + function exactContainerExists(engine: ContainerEngine, runtimeId: string): boolean { const result = engine.capture(["container", "exists", runtimeId], PROBE_TIMEOUT_MS); if (result.error) { @@ -1322,22 +1338,7 @@ function exactContainerExists(engine: ContainerEngine, runtimeId: string): boole throw new Error(`Podman inference container existence check failed: ${commandEvidence(result)}`); } -function lookupContainerId(engine: ContainerEngine, containerName: string): string | null { - const output = requireSuccess( - "container lookup", - engine.capture( - [ - "ps", - "--all", - "--no-trunc", - "--filter", - `name=^${containerName}$`, - "--format", - "{{.ID}}\t{{.Names}}", - ], - PROBE_TIMEOUT_MS, - ), - ); +function parseContainerLookup(output: string, containerName: string): string | null { const rows = output .split(/\r?\n/u) .map((row) => row.trim()) @@ -1353,6 +1354,28 @@ function lookupContainerId(engine: ContainerEngine, containerName: string): stri return exactContainerId(fields[0]); } +function containerLookupArgs(containerName: string): readonly string[] { + return Object.freeze([ + "ps", + "--all", + "--no-trunc", + "--filter", + `name=^${containerName}$`, + "--format", + "{{.ID}}\t{{.Names}}", + ]); +} + +function lookupContainerId(engine: ContainerEngine, containerName: string): string | null { + return parseContainerLookup( + requireSuccess( + "container lookup", + engine.capture(containerLookupArgs(containerName), PROBE_TIMEOUT_MS), + ), + containerName, + ); +} + function requireManagedIdentity( container: ManagedContainer, expected: { @@ -1903,25 +1926,157 @@ function requireProbeIdentity( return container; } +type ProbeCleanupObservation = + | { readonly kind: "absent" } + | { readonly kind: "present"; readonly container: ProbeContainer } + | { readonly kind: "retry" }; + +function defaultProbeCleanupSleep(milliseconds: number): void { + if (milliseconds > 0) { + Atomics.wait(PROBE_CLEANUP_SLEEP_BUFFER, 0, 0, milliseconds); + } +} + +function monotonicProbeCleanupNow(): number { + return Number(process.hrtime.bigint() / 1_000_000n); +} + +function probeCleanupClock(now: () => number): () => number { + let previous: number | undefined; + return () => { + const current = now(); + if ( + !Number.isFinite(current) || + current < 0 || + (previous !== undefined && current < previous) + ) { + throw new Error("Podman inference probe cleanup clock is invalid."); + } + previous = current; + return current; + }; +} + +function observeProbeCleanup( + engine: ContainerEngine, + runtimeId: string, + spec: ProbeSpec, + assertAuthority: () => void, + mode: "owned" | "retained-legacy", +): ProbeCleanupObservation { + assertAuthority(); + const exists = engine.capture(["container", "exists", runtimeId], PROBE_TIMEOUT_MS); + if (commandTimedOut(exists)) return Object.freeze({ kind: "retry" }); + if (exists.error || (exists.status !== 0 && exists.status !== 1)) { + throw new Error(`Podman inference probe existence check failed: ${commandEvidence(exists)}`); + } + + let current: ProbeContainer | null = null; + if (exists.status === 0) { + assertAuthority(); + const inspection = engine.capture(["container", "inspect", runtimeId], PROBE_TIMEOUT_MS); + if (commandTimedOut(inspection)) return Object.freeze({ kind: "retry" }); + current = requireProbeIdentity( + parseProbeContainerInspection(requireSuccess("probe container inspection", inspection)), + spec, + runtimeId, + ); + if (current.running || !AT_REST_STATES.has(current.status)) { + throw new Error( + mode === "retained-legacy" + ? "retained legacy probe is not in an exact at-rest state" + : "Podman inference probe cleanup requires an exact at-rest identity.", + ); + } + } + + assertAuthority(); + const lookup = engine.capture(containerLookupArgs(spec.name), PROBE_TIMEOUT_MS); + if (commandTimedOut(lookup)) return Object.freeze({ kind: "retry" }); + const nameId = parseContainerLookup(requireSuccess("container lookup", lookup), spec.name); + if (current !== null) { + if (nameId !== null && nameId !== runtimeId) { + throw new Error("Podman inference probe name is owned by another container."); + } + return Object.freeze({ kind: "present", container: current }); + } + if (nameId === null) return Object.freeze({ kind: "absent" }); + if (nameId === runtimeId) return Object.freeze({ kind: "retry" }); + throw new Error("Podman inference probe name was reused by another container."); +} + +function settleProbeBeforeRemoval( + engine: ContainerEngine, + runtimeId: string, + spec: ProbeSpec, + assertAuthority: () => void, + mode: "owned" | "retained-legacy", +): ProbeContainer | null { + for (let attempt = 0; attempt < POST_CREATE_PROBE_INSPECT_MAX_ATTEMPTS; attempt += 1) { + const observation = observeProbeCleanup(engine, runtimeId, spec, assertAuthority, mode); + if (observation.kind === "present") return observation.container; + if (observation.kind === "absent") { + assertAuthority(); + return null; + } + } + throw new Error("Podman inference probe cleanup could not establish its pre-remove state."); +} + +function settleProbeRemoval( + engine: ContainerEngine, + runtimeId: string, + spec: ProbeSpec, + assertAuthority: () => void, + mode: "owned" | "retained-legacy", + timing: PodmanProbeCleanupTiming, +): void { + const now = probeCleanupClock(timing.now ?? monotonicProbeCleanupNow); + const sleep = timing.sleep ?? defaultProbeCleanupSleep; + const startedAt = now(); + const deadline = startedAt + PROBE_CLEANUP_SETTLEMENT_TIMEOUT_MS; + if (!Number.isFinite(deadline)) { + throw new Error("Podman inference probe cleanup deadline is invalid."); + } + for (;;) { + const observation = observeProbeCleanup(engine, runtimeId, spec, assertAuthority, mode); + const observedAt = now(); + if (observedAt > deadline) { + throw new Error("Podman inference probe removal exceeded its settlement deadline."); + } + if (observation.kind === "absent") { + assertAuthority(); + if (now() > deadline) { + throw new Error("Podman inference probe removal exceeded its settlement deadline."); + } + return; + } + const remaining = deadline - observedAt; + if (remaining <= 0) { + throw new Error("Podman inference probe removal did not settle into exact absence."); + } + const delay = Math.min(PROBE_CLEANUP_SETTLEMENT_INTERVAL_MS, remaining); + sleep(delay); + if (now() <= observedAt) { + throw new Error("Podman inference probe cleanup clock did not advance."); + } + } +} + function cleanupExactProbe( engine: ContainerEngine, + assertAuthority: () => void, container: Pick, spec: ProbeSpec, phase: PodmanInferenceFailureEvidence["phase"], onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void, redactor: PodmanInferenceRedactor, + timing: PodmanProbeCleanupTiming, mode: "owned" | "retained-legacy" = "owned", ): void { - let current: ProbeContainer; + let current: ProbeContainer | null; try { - current = requireProbeIdentity( - inspectProbeContainer(engine, container.runtimeId), - spec, - container.runtimeId, - ); - if (mode === "retained-legacy" && (current.running || !AT_REST_STATES.has(current.status))) { - throw new Error("retained legacy probe is not in an exact at-rest state"); - } + current = settleProbeBeforeRemoval(engine, container.runtimeId, spec, assertAuthority, mode); } catch (error) { emitProviderFailure( phase, @@ -1933,6 +2088,8 @@ function cleanupExactProbe( `Podman inference probe cleanup lost exact identity: ${errorEvidence(redactor, error)}`, ); } + if (current === null) return; + assertAuthority(); const removal = engine.capture( mode === "owned" ? ["rm", "--force", current.runtimeId] : ["rm", current.runtimeId], MUTATION_TIMEOUT_MS, @@ -1946,15 +2103,7 @@ function cleanupExactProbe( ); } try { - const stillExists = exactContainerExists(engine, current.runtimeId); - const nameAfter = lookupContainerId(engine, current.name); - if (stillExists || nameAfter !== null) { - throw new Error( - stillExists - ? `exact probe '${current.runtimeId}' remains present` - : `probe name '${current.name}' was reused by '${String(nameAfter)}'`, - ); - } + settleProbeRemoval(engine, current.runtimeId, spec, assertAuthority, mode, timing); } catch (error) { emitProviderFailure( phase, @@ -1977,6 +2126,7 @@ function executeExactProbe( validateOutput: (output: string) => void, onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void, redactor: PodmanInferenceRedactor, + timing: PodmanProbeCleanupTiming, ): string { const spec = specs.current; const phase = spec.phase; @@ -1996,11 +2146,13 @@ function executeExactProbe( if (legacyId !== null) { cleanupExactProbe( engine, + assertAuthority, { runtimeId: legacyId }, legacy, phase, onFailureEvidence, redactor, + timing, "retained-legacy", ); assertAuthority(); @@ -2077,19 +2229,12 @@ function executeExactProbe( "Podman inference probe identity is indeterminate after create.", ); } + let failure: Error | null = null; if (acknowledgementFailure !== null) { captureFailure(acknowledgementFailure); - cleanupExactProbe(engine, container, spec, phase, onFailureEvidence, redactor); - try { - assertAuthority(); - } catch (error) { - captureFailure(error); - throw new PodmanInferenceCapturedFailureError(errorEvidence(redactor, error)); - } - throw new PodmanInferenceCapturedFailureError(errorEvidence(redactor, acknowledgementFailure)); + failure = acknowledgementFailure; } - let failure: Error | null = null; const wait = engine.capture(["wait", container.runtimeId], timeoutMs); if (wait.status !== 0 || wait.error) { failure = new Error( @@ -2182,7 +2327,16 @@ function executeExactProbe( captureFailure(failure); } } - cleanupExactProbe(engine, container, spec, phase, onFailureEvidence, redactor); + cleanupExactProbe( + engine, + assertAuthority, + container, + spec, + phase, + onFailureEvidence, + redactor, + timing, + ); try { assertAuthority(); } catch (error) { @@ -2214,6 +2368,7 @@ function probeOllamaReady( parent: ProbeParentAuthority, onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void, redactor: PodmanInferenceRedactor, + timing: PodmanProbeCleanupTiming, ): void { const spec = createProbeSpec( "ollama", @@ -2240,6 +2395,7 @@ function probeOllamaReady( }, onFailureEvidence, redactor, + timing, ); } @@ -2255,6 +2411,7 @@ function probeOllamaAcceleration( parent: ProbeParentAuthority, onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void, redactor: PodmanInferenceRedactor, + timing: PodmanProbeCleanupTiming, ): OllamaModelPlacementAuthority { const spec = createProbeSpec( "ollama", @@ -2328,6 +2485,7 @@ function probeOllamaAcceleration( }, onFailureEvidence, redactor, + timing, ); if (observed === null) { throw new Error("Ollama acceleration probe did not return placement authority."); @@ -2343,6 +2501,7 @@ function probeManagedReady( parent: ProbeParentAuthority, onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void, redactor: PodmanInferenceRedactor, + timing: PodmanProbeCleanupTiming, ): void { const healthPath = spec.service === "ollama" @@ -2377,6 +2536,7 @@ function probeManagedReady( () => undefined, onFailureEvidence, redactor, + timing, ); } @@ -2439,6 +2599,7 @@ function probeOpenAiInference( parent: ProbeParentAuthority, onFailureEvidence: (evidence: PodmanInferenceFailureEvidence) => void, redactor: PodmanInferenceRedactor, + timing: PodmanProbeCleanupTiming, ): void { const completionRequest = (maxTokens: number, deterministic: boolean) => ({ model, @@ -2554,6 +2715,7 @@ function probeOpenAiInference( }, onFailureEvidence, redactor, + timing, ); } @@ -2964,6 +3126,7 @@ export function createPodmanHostLocalInferenceRuntime( ) { throw new Error("Podman published inference has invalid creation engine authority."); } + const probeCleanupTiming = options.probeCleanupTiming ?? Object.freeze({}); const inspectNetwork = ( expected: Parameters[2], ): PodmanInferenceNetworkAuthority => @@ -3179,6 +3342,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(normalized), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); assertAuthority(); probeOpenAiInference( @@ -3193,6 +3357,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(normalized), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); probeOllamaAcceleration( engine, @@ -3206,6 +3371,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(normalized), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); assertReceiptAuthority(); return normalized; @@ -3233,6 +3399,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(inspected.receipt), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); if (!("devices" in inspected.receipt.runtime.gpu)) { throw new Error("Podman managed inference receipt lacks exact CDI device authority."); @@ -3255,6 +3422,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(inspected.receipt), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); if (service === "ollama") { probeOllamaAcceleration( @@ -3269,6 +3437,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(inspected.receipt), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); } assertReceiptAuthority(); @@ -3372,6 +3541,7 @@ export function createPodmanHostLocalInferenceRuntime( managedSpecProbeParent(spec), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); if (spec.service === "ollama") { pullManagedOllamaModel(engine, assertSpecAuthority, container.runtimeId, spec.model); @@ -3391,6 +3561,7 @@ export function createPodmanHostLocalInferenceRuntime( managedSpecProbeParent(spec), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); const placement = spec.service === "ollama" @@ -3406,6 +3577,7 @@ export function createPodmanHostLocalInferenceRuntime( managedSpecProbeParent(spec), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ) : null; assertSpecAuthority(); @@ -3501,6 +3673,7 @@ export function createPodmanHostLocalInferenceRuntime( managedSpecProbeParent(spec), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); if (spec.service === "ollama") { pullManagedOllamaModel(engine, assertSpecAuthority, created.runtimeId, spec.model); @@ -3520,6 +3693,7 @@ export function createPodmanHostLocalInferenceRuntime( managedSpecProbeParent(spec), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); const placement = spec.service === "ollama" @@ -3535,6 +3709,7 @@ export function createPodmanHostLocalInferenceRuntime( managedSpecProbeParent(spec), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ) : null; assertSpecAuthority(); @@ -3743,6 +3918,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(receipt), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); phase = "gpu"; proveManagedGpu( @@ -3764,6 +3940,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(receipt), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); if (receipt.service === "ollama") { probeOllamaAcceleration( @@ -3778,6 +3955,7 @@ export function createPodmanHostLocalInferenceRuntime( receiptProbeParent(receipt), onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); } assertReceiptAuthority(); @@ -3894,6 +4072,7 @@ export function createPodmanHostLocalInferenceRuntime( qualificationParent, onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); phase = "inference"; probeOpenAiInference( @@ -3908,6 +4087,7 @@ export function createPodmanHostLocalInferenceRuntime( qualificationParent, onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); phase = "gpu"; placement = probeOllamaAcceleration( @@ -3922,6 +4102,7 @@ export function createPodmanHostLocalInferenceRuntime( qualificationParent, onFailureEvidence, sensitiveRedactor, + probeCleanupTiming, ); assertOllamaAuthority(); } catch (error) { diff --git a/test/helpers/podman-host-local-inference-test-harness.ts b/test/helpers/podman-host-local-inference-test-harness.ts index d831455a68d..8cb6ec33564 100644 --- a/test/helpers/podman-host-local-inference-test-harness.ts +++ b/test/helpers/podman-host-local-inference-test-harness.ts @@ -32,6 +32,7 @@ import { PODMAN_INFERENCE_SPEC_LABEL, PODMAN_INFERENCE_TRANSACTION_LABEL, type PodmanInferenceFailureEvidence, + type PodmanProbeCleanupTiming, } from "../../src/lib/onboard/runtime-provider/podman-host-local-inference"; import { qualifyPodmanInferenceAuthority } from "../../src/lib/onboard/runtime-provider/podman-preflight"; import { redact, redactFull, redactSensitiveText } from "../../src/lib/security/redact"; @@ -84,6 +85,7 @@ export interface PodmanHostLocalInferenceHarnessOptions { readonly authorityId?: string; readonly service?: "ollama" | "nim" | "vllm"; readonly probeImageRef?: string; + readonly probeCleanupTiming?: PodmanProbeCleanupTiming; } export interface PodmanHostLocalInferenceHarness { @@ -95,6 +97,7 @@ export interface PodmanHostLocalInferenceHarness { readonly failureProbeIds: Array; readonly input: HostLocalManagedInferenceInput; readonly operationAcceleration: HostLocalOllamaAccelerationAuthority; + readonly probeCleanupTiming: PodmanProbeCleanupTiming; readonly routeAuthorityStore: HostLocalInferenceRouteAuthorityStore; readonly writer: HostLocalInferenceReceiptWriter; readonly written: string[]; @@ -131,8 +134,25 @@ export interface PodmanHostLocalInferenceHarness { legacyInferenceProbeRunning: boolean; probeWaitFailure: boolean; probeRemoveLostAcknowledgement: boolean; + probeRemoveTimeout: boolean; probeRemoveLeavesContainer: boolean; probeReuseNameAfterRemoval: boolean; + probeCleanupExistenceTimeoutsRemaining: number; + probeCleanupExistenceFailure: boolean; + probeCleanupInspectTimeoutsRemaining: number; + probeCleanupInspectFailure: boolean; + probeCleanupMalformedInspection: boolean; + probeCleanupAmbiguousLookup: boolean; + probeDisappearBeforeCleanupCount: number; + probeRemovalIdObservationsRemaining: number; + probeRemovalNameObservationsRemaining: number; + probeCleanupLabelDriftAfterRemoval: boolean; + probeCleanupSpecDriftAfterRemoval: boolean; + probeNetworkDriftBeforeRemoval: boolean; + probeNetworkDriftAfterRemoval: boolean; + probeEngineDriftBeforeRemoval: boolean; + probeEngineDriftAfterRemoval: boolean; + engineCurrent: boolean; probeInheritedImageLabel: boolean; parentInheritedImageLabel: boolean; parentExtraControlledLabel: boolean; @@ -399,6 +419,16 @@ export function createPodmanHostLocalInferenceTestHarness( const failures: PodmanInferenceFailureEvidence[] = []; const failureProbeIds: Array = []; const written: string[] = []; + let probeCleanupNow = 0; + const probeCleanupTiming = + options.probeCleanupTiming ?? + Object.freeze({ + now: () => probeCleanupNow, + sleep: (milliseconds: number) => { + events.push(`probe-cleanup:sleep ${String(milliseconds)}`); + probeCleanupNow += milliseconds; + }, + }); const state = { cdiDevices: [...(options.cdiDevices ?? [`nvidia.com/gpu=${GPU_UUID}`])], omitDiscoveredDevices: options.omitDiscoveredDevices ?? false, @@ -441,8 +471,25 @@ export function createPodmanHostLocalInferenceTestHarness( legacyInferenceProbeRunning: false, probeWaitFailure: false, probeRemoveLostAcknowledgement: false, + probeRemoveTimeout: false, probeRemoveLeavesContainer: false, probeReuseNameAfterRemoval: false, + probeCleanupExistenceTimeoutsRemaining: 0, + probeCleanupExistenceFailure: false, + probeCleanupInspectTimeoutsRemaining: 0, + probeCleanupInspectFailure: false, + probeCleanupMalformedInspection: false, + probeCleanupAmbiguousLookup: false, + probeDisappearBeforeCleanupCount: 0, + probeRemovalIdObservationsRemaining: 0, + probeRemovalNameObservationsRemaining: 0, + probeCleanupLabelDriftAfterRemoval: false, + probeCleanupSpecDriftAfterRemoval: false, + probeNetworkDriftBeforeRemoval: false, + probeNetworkDriftAfterRemoval: false, + probeEngineDriftBeforeRemoval: false, + probeEngineDriftAfterRemoval: false, + engineCurrent: true, probeInheritedImageLabel: false, parentInheritedImageLabel: false, parentExtraControlledLabel: false, @@ -455,6 +502,8 @@ export function createPodmanHostLocalInferenceTestHarness( let currentContainer: TestContainer | null = null; let currentProbe: TestProbeContainer | null = null; let probeInspectCount = 0; + let probeCleanupStarted = false; + let probeRemovalIssued = false; let networkEngineAuthoritySha256 = ""; let persistedAuthority: PersistedEngineAuthority | null = null; let routeAuthority: HostLocalInferenceRouteAuthority | null = null; @@ -502,6 +551,199 @@ export function createPodmanHostLocalInferenceTestHarness( }, }; + const captureContainerLookup = (args: readonly string[]): ContainerEngineCommandResult => { + const expectedName = String(args.find((arg) => arg.startsWith("name=^")) ?? "").slice( + "name=^".length, + -1, + ); + let candidate = [currentContainer, currentProbe].find( + (container) => container?.name === expectedName, + ); + if (!candidate && state.retainLegacyInferenceProbe) { + const retained = retainedLegacyInferenceProbeForName(expectedName); + if (retained !== null) { + currentProbe = retained; + state.retainLegacyInferenceProbe = false; + candidate = retained; + } + } + if ( + state.probePostCreateNameLookupTimeout && + candidate === currentProbe && + currentProbe !== null + ) { + state.probePostCreateNameLookupTimeout = false; + const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); + } + if (!candidate) return result(); + if (probeCleanupStarted && state.probeCleanupAmbiguousLookup) { + return result( + 0, + `${candidate.id}\t${candidate.name}\n${REUSED_PROBE_CONTAINER_ID}\t${candidate.name}\n`, + ); + } + if (probeCleanupStarted && probeRemovalIssued) { + if (state.probeRemovalNameObservationsRemaining > 0) { + state.probeRemovalNameObservationsRemaining -= 1; + return result(0, `${candidate.id}\t${candidate.name}\n`); + } + if (state.probeRemovalIdObservationsRemaining === 0) { + currentProbe = null; + probeCleanupStarted = false; + probeRemovalIssued = false; + } + return result(); + } + const output = result(0, `${candidate.id}\t${candidate.name}\n`); + if (probeCleanupStarted && state.probeNetworkDriftBeforeRemoval) { + state.networkName = "drifted-network"; + state.probeNetworkDriftBeforeRemoval = false; + } + if (probeCleanupStarted && state.probeEngineDriftBeforeRemoval) { + state.engineCurrent = false; + state.probeEngineDriftBeforeRemoval = false; + } + return output; + }; + + const captureContainerInspect = (args: readonly string[]): ContainerEngineCommandResult => { + if (currentContainer?.id === args[2]) return result(0, inspectPayload(currentContainer)); + if (currentProbe?.id !== args[2]) return result(125, "", "no such container"); + probeInspectCount += 1; + if (probeCleanupStarted && state.probeCleanupInspectFailure) { + const error = Object.assign(new Error("permission denied"), { code: "EACCES" }); + return result(1, "", "permission denied", error); + } + if (probeCleanupStarted && state.probeCleanupInspectTimeoutsRemaining > 0) { + state.probeCleanupInspectTimeoutsRemaining -= 1; + const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); + } + if (probeCleanupStarted && state.probeCleanupMalformedInspection) return result(0, "{"); + if (state.probePostCreateInspectFailuresRemaining > 0) { + state.probePostCreateInspectFailuresRemaining -= 1; + const error = Object.assign(new Error("permission denied"), { code: "EACCES" }); + return result(1, "", "permission denied", error); + } + if (state.probePostCreateInspectTimeoutsRemaining > 0) { + state.probePostCreateInspectTimeoutsRemaining -= 1; + const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); + } + const inspectedProbe = + state.probeInspectRuntimeIdMismatchAt === probeInspectCount + ? { ...currentProbe, id: REUSED_PROBE_CONTAINER_ID } + : currentProbe; + return result(0, probeInspectPayload(inspectedProbe)); + }; + + const captureContainerExists = (args: readonly string[]): ContainerEngineCommandResult => { + if (currentProbe?.id === args[2]) { + probeCleanupStarted = true; + if (state.probeCleanupExistenceFailure) { + const error = Object.assign(new Error("permission denied"), { code: "EACCES" }); + return result(1, "", "permission denied", error); + } + if (state.probeCleanupExistenceTimeoutsRemaining > 0) { + state.probeCleanupExistenceTimeoutsRemaining -= 1; + const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); + } + if (probeRemovalIssued) { + if (state.probeRemovalIdObservationsRemaining > 0) { + state.probeRemovalIdObservationsRemaining -= 1; + return result(0); + } + return result(1); + } + } + return result(currentContainer?.id === args[2] || currentProbe?.id === args[2] ? 0 : 1); + }; + + const captureRemove = (args: readonly string[]): ContainerEngineCommandResult => { + const probe = currentProbe; + if (probe !== null && probe.id === args.at(-1)) { + const removedProbe = probe; + if ( + !state.probeRemoveLeavesContainer && + (state.probeRemovalIdObservationsRemaining > 0 || + state.probeRemovalNameObservationsRemaining > 0) + ) { + probeRemovalIssued = true; + } else if (!state.probeRemoveLeavesContainer) { + currentProbe = state.probeReuseNameAfterRemoval + ? { + ...removedProbe, + id: REUSED_PROBE_CONTAINER_ID, + labels: {}, + running: true, + status: "running", + exitCode: 0, + } + : null; + } + if (state.probeCleanupLabelDriftAfterRemoval && currentProbe !== null) { + currentProbe.labels[PODMAN_INFERENCE_PROBE_SPEC_LABEL] = "f".repeat(64); + state.probeCleanupLabelDriftAfterRemoval = false; + } + if (state.probeCleanupSpecDriftAfterRemoval && currentProbe !== null) { + currentProbe = { + ...currentProbe, + createArguments: [...currentProbe.createArguments, "unexpected-argument"], + }; + state.probeCleanupSpecDriftAfterRemoval = false; + } + if (state.probeNetworkDriftAfterRemoval) { + state.networkName = "drifted-network"; + state.probeNetworkDriftAfterRemoval = false; + } + if (state.probeEngineDriftAfterRemoval) { + state.engineCurrent = false; + state.probeEngineDriftAfterRemoval = false; + } + if (state.probeRemoveTimeout) { + const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); + } + return state.probeRemoveLostAcknowledgement + ? result(125, "", "transport closed after probe remove") + : result(0, `${PROBE_CONTAINER_ID}\n`); + } + if (!currentContainer || currentContainer.id !== args.at(-1)) { + return result(125, "", "missing"); + } + const removedName = currentContainer.name; + const removedImage = currentContainer.imageRef; + if (!state.removeLeavesContainer) { + currentContainer = state.reuseNameAfterRemoval + ? { + id: REUSED_CONTAINER_ID, + name: removedName, + imageRef: removedImage, + labels: {}, + createArguments: currentContainer.createArguments, + running: true, + status: "running", + exitCode: 0, + } + : null; + } + return state.removeLostAcknowledgement + ? result(125, "", "transport closed after remove") + : result(0, `${CONTAINER_ID}\n`); + }; + const engine: PodmanContainerEngine = { operation: "host-local-inference", engineId: "podman", @@ -509,6 +751,7 @@ export function createPodmanHostLocalInferenceTestHarness( authorityId: options.authorityId ?? "test:podman-inference", endpointAuthorityId: options.authorityId ?? "test:podman-inference", capture: (args, timeoutMs) => { + if (!state.engineCurrent) throw new Error("test engine authority changed"); const probeAction = args[0] === "logs" || args[0] === "rm" || args[0] === "wait" ? args[0] : null; const probeActionId = probeAction === "rm" ? args.at(-1) : args[1]; @@ -564,59 +807,9 @@ export function createPodmanHostLocalInferenceTestHarness( ]), ); } - if (args[0] === "ps") { - const expectedName = String(args.find((arg) => arg.startsWith("name=^")) ?? "").slice( - "name=^".length, - -1, - ); - let candidate = [currentContainer, currentProbe].find( - (container) => container?.name === expectedName, - ); - if (!candidate && state.retainLegacyInferenceProbe) { - const retained = retainedLegacyInferenceProbeForName(expectedName); - if (retained !== null) { - currentProbe = retained; - state.retainLegacyInferenceProbe = false; - candidate = retained; - } - } - if ( - state.probePostCreateNameLookupTimeout && - candidate === currentProbe && - currentProbe !== null - ) { - return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", new Error("ETIMEDOUT")); - } - if (!candidate) return result(); - return result(0, `${candidate.id}\t${candidate.name}\n`); - } - if (args[0] === "container" && args[1] === "inspect") { - if (currentContainer?.id === args[2]) return result(0, inspectPayload(currentContainer)); - if (currentProbe?.id === args[2]) { - probeInspectCount += 1; - if (state.probePostCreateInspectFailuresRemaining > 0) { - state.probePostCreateInspectFailuresRemaining -= 1; - const error = Object.assign(new Error("permission denied"), { code: "EACCES" }); - return result(1, "", "permission denied", error); - } - if (state.probePostCreateInspectTimeoutsRemaining > 0) { - state.probePostCreateInspectTimeoutsRemaining -= 1; - const error = Object.assign(new Error("spawnSync /usr/local/bin/podman ETIMEDOUT"), { - code: "ETIMEDOUT", - }); - return result(1, "", "spawnSync /usr/local/bin/podman ETIMEDOUT", error); - } - const inspectedProbe = - state.probeInspectRuntimeIdMismatchAt === probeInspectCount - ? { ...currentProbe, id: REUSED_PROBE_CONTAINER_ID } - : currentProbe; - return result(0, probeInspectPayload(inspectedProbe)); - } - return result(125, "", "no such container"); - } - if (args[0] === "container" && args[1] === "exists") { - return result(currentContainer?.id === args[2] || currentProbe?.id === args[2] ? 0 : 1); - } + if (args[0] === "ps") return captureContainerLookup(args); + if (args[0] === "container" && args[1] === "inspect") return captureContainerInspect(args); + if (args[0] === "container" && args[1] === "exists") return captureContainerExists(args); if (args[0] === "run") { const name = valueAfter(args, "--name"); const labels = labelsFrom(args); @@ -637,6 +830,8 @@ export function createPodmanHostLocalInferenceTestHarness( logsStdout: "", logsStderr: "", }; + probeCleanupStarted = false; + probeRemovalIssued = false; return state.probeRunLostAcknowledgement ? result(125, "", "transport closed after probe create") : result(0, state.probeRunAcknowledgementText ?? `${PROBE_CONTAINER_ID}\n`); @@ -673,7 +868,12 @@ export function createPodmanHostLocalInferenceTestHarness( } if (args[0] === "logs") { if (!currentProbe || currentProbe.id !== args[1]) return result(125, "", "missing probe"); - return result(0, currentProbe.logsStdout, currentProbe.logsStderr); + const output = result(0, currentProbe.logsStdout, currentProbe.logsStderr); + if (state.probeDisappearBeforeCleanupCount > 0) { + state.probeDisappearBeforeCleanupCount -= 1; + currentProbe = null; + } + return output; } if (args[0] === "exec") { if (args[2] === "ollama" && args[3] === "pull" && state.ollamaPullFailure !== null) { @@ -716,48 +916,7 @@ export function createPodmanHostLocalInferenceTestHarness( ? result(125, "", "transport closed after stop") : result(0, `${currentContainer.id}\n`); } - if (args[0] === "rm") { - const probe = currentProbe; - if (probe !== null && probe.id === args.at(-1)) { - const removedProbe = probe; - if (!state.probeRemoveLeavesContainer) { - currentProbe = state.probeReuseNameAfterRemoval - ? { - ...removedProbe, - id: REUSED_PROBE_CONTAINER_ID, - labels: {}, - running: true, - status: "running", - exitCode: 0, - } - : null; - } - return state.probeRemoveLostAcknowledgement - ? result(125, "", "transport closed after probe remove") - : result(0, `${PROBE_CONTAINER_ID}\n`); - } - if (!currentContainer || currentContainer.id !== args.at(-1)) - return result(125, "", "missing"); - const removedName = currentContainer.name; - const removedImage = currentContainer.imageRef; - if (!state.removeLeavesContainer) { - currentContainer = state.reuseNameAfterRemoval - ? { - id: REUSED_CONTAINER_ID, - name: removedName, - imageRef: removedImage, - labels: {}, - createArguments: currentContainer.createArguments, - running: true, - status: "running", - exitCode: 0, - } - : null; - } - return state.removeLostAcknowledgement - ? result(125, "", "transport closed after remove") - : result(0, `${CONTAINER_ID}\n`); - } + if (args[0] === "rm") return captureRemove(args); return result(125, "", `unexpected test command: ${args.join(" ")}`); }, captureHost: () => result(125, "", "host capture is forbidden"), @@ -902,6 +1061,7 @@ export function createPodmanHostLocalInferenceTestHarness( failureProbeIds, input, operationAcceleration, + probeCleanupTiming, routeAuthorityStore, writer, written,