Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
2672339
fix(portable): recover published Hermes inference
senthilr-nv Aug 27, 2026
aa62e9b
test(portable): harden inference receipt snapshot
senthilr-nv Aug 27, 2026
7cb436b
fix(inference): protect published resume receipts
senthilr-nv Aug 27, 2026
93e8c5b
fix(inference): recover portable registry after session restart
senthilr-nv Aug 27, 2026
045445d
fix(inference): preserve portable recovery failures
senthilr-nv Aug 27, 2026
46b42f3
fix(inference): reconcile timed-out registry starts
senthilr-nv Aug 28, 2026
6054e8f
fix(onboard): reconcile portable registry start
senthilr-nv Aug 28, 2026
ab4d34d
fix(onboard): settle portable registry start
senthilr-nv Aug 28, 2026
122d8e1
fix(onboard): classify portable inference recovery failures
senthilr-nv Aug 28, 2026
67f0982
test(onboard): pin portable registry authority fixture
senthilr-nv Aug 28, 2026
a0799e9
Merge branch 'main' into codex/10423-hermes-inference-session-recovery
cv Aug 28, 2026
150a3f9
Merge remote-tracking branch 'upstream/main' into codex/10423-hermes-…
senthilr-nv Aug 28, 2026
b79c6f0
Merge branch 'main' into codex/10423-hermes-inference-session-recovery
prekshivyas Aug 28, 2026
cb4e799
fix(hermes): requalify published inference runtime
senthilr-nv Aug 28, 2026
9f9513f
Merge remote-tracking branch 'upstream/codex/10423-hermes-inference-s…
senthilr-nv Aug 28, 2026
c68ccda
ci: retrigger PR review advisor
cv Aug 28, 2026
13b057a
fix(hermes): recover portable forwards after session restart
senthilr-nv Aug 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/lib/actions/sandbox/connect-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1015,6 +1015,7 @@ describe("connectSandbox flow", () => {
Array.isArray(args) ? args[0] === "inference" && args[1] === "set" : false,
),
).toBe(false);
expect(harness.recoverHermesPortableOllamaInferenceSpy).toHaveBeenCalledOnce();
expect(harness.recoverPortableDemoLifecycleSpy).toHaveBeenCalled();
});

Expand Down Expand Up @@ -1159,6 +1160,7 @@ describe("connectSandbox flow", () => {
expect(harness.getSandboxDockerRuntimeSpy).not.toHaveBeenCalled();
expect(harness.dockerStartSpy).not.toHaveBeenCalled();
expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled();
expect(harness.recoverHermesPortableOllamaInferenceSpy).toHaveBeenCalledOnce();
});

it("rejects accepted probe evidence when schema-5 authority disappears (#9203)", async () => {
Expand Down Expand Up @@ -1228,6 +1230,7 @@ describe("connectSandbox flow", () => {
expect(harness.preflightVllmSpy).not.toHaveBeenCalled();
expect(harness.readSandboxConfigSpy).not.toHaveBeenCalled();
expect(harness.writeSandboxConfigSpy).not.toHaveBeenCalled();
expect(harness.recoverHermesPortableOllamaInferenceSpy).not.toHaveBeenCalled();
expect(sandboxVersion.checkAgentVersion).not.toHaveBeenCalled();
expect(brokerSpy).not.toHaveBeenCalled();
const connectCall = harness.runSandboxExecChildSpy.mock.calls.find(
Expand Down
104 changes: 90 additions & 14 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import { retryUntil } from "../../core/retry";

import { spawnExitCode } from "../../core/process-exit";
import { shellQuote } from "../../core/shell-quote";
import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action";
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import {
formatInferenceRouteDriftForDisplay,
Expand Down Expand Up @@ -82,6 +81,7 @@ import {
assertHermesPortableLifecycleForConnect,
buildHermesPortableCommandAuthority,
type HermesPortableActiveLifecycleAuthority,
getNamedGatewayLifecycleState,
printGatewayLifecycleHint,
qualifyPortableAgentLifecycleAuthority,
requalifyPortableAgentSandboxAuthority,
Expand Down Expand Up @@ -112,6 +112,7 @@ import {
resolveSandboxDashboardPort,
waitForManagedGatewaySupervisor,
} from "./process-recovery";
import { recoverHermesPortableInferenceForConnectProbe } from "./probe/hermes-portable-inference-recovery";
import { runTerminalAgentConnectProbe } from "./terminal-connect-probe";
import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch";

Expand Down Expand Up @@ -304,9 +305,11 @@ async function runSandboxConnectProbe(
sandboxName: string,
{
hermesPortable = false,
probeOnly,
probeTiming,
}: { hermesPortable?: boolean; probeTiming?: ProbeTimingRecorder } = {},
}: { hermesPortable?: boolean; probeOnly: true; probeTiming?: ProbeTimingRecorder },
): Promise<void> {
if (probeOnly !== true) throw new Error("connect recovery requires probe-only authority");
const measure = <T>(stage: "inference" | "pairing", operation: () => T): T =>
probeTiming ? probeTiming.measure(stage, operation) : operation();
const measureAsync = <T>(
Expand All @@ -316,7 +319,10 @@ async function runSandboxConnectProbe(
const agent = agentRuntime.getSessionAgent(sandboxName);
const agentName = agentRuntime.getAgentDisplayName(agent);
if (hermesPortable) {
measure("inference", () => verifyHermesPortableInferenceRouteOrExit(sandboxName, agent));
if (probeOnly !== true) throw new Error("Hermes inference recovery requires probe-only mode");
measure("inference", () =>
recoverHermesPortableInferenceRouteForProbeOnlyOrExit(sandboxName, agent),
);
console.log(
` Probe complete: ${agentName} passed receipt-owned authenticated health in '${sandboxName}'.`,
);
Expand Down Expand Up @@ -463,8 +469,18 @@ function portableAgentLifecycleAuthorityDeps() {
return { readRegistry: registry.getSandbox };
}

/** Verify the recorded Hermes route without invoking any inference repair. */
function verifyHermesPortableInferenceRouteOrExit(
class HermesPortableInferenceRouteVerificationError extends Error {
constructor(readonly reason: string) {
super("Hermes Portable inference route verification failed");
}
}

function refuseHermesPortableInferenceRoute(reason: string): never {
throw new HermesPortableInferenceRouteVerificationError(reason);
}

/** Verify the recorded Hermes route without invoking repair or process exit. */
function verifyHermesPortableInferenceRoute(
sandboxName: string,
agent: InferenceRouteProbeAgent,
expectedAuthority?: HermesPortableActiveLifecycleAuthority,
Expand All @@ -477,12 +493,12 @@ function verifyHermesPortableInferenceRouteOrExit(
portableAgentLifecycleAuthorityDeps(),
);
} catch {
failHermesPortableInferenceRoute(sandboxName, "missing or incomplete");
refuseHermesPortableInferenceRoute("missing or incomplete");
}
const sandbox = authority.entry;
const inference = registry.getSandboxEntryInference(sandbox);
if (inference.kind !== "configured") {
failHermesPortableInferenceRoute(sandboxName, "not configured");
refuseHermesPortableInferenceRoute("not configured");
}
assertNoOpenShellGatewayEndpointOverride();
const routeAuthority = {
Expand All @@ -497,11 +513,11 @@ function verifyHermesPortableInferenceRouteOrExit(
{ timeout: OPENSHELL_PROBE_TIMEOUT_MS },
);
if (liveResult.status !== 0 || liveResult.error) {
failHermesPortableInferenceRoute(sandboxName, "unreachable");
refuseHermesPortableInferenceRoute("unreachable");
}
const live = parseGatewayInference(liveResult.output);
if (planInferenceRouteReconcile(live, inference).kind !== "aligned") {
failHermesPortableInferenceRoute(sandboxName, "different from its recorded provider or model");
refuseHermesPortableInferenceRoute("different from its recorded provider or model");
}
const probe = parseSandboxInferenceRouteProbeResult(
captureHermesPortableOpenShell(
Expand All @@ -518,7 +534,7 @@ function verifyHermesPortableInferenceRouteOrExit(
(inference.provider === "ollama-local" &&
(probe.httpStatus === undefined || probe.httpStatus < 200 || probe.httpStatus >= 300))
) {
failHermesPortableInferenceRoute(sandboxName, "unreachable");
refuseHermesPortableInferenceRoute("unreachable");
}
let finalAuthority: ReturnType<typeof requireHermesPortableActiveLifecycleAuthority>;
try {
Expand All @@ -528,19 +544,72 @@ function verifyHermesPortableInferenceRouteOrExit(
portableAgentLifecycleAuthorityDeps(),
);
} catch {
failHermesPortableInferenceRoute(sandboxName, "changed during verification");
refuseHermesPortableInferenceRoute("changed during verification");
}
if (
finalAuthority.entry.gatewayName !== routeAuthority.gatewayName ||
finalAuthority.entry.lifecycleGeneration !== routeAuthority.lifecycleGeneration ||
finalAuthority.entry.provider !== routeAuthority.provider ||
finalAuthority.entry.model !== routeAuthority.model
) {
failHermesPortableInferenceRoute(sandboxName, "changed during verification");
refuseHermesPortableInferenceRoute("changed during verification");
}
return finalAuthority.entry;
}

/** Verify the recorded Hermes route without invoking any inference repair. */
function verifyHermesPortableInferenceRouteOrExit(
sandboxName: string,
agent: InferenceRouteProbeAgent,
expectedAuthority?: HermesPortableActiveLifecycleAuthority,
): SandboxEntry {
try {
return verifyHermesPortableInferenceRoute(sandboxName, agent, expectedAuthority);
} catch (error) {
failHermesPortableInferenceRoute(
sandboxName,
error instanceof HermesPortableInferenceRouteVerificationError ? error.reason : "unreachable",
);
}
}

/** Resume published Ollama authority only for the explicit probe-only command. */
function recoverHermesPortableInferenceRouteForProbeOnlyOrExit(
sandboxName: string,
agent: InferenceRouteProbeAgent,
expectedAuthority?: HermesPortableActiveLifecycleAuthority,
): SandboxEntry {
let authority: HermesPortableActiveLifecycleAuthority;
try {
authority = requireHermesPortableActiveLifecycleAuthority(
sandboxName,
expectedAuthority,
portableAgentLifecycleAuthorityDeps(),
);
} catch {
failHermesPortableInferenceRoute(sandboxName, "missing or incomplete");
}
let verified: SandboxEntry | null = null;
try {
recoverHermesPortableInferenceForConnectProbe({
sandboxName,
authority,
readRegistry: registry.getSandbox,
verifyRoute: () => {
verified = verifyHermesPortableInferenceRoute(sandboxName, agent, authority);
return verified;
},
});
} catch (error) {
failHermesPortableInferenceRoute(
sandboxName,
error instanceof HermesPortableInferenceRouteVerificationError ? error.reason : "unreachable",
);
}
if (!verified) failHermesPortableInferenceRoute(sandboxName, "unreachable");
return verified;
}

const GATEWAY_UNAVAILABLE_RE =
/No gateway configured|No active gateway|Connection refused|client error \(Connect\)|tcp connect error|Status:\s*Disconnected/i;

Expand Down Expand Up @@ -1770,8 +1839,11 @@ async function prepareConnectSandboxWithinLifecycleFence(
portableAgentLifecycleAuthorityDeps(),
),
);
if (probeOnly !== true) {
throw new Error("Hermes inference recovery requires probe-only mode");
}
const verified = probeTiming!.measure("inference", () =>
verifyHermesPortableInferenceRouteOrExit(
recoverHermesPortableInferenceRouteForProbeOnlyOrExit(
sandboxName,
acceptedReadiness.agent,
activeAuthority,
Expand Down Expand Up @@ -1839,7 +1911,11 @@ async function prepareConnectSandboxWithinLifecycleFence(
);
}
requalify();
await runSandboxConnectProbe(sandboxName, { hermesPortable, probeTiming });
await runSandboxConnectProbe(sandboxName, {
hermesPortable,
probeOnly: true,
probeTiming,
});
requalify();
},
});
Expand Down
16 changes: 15 additions & 1 deletion src/lib/actions/sandbox/gateway-state-observe-mode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import * as gatewayRuntime from "../../gateway-runtime-action";
import * as registry from "../../state/registry";
import * as gatewaySelect from "./gateway-select";
import { getReconciledSandboxGatewayState } from "./gateway-state";
import {
captureHermesPortableInferenceRecoveryGateway,
getReconciledSandboxGatewayState,
} from "./gateway-state";

describe("getReconciledSandboxGatewayState observe mode", () => {
beforeEach(() => {
Expand Down Expand Up @@ -112,3 +115,14 @@ describe("getReconciledSandboxGatewayState observe mode", () => {
expect(result).toMatchObject({ state: "present" });
});
});

describe("Hermes Portable inference recovery gateway", () => {
it("rejects command environment drift before executable qualification", () => {
expect(() =>
captureHermesPortableInferenceRecoveryGateway("alpha", ["provider", "get", "ollama"], {
env: { UNEXPECTED: "value" },
timeout: 1_000,
}),
).toThrow("rejected command environment drift");
});
});
22 changes: 22 additions & 0 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
getNamedGatewayLifecycleState,
recoverNamedGatewayRuntime,
} from "../../gateway-runtime-action";
export { getNamedGatewayLifecycleState };
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import { assertNoOpenShellGatewayEndpointOverride } from "../../openshell-gateway-endpoint-guard";
import { isTerminalSandboxPhase, TERMINAL_SANDBOX_PHASES } from "../../state/gateway";
Expand Down Expand Up @@ -51,6 +52,7 @@ import {
import {
captureOpenshell,
captureOpenshellForStatus,
captureResolvedOpenshell,
getOpenshellBinary,
getStatusProbeTimeoutMs,
isCommandTimeout,
Expand Down Expand Up @@ -124,6 +126,26 @@ export const withSandboxLifecycleLock = withMcpLifecycleLock;
export const withSandboxLifecycleLockSync = withMcpLifecycleLockSync;
export const withConnectSandboxLifecycleLock = withMcpLifecycleLock;

/** Capture one recovery-only gateway command under receipt-owned authority. */
export function captureHermesPortableInferenceRecoveryGateway(
sandboxName: string,
args: string[],
options: { readonly env?: NodeJS.ProcessEnv; readonly timeout: number },
) {
if (options.env && Object.keys(options.env).length > 0) {
throw new Error("Hermes Portable inference recovery rejected command environment drift.");
}
const command = buildHermesPortableCommandAuthority(sandboxName);
return captureResolvedOpenshell(args, {
env: command.env,
openshellBinary: command.executablePath,
replaceEnv: true,
ignoreError: true,
includeStreams: true,
timeout: options.timeout,
});
}

type SandboxGatewayStateLookup = (
sandboxName: string,
gatewayName?: string,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { recoverHermesPortableOllamaInference } from "../../../onboard/experimental/hermes-portable-ollama-inference";
import type { SandboxEntry } from "../../../state/registry";
import {
captureHermesPortableInferenceRecoveryGateway,
type HermesPortableActiveLifecycleAuthority,
} from "../gateway-state";

export interface HermesPortableInferenceConnectRecoveryInput {
readonly sandboxName: string;
readonly authority: HermesPortableActiveLifecycleAuthority;
readonly readRegistry: (sandboxName: string) => SandboxEntry | null;
readonly verifyRoute: () => SandboxEntry;
}

/** Resume exact published Ollama authority for one probe-only connect operation. */
export function recoverHermesPortableInferenceForConnectProbe(
input: HermesPortableInferenceConnectRecoveryInput,
) {
return recoverHermesPortableOllamaInference({
intent: "connect-probe-only",
sandboxName: input.sandboxName,
entry: input.authority.entry,
runGatewayOpenshell: (args, options) =>
captureHermesPortableInferenceRecoveryGateway(input.sandboxName, args, options),
readRegistry: input.readRegistry,
verifyRoute: input.verifyRoute,
});
}
Loading
Loading