Skip to content

perf(cli): reduce Hermes recovery currentness latency #10641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
prekshivyas merged 11 commits into from
Aug 31, 2026
Merged

perf(cli): reduce Hermes recovery currentness latency #10641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
df7a261
chore(cli): start Hermes recovery currentness follow-up
senthilr-nv Aug 31, 2026
1967fef
perf(cli): reduce Hermes lifecycle recovery latency
senthilr-nv Aug 31, 2026
831adf6
perf(cli): reduce Hermes inference recovery latency
senthilr-nv Aug 31, 2026
dca34f1
fix(cli): recheck Hermes startup binding
senthilr-nv Aug 31, 2026
4c25d6a
merge(cli): refresh Hermes recovery currentness from main
senthilr-nv Aug 31, 2026
7fce902
merge(cli): refresh Hermes recovery currentness from main
senthilr-nv Aug 31, 2026
3da2ff8
fix(cli): report failed Hermes inference recovery timing
prekshivyas Aug 31, 2026
264be66
merge(cli): refresh Hermes recovery currentness from main
prekshivyas Aug 31, 2026
15801a0
fix(cli): finalize recovery timing at entry failures
prekshivyas Aug 31, 2026
f7ea1e2
fix(cli): report failed Hermes forward timing
prekshivyas Aug 31, 2026
601d001
refactor(cli): share Hermes resume timing default
prekshivyas Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 103 additions & 1 deletion src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,32 @@ describe("Hermes accepted launch-readiness probe", () => {
snapshot: {},
assertCurrent: assertRequalifiedReceiptCurrent,
} as never);
harness.recoverPortableDemoLifecycleSpy.mockReturnValue({ kind: "recovered" });
harness.recoverPortableDemoLifecycleSpy.mockImplementation((...args) => {
args[4]?.onComplete({
entryQualificationMs: 101,
containerStartMs: 102,
postStartCurrentnessMs: 103,
execReadyMs: 104,
preHealthCurrentnessMs: 105,
authenticatedHealthMs: 106,
startupLaunchMs: 107,
healthPollCurrentnessMs: 108,
finalQualificationMs: 109,
rollbackMs: 0,
qualificationCount: 2,
transactionCurrentnessCount: 20,
containerInspectionCount: 8,
containerStartCount: 1,
execReadyAttempts: 1,
authenticatedHealthCount: 1,
startupLaunchCount: 0,
rollbackCount: 0,
totalMs: 938,
containerAction: "started",
result: "recovered",
});
return { kind: "recovered" };
});

await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined();

Expand All @@ -406,6 +431,83 @@ describe("Hermes accepted launch-readiness probe", () => {
expect(harness.inspectLaunchReadinessSpy).toHaveBeenCalledOnce();
expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled();
expect(harness.logSpy.mock.calls.flat().join("\n")).toMatch(/result=ready/);
expect(harness.logSpy.mock.calls.flat().join("\n")).toContain(
"Hermes Portable lifecycle recovery timing: entryQualification=101ms containerStart=102ms postStartCurrentness=103ms execReady=104ms preHealthCurrentness=105ms authenticatedHealth=106ms startupLaunch=107ms healthPollCurrentness=108ms finalQualification=109ms rollback=0ms qualificationCount=2 transactionCurrentnessCount=20 containerInspectionCount=8 containerStartCount=1 execReadyAttempts=1 authenticatedHealthCount=1 startupLaunchCount=0 rollbackCount=0 total=938ms containerAction=started result=recovered",
);
});

it("reuses one recovered lifecycle when missing readiness routes to stopped inference", async () => {
const harness = missingHermesHarness("stopped");
harness.qualifyHermesPortableAcceptedReadinessAuthoritySpy
.mockImplementationOnce(() => {
throw new Error("stopped container has no current operating authority");
})
.mockReturnValue({
kind: "current",
commandAuthority: {
assertCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy,
assertTransactionCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy,
receipt: {} as never,
env: {},
executablePath: "/usr/bin/openshell",
},
});
const assertRequalifiedReceiptCurrent = vi.fn();
harness.requalifyPortableAgentAuthoritySpy.mockReturnValue({
kind: "already-current",
snapshot: {},
assertCurrent: assertRequalifiedReceiptCurrent,
} as never);
harness.recoverPortableDemoLifecycleSpy.mockReturnValue({ kind: "recovered" });

await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined();

expect(harness.recoverPortableDemoLifecycleSpy).toHaveBeenCalledOnce();
expect(harness.inspectHermesPortableOllamaReadinessRuntimeSpy).toHaveBeenCalledOnce();
expect(harness.recoverHermesPortableOllamaInferenceSpy).toHaveBeenCalledOnce();
expect(harness.publishLaunchReadinessSpy).toHaveBeenCalledOnce();
expect(assertRequalifiedReceiptCurrent.mock.calls.length).toBeGreaterThanOrEqual(2);
expect(harness.logSpy.mock.calls.flat().join("\n")).toMatch(
/lifecycleAction=recovered forwardAction=verified result=ready/,
);
});

it("rejects recovered lifecycle drift before stopped inference recovery", async () => {
const harness = missingHermesHarness("stopped");
harness.qualifyHermesPortableAcceptedReadinessAuthoritySpy
.mockImplementationOnce(() => {
throw new Error("stopped container has no current operating authority");
})
.mockReturnValue({
kind: "current",
commandAuthority: {
assertCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy,
assertTransactionCurrent: harness.assertHermesPortableOperatingCommandCurrentSpy,
receipt: {} as never,
env: {},
executablePath: "/usr/bin/openshell",
},
});
const assertRequalifiedReceiptCurrent = vi
.fn()
.mockImplementationOnce(() => undefined)
.mockImplementation(() => {
throw new Error("recovered receipt authority changed");
});
harness.requalifyPortableAgentAuthoritySpy.mockReturnValue({
kind: "already-current",
snapshot: {},
assertCurrent: assertRequalifiedReceiptCurrent,
} as never);
harness.recoverPortableDemoLifecycleSpy.mockReturnValue({ kind: "recovered" });

await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow(
"process.exit(1)",
);

expect(harness.recoverPortableDemoLifecycleSpy).toHaveBeenCalledOnce();
expect(harness.recoverHermesPortableOllamaInferenceSpy).not.toHaveBeenCalled();
expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled();
});

it("does not recover when stopped schema-6 requalification fails", async () => {
Expand Down
120 changes: 100 additions & 20 deletions src/lib/actions/sandbox/connect.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ import {
settlePortableOpenClawPairing,
withLaunchReadinessMutationGate,
} from "./launch-readiness";
import type { HermesPortableLifecycleRecoveryTimingEvidence } from "../../onboard/experimental/hermes-portable-lifecycle";
import {
checkAndRecoverSandboxProcesses,
executeSandboxExecCommand,
Expand Down Expand Up @@ -724,7 +725,15 @@ function writeHermesPortableForwardRecoveryTiming(
evidence: HermesPortableForwardRecoveryTimingEvidence,
): void {
console.log(
` Hermes Portable forward recovery timing: list=${String(evidence.listMs)}ms listCount=${String(evidence.listCount)} stop=${String(evidence.stopMs)}ms stopCount=${String(evidence.stopCount)} start=${String(evidence.startMs)}ms startCount=${String(evidence.startCount)} settle=${String(evidence.settleMs)}ms settleCount=${String(evidence.settleCount)} total=${String(evidence.totalMs)}ms result=proved`,
` Hermes Portable forward recovery timing: list=${String(evidence.listMs)}ms listCount=${String(evidence.listCount)} stop=${String(evidence.stopMs)}ms stopCount=${String(evidence.stopCount)} start=${String(evidence.startMs)}ms startCount=${String(evidence.startCount)} settle=${String(evidence.settleMs)}ms settleCount=${String(evidence.settleCount)} total=${String(evidence.totalMs)}ms result=${evidence.result}`,
);
}

function writeHermesPortableLifecycleRecoveryTiming(
evidence: HermesPortableLifecycleRecoveryTimingEvidence,
): void {
console.log(
` Hermes Portable lifecycle recovery timing: entryQualification=${String(evidence.entryQualificationMs)}ms containerStart=${String(evidence.containerStartMs)}ms postStartCurrentness=${String(evidence.postStartCurrentnessMs)}ms execReady=${String(evidence.execReadyMs)}ms preHealthCurrentness=${String(evidence.preHealthCurrentnessMs)}ms authenticatedHealth=${String(evidence.authenticatedHealthMs)}ms startupLaunch=${String(evidence.startupLaunchMs)}ms healthPollCurrentness=${String(evidence.healthPollCurrentnessMs)}ms finalQualification=${String(evidence.finalQualificationMs)}ms rollback=${String(evidence.rollbackMs)}ms qualificationCount=${String(evidence.qualificationCount)} transactionCurrentnessCount=${String(evidence.transactionCurrentnessCount)} containerInspectionCount=${String(evidence.containerInspectionCount)} containerStartCount=${String(evidence.containerStartCount)} execReadyAttempts=${String(evidence.execReadyAttempts)} authenticatedHealthCount=${String(evidence.authenticatedHealthCount)} startupLaunchCount=${String(evidence.startupLaunchCount)} rollbackCount=${String(evidence.rollbackCount)} total=${String(evidence.totalMs)}ms containerAction=${evidence.containerAction} result=${evidence.result}`,
);
}

Expand Down Expand Up @@ -1846,11 +1855,16 @@ async function runConnectEntryPreflight(
probeOnly,
probeTiming,
hermesPortableCommandAuthority,
retainedHermesLifecycleRecovery,
withinLifecycleFence,
}: {
probeOnly: boolean;
probeTiming?: ProbeTimingRecorder;
hermesPortableCommandAuthority?: HermesPortableReadinessCommandAuthority;
retainedHermesLifecycleRecovery?: {
readonly kind: "already-running" | "recovered";
readonly assertCurrent: () => void;
};
withinLifecycleFence?: (route: {
readonly hermesPortable: boolean;
readonly hermesPortableCommandAuthority?: HermesPortableReadinessCommandAuthority;
Expand Down Expand Up @@ -1901,16 +1915,36 @@ async function runConnectEntryPreflight(
assertSandboxGatewayRouteCompatible(sandboxName, registered, gatewayName),
);
}
const initialRecovery = measure("lifecycle", () =>
hermesPortableCommandAuthority
? recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
hermesPortableCommandAuthority,
)
: recoverPortableDemoSandboxLifecycleForConnect(sandboxName, registered, gatewayName),
);
const initialRecovery = retainedHermesLifecycleRecovery
? measure("authority", () => {
retainedHermesLifecycleRecovery.assertCurrent();
return { kind: retainedHermesLifecycleRecovery.kind } as const;
})
: measure("lifecycle", () =>
hermesPortableCommandAuthority
? recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
hermesPortableCommandAuthority,
hermesPortable && probeTiming
? { onComplete: writeHermesPortableLifecycleRecoveryTiming }
: undefined,
)
: hermesPortable && probeTiming
? recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
undefined,
{ onComplete: writeHermesPortableLifecycleRecoveryTiming },
)
: recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
),
);
probeTiming?.setLifecycleAction(
initialRecovery.kind === "recovered"
? "recovered"
Expand Down Expand Up @@ -1969,12 +2003,23 @@ async function runConnectEntryPreflight(
activeAuthority.entry,
currentGateway,
hermesPortableCommandAuthority,
probeTiming
? { onComplete: writeHermesPortableLifecycleRecoveryTiming }
: undefined,
)
: recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
activeAuthority.entry,
currentGateway,
),
: hermesPortable && probeTiming
? recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
activeAuthority.entry,
currentGateway,
undefined,
{ onComplete: writeHermesPortableLifecycleRecoveryTiming },
)
: recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
activeAuthority.entry,
currentGateway,
),
);
if (recovery.kind === "not-installed") {
probeTiming?.setLifecycleAction("failed");
Expand Down Expand Up @@ -2235,13 +2280,14 @@ async function prepareConnectSandboxWithinLifecycleFence(
readonly active: HermesPortableActiveLifecycleAuthority;
readonly command: HermesPortableReadinessCommandAuthority;
} | null = null;
let retainedHermesLifecycleRecovery: {
readonly kind: "already-running" | "recovered";
readonly assertCurrent: () => void;
} | null = null;
let initialPortableAuthority: ReturnType<typeof qualifyPortableAgentLifecycleAuthority>;
try {
initialPortableAuthority = probeTiming!.measure("authority", () =>
qualifyPortableAgentLifecycleAuthority(
sandboxName,
portableAgentLifecycleAuthorityDeps(),
),
qualifyPortableAgentLifecycleAuthority(sandboxName, portableAgentLifecycleAuthorityDeps()),
);
} catch {
probeTiming!.markFailureStage("authority");
Expand All @@ -2257,6 +2303,12 @@ async function prepareConnectSandboxWithinLifecycleFence(
),
);
let qualified;
let recoveredLifecycle:
| {
readonly kind: "already-running" | "recovered";
readonly assertReceiptCurrent: () => void;
}
| undefined;
try {
qualified = probeTiming!.measure("authority", () =>
qualifyHermesPortableAcceptedReadinessAuthority(sandboxName),
Expand Down Expand Up @@ -2292,13 +2344,19 @@ async function prepareConnectSandboxWithinLifecycleFence(
sandboxName,
registered,
resolveSandboxGatewayName(registered),
undefined,
{ onComplete: writeHermesPortableLifecycleRecoveryTiming },
),
);
if (recovery.kind === "not-installed") {
probeTiming!.setLifecycleAction("failed");
throw new Error("Hermes portable lifecycle authority disappeared during probe");
}
probeTiming!.setLifecycleAction(recovery.kind === "recovered" ? "recovered" : "reused");
recoveredLifecycle = {
kind: recovery.kind,
assertReceiptCurrent: requalified.assertCurrent,
};
active = probeTiming!.measure("authority", () =>
requireHermesPortableActiveLifecycleAuthority(
sandboxName,
Expand All @@ -2311,6 +2369,8 @@ async function prepareConnectSandboxWithinLifecycleFence(
priorReceiptAuthority: requalified,
}),
);
hermesMissingFastPathEligible =
requalified.kind === "already-current" && qualified.kind === "current";
}
if (qualified.kind === "requalification-required") {
const requalified = probeTiming!.measure("authority", () =>
Expand Down Expand Up @@ -2344,6 +2404,25 @@ async function prepareConnectSandboxWithinLifecycleFence(
active,
command: qualified.commandAuthority,
};
if (recoveredLifecycle) {
const retainedActive = active;
const retainedCommand = qualified.commandAuthority;
retainedHermesLifecycleRecovery = {
kind: recoveredLifecycle.kind,
assertCurrent: () => {
recoveredLifecycle.assertReceiptCurrent();
retainedCommand.assertCurrent();
const current = requireHermesPortableActiveLifecycleAuthority(
sandboxName,
retainedActive,
portableAgentLifecycleAuthorityDeps(),
);
if (!isDeepStrictEqual(current.entry, retainedActive.entry)) {
throw new Error("Hermes portable lifecycle authority changed after recovery");
}
},
};
}
} catch {
probeTiming!.markFailureStage("authority");
failHermesPortableReadinessAuthority(sandboxName);
Expand Down Expand Up @@ -2550,6 +2629,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
...(hermesReadinessAuthority
? { hermesPortableCommandAuthority: hermesReadinessAuthority.command }
: {}),
...(retainedHermesLifecycleRecovery ? { retainedHermesLifecycleRecovery } : {}),
withinLifecycleFence: async ({
hermesPortable,
hermesPortableCommandAuthority,
Expand Down
Loading
Loading