diff --git a/apps/desktop/e2e/update/build-fail.spec.ts b/apps/desktop/e2e/update/build-fail.spec.ts index 2db60bdf6291..f7aa698a72e1 100644 --- a/apps/desktop/e2e/update/build-fail.spec.ts +++ b/apps/desktop/e2e/update/build-fail.spec.ts @@ -4,10 +4,15 @@ * #124040: the Desktop build failed inside the update takeover's completion * stage and the failure was swallowed / kept looping instead of reaching the * user. Here upstream main carries a Desktop source file that does not - * compile; the user clicks "Update now". The update must end with the app - * telling the user the update FAILED (the relaunched app's result dialog is - * logged as `[updates] detached update FAILED`), never "finished OK", and the - * updater must not be left running. + * compile; the user clicks "Update now". + * + * The build runs after the code committed, so under contract C3 the update + * succeeds with an owed follow-up: Hermes IS on the new code and only this + * app was not rebuilt. The relaunched (old-build) app must say exactly that + * and what to do — the action-required dialog, logged as `[updates] detached + * update finished with manual action` — never "finished OK" (a silent stale + * app) and never "FAILED / still on the previous version" (false: the code + * moved). The updater must not be left running. */ import * as path from 'node:path' @@ -32,7 +37,7 @@ import { waitForUpdateOffer } from './harness' -test('a Desktop build failure during Update now is reported to the user as a failed update', async () => { +test('a Desktop build failure during Update now tells the user the app was not rebuilt and how to fix it', async () => { test.setTimeout(20 * 60_000) const session = await startInstallSession() const { facts, env } = session @@ -52,7 +57,7 @@ test('a Desktop build failure during Update now is reported to the user as a fai await clickUpdateNowAndExpectHandoff(launched, facts) }) - await test.step('the failed build reaches the user as a failed update', async () => { + await test.step('the failed build reaches the user as an app that still needs rebuilding', async () => { await waitFor( 'the relaunched app to report the update outcome', () => /\[updates\] detached update (finished|FAILED)/.test(desktopLog(facts)), @@ -64,8 +69,15 @@ test('a Desktop build failure during Update now is reported to the user as a fai .filter(line => /\[updates\] detached update/.test(line)) .join('\n') - expect(outcome, `the update is reported as FAILED, not finished\n${explain()}`).toMatch(/detached update FAILED/) + expect(outcome, `the owed rebuild reaches the user as an action, not a silent OK\n${explain()}`).toMatch( + /detached update finished with manual action/ + ) + expect(outcome, 'the user is told the Desktop app could not be rebuilt and how to rebuild it').toMatch( + /Desktop app could not be rebuilt.*hermes desktop --force-build/ + ) expect(outcome).not.toMatch(/detached update finished OK/) + // The code committed: "FAILED / still on the previous version" would be false. + expect(outcome).not.toMatch(/detached update FAILED|previous version/) // The relaunched app reports the result while posix.sh is still inside launch_app's 1.5 s // acceptance window, so the updater gets a bounded moment to exit instead of none. await expect @@ -77,10 +89,8 @@ test('a Desktop build failure during Update now is reported to the user as a fai { timeout: 30_000, message: 'no updater is left running after the failure' } ) .toEqual([]) - // What the user is left on: record it for triage (the checkout moved; the app bundle did not). - test - .info() - .annotations.push({ type: 'checkout-after-failure', description: git(facts.checkout, 'rev-parse', 'HEAD') }) + // The checkout moved (that is why it is not "previous version"); the app bundle did not. + expect(git(facts.checkout, 'rev-parse', 'HEAD'), 'the code update committed').toBe(target) }) } finally { await session.close() diff --git a/apps/desktop/electron/managed-ssh-update.test.ts b/apps/desktop/electron/managed-ssh-update.test.ts index adef9261f098..348e83e1c7f4 100644 --- a/apps/desktop/electron/managed-ssh-update.test.ts +++ b/apps/desktop/electron/managed-ssh-update.test.ts @@ -507,6 +507,105 @@ test.runIf(process.platform !== 'win32')( } ) +test.runIf(process.platform !== 'win32')( + 'a committed update that still owes a step stays a success but names the owed step', + async () => { + const home = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-owed-')) + + const run = async ( + followups: unknown[], + { exitCode = 0, outcome = 'success', restoreFails = false, userAction = null as unknown } = {} + ) => { + const receipts = path.join(home, 'logs', 'update_receipts') + await mkdir(receipts, { recursive: true }) + await writeFile(path.join(home, `.update_exit_code.${CORRELATION}`), String(exitCode)) + await writeFile( + path.join(receipts, 'update_x.json'), + JSON.stringify({ + correlation_id: CORRELATION, + outcome, + started_at: '2026-08-23T00:00:00Z', + finished_at: '2026-08-23T00:01:00Z', + followups, + user_action: userAction + }) + ) + + const command = buildRemoteUpdateObservationCommand( + { ssh: { exec: async () => '' }, platform: 'Linux', hermesPath: '/opt/hermes/hermes', hermesHome: home }, + CORRELATION + ) + + const observed = parseRemoteUpdateObservation((await exec(command, { shell: 'sh' })).stdout, CORRELATION) + + return runManagedSshUpdate({ + connectionId: 'home', + correlationId: CORRELATION, + scopes: [{ key: 'conn:home::default', profile: 'default' }], + preflightRemote: async () => {}, + drainScope: async () => {}, + updateRemote: async () => ({ exitCode: observed.exitCode!, receipt: observed.receipt! }), + awaitRestoreClearance: async () => {}, + closeTransports: async () => {}, + restoreScope: async () => { + if (restoreFails) { + throw new Error('restore refused') + } + }, + releaseGate: () => {} + }) + } + + try { + const owed = await run([{ step: 'dependencies', reason: 'sync failed', at: '2026-08-23T00:01:00Z' }]) + + assert.equal(owed.ok, true) + assert.equal(owed.updateOk, true) + assert.equal(owed.restoreOk, true) + assert.deepEqual(owed.owed, [{ step: 'dependencies', reason: 'sync failed' }]) + assert.match(owed.message || '', /still owed: dependencies \(sync failed\)/) + assert.match(owed.message || '', /hermes update/) + assert.match(managedSshUpdateAllRow({ id: 'home' }, owed).detail || '', /dependencies/) + + const clean = await run([]) + + assert.equal(clean.ok, true) + assert.equal(clean.owed, undefined) + assert.equal(clean.message, 'Remote Hermes updated and every managed SSH profile is ready.') + + // The debt survives a failed restoration: update and restore verdicts stay independent. + const unrestored = await run([{ step: 'dependencies', reason: 'sync failed' }], { restoreFails: true }) + + assert.equal(unrestored.updateOk, true) + assert.equal(unrestored.restoreOk, false) + assert.deepEqual(unrestored.owed, [{ step: 'dependencies', reason: 'sync failed' }]) + assert.match( + managedSshUpdateAllRow({ id: 'home' }, unrestored).detail || '', + /still owed: dependencies \(sync failed\)/ + ) + + // record_user_action turns a success into partial/exit 1: its exact manual instruction is shown, + // and the rerun advice is never offered for a parked stash. + const instruction = 'Run `git stash apply stash@{0}` in /opt/hermes to restore your local changes.' + const partial = await run([], { + exitCode: 1, + outcome: 'partial', + userAction: { step: 'local_changes', reason: instruction } + }) + + assert.equal(partial.updateOk, false) + assert.equal(partial.restoreOk, true) + assert.deepEqual(partial.owed, [{ step: 'local_changes', reason: instruction }]) + const detail = managedSshUpdateAllRow({ id: 'home' }, partial).detail || '' + assert.ok(detail.includes(instruction), detail) + assert.match(detail, /local_changes/) + assert.doesNotMatch(detail, /Re-run `hermes update`/) + } finally { + await rm(home, { force: true, recursive: true }) + } + } +) + test.runIf(process.platform !== 'win32')( 'managed observer unwraps a named profile home for the install-wide marker', async () => { @@ -538,6 +637,53 @@ test.runIf(process.platform !== 'win32')( } ) +test.runIf(process.platform !== 'win32')( + 'managed observer reads a named profile update receipt from the root home the CLI writes', + async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-profile-receipt-')) + const profileHome = path.join(root, 'profiles', 'research') + + const receipt = (outcome: string) => + JSON.stringify({ + correlation_id: CORRELATION, + outcome, + started_at: '2026-08-23T00:00:00Z', + finished_at: '2026-08-23T00:01:00Z' + }) + + try { + // The CLI's receipt store is install-wide; a stale copy under the profile is not its source. + await mkdir(path.join(root, 'logs', 'update_receipts'), { recursive: true }) + await mkdir(path.join(profileHome, 'logs', 'update_receipts'), { recursive: true }) + await writeFile(path.join(profileHome, `.update_exit_code.${CORRELATION}`), '0') + await writeFile(path.join(root, 'logs', 'update_receipts', `update_${CORRELATION}.json`), receipt('success')) + await writeFile( + path.join(profileHome, 'logs', 'update_receipts', `update_${CORRELATION}.json`), + receipt('failed') + ) + + const command = buildRemoteUpdateObservationCommand( + { + ssh: { exec: async () => '' }, + platform: 'Linux', + hermesPath: '/opt/hermes/hermes', + hermesHome: profileHome + }, + CORRELATION + ) + + const { stdout } = await exec(command, { shell: 'sh' }) + const parsed = parseRemoteUpdateObservation(stdout, CORRELATION) + + assert.equal(parsed.exitCode, 0) + assert.equal(parsed.receipt?.correlationId, CORRELATION) + assert.equal(parsed.receipt?.outcome, 'success') + } finally { + await rm(root, { force: true, recursive: true }) + } + } +) + test('Windows coordinator handoff is pending until its marker clears and correlated receipt is durable', async () => { const replies = [ observation({ marker: 'live', markerPid: 44 }), diff --git a/apps/desktop/electron/managed-ssh-update.ts b/apps/desktop/electron/managed-ssh-update.ts index c87796d11d09..808ee04525c3 100644 --- a/apps/desktop/electron/managed-ssh-update.ts +++ b/apps/desktop/electron/managed-ssh-update.ts @@ -29,6 +29,11 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0- type ManagedUpdateOutcome = 'updated' | 'update-failed' | 'restore-failed' | 'update-and-restore-failed' | 'refused' +interface ManagedUpdateOwedStep { + step: string + reason: string +} + interface ManagedUpdateReceiptSummary { correlationId: string outcome: string @@ -39,6 +44,9 @@ interface ManagedUpdateReceiptSummary { preVersion?: string postVersion?: string stopReason?: string + // Post-commit steps a successful update still owes (contract C3). + followups?: ManagedUpdateOwedStep[] + userAction?: ManagedUpdateOwedStep | null } interface ManagedUpdateScopeResult { @@ -57,6 +65,8 @@ interface ManagedConnectionUpdateResult { exitCode: null | number receipt: ManagedUpdateReceiptSummary | null scopes: ManagedUpdateScopeResult[] + // Every step a committed update still owes; set only when non-empty. + owed?: ManagedUpdateOwedStep[] error?: string message?: string // Set when the connection was deliberately not attempted (a known safety @@ -324,9 +334,10 @@ from pathlib import Path home=Path(os.path.expanduser(sys.argv[1])) correlation=sys.argv[2] -# The update marker is install-wide even when the launcher was invoked with a -# named profile home (/profiles/). Correlated status/intent/receipt -# remain under the launch home, matching the CLI writer. +# The update marker and the receipt store are install-wide even when the launcher +# was invoked with a named profile home (/profiles/): the CLI writes +# receipts under the root home. Correlated status/ready/intent stay under the +# launch home, matching their writers. profile_parent=home.parent.name is_profile_home=(profile_parent.lower()=='profiles') if os.name=='nt' else (profile_parent=='profiles') install_root=home.parent.parent if is_profile_home else home @@ -416,7 +427,7 @@ def terminal_code(): except ValueError:return 'malformed' def receipt(): - directory=home/'logs'/'update_receipts' + directory=install_root/'logs'/'update_receipts' try:paths=sorted(directory.glob('update_*.json'),key=lambda p:p.stat().st_mtime_ns,reverse=True) except OSError:return None for path in paths: @@ -426,12 +437,15 @@ def receipt(): if payload.get('outcome')=='running' or not payload.get('finished_at'):continue pre=payload.get('pre_update') if isinstance(payload.get('pre_update'),dict) else {} post=payload.get('post_update') if isinstance(payload.get('post_update'),dict) else {} + action=payload.get('user_action') return { 'correlationId':correlation,'outcome':str(payload.get('outcome') or 'unknown'), 'startedAt':payload.get('started_at'),'finishedAt':payload.get('finished_at'), 'preSha':pre.get('sha'),'postSha':post.get('sha'), 'preVersion':pre.get('version'),'postVersion':post.get('version'), 'stopReason':payload.get('stop_reason'), + 'followups':[{'step':str(f.get('step')),'reason':str(f.get('reason') or '')} for f in payload.get('followups') or [] if isinstance(f,dict)], + 'userAction':{'step':str(action.get('step')),'reason':str(action.get('reason') or '')} if isinstance(action,dict) else None, } return None @@ -480,6 +494,59 @@ function buildRemoteUpdateObservationCommand(target: RemoteUpdateTarget, correla return `python3 -c ${shq(OBSERVATION_SCRIPT)} ${shq(home)} ${shq(correlation)}` } +function owedStep(value: any): ManagedUpdateOwedStep | null { + return value && typeof value === 'object' ? { step: String(value.step), reason: String(value.reason ?? '') } : null +} + +// An older remote observer omits the debt fields: read them as none. +function withReceiptDebt(receipt: any): ManagedUpdateReceiptSummary { + const followups = Array.isArray(receipt.followups) ? receipt.followups.map(owedStep).filter(Boolean) : [] + + return { ...receipt, followups, userAction: owedStep(receipt.userAction) } +} + +// C3: a committed update is a success even when post-commit steps are owed; +// name each one and its remedy instead of claiming everything is ready. The +// debt rides on every terminal receipt, so it is composed with the update and +// restoration verdicts, never only on success: a committed run whose profile +// restore failed, and a partial run that parked local changes, still owe it. +function managedUpdateMessage( + outcome: ManagedUpdateOutcome, + restoreOk: boolean, + receipt: ManagedUpdateReceiptSummary | undefined +): { owed?: ManagedUpdateOwedStep[]; message: string } { + const followups = receipt?.followups ?? [] + const userAction = receipt?.userAction + const owed = [...followups, ...(userAction ? [userAction] : [])] + + const status = + outcome === 'updated' + ? 'Remote Hermes updated' + : restoreOk + ? 'The remote update failed, but every managed SSH profile was restored.' + : 'The remote update transaction could not restore every managed SSH profile.' + + if (!owed.length) { + return { message: outcome === 'updated' ? `${status} and every managed SSH profile is ready.` : status } + } + + const message = + (outcome === 'updated' ? `${status}, but these steps are still owed: ` : `${status} These steps are still owed: `) + + [ + ...followups.map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)), + ...(userAction ? [userAction.step] : []) + ].join('; ') + + '.' + + // Only follow-ups are finished by a rerun; a user action (e.g. a parked + // stash) is the producer's exact manual instruction, shown verbatim. + (followups.length + ? ` Re-run \`hermes update\` on the remote to finish ${followups.map(step => step.step).join(', ')}.` + : '') + + (userAction?.reason ? ` ${userAction.reason}` : '') + + return { owed, message } +} + function parseRemoteUpdateObservation(raw: string, correlationId: string): RemoteUpdateObservation { const correlation = validateCorrelationId(correlationId) let parsed: any @@ -525,7 +592,7 @@ function parseRemoteUpdateObservation(raw: string, correlationId: string): Remot throw new Error('Remote update receipt did not match this transaction.') } - receipt = parsed.receipt as ManagedUpdateReceiptSummary + receipt = withReceiptDebt(parsed.receipt) } let coordinatorReady: RemoteUpdateObservation['coordinatorReady'] = null @@ -937,12 +1004,7 @@ async function runManagedSshUpdate( receipt: proof?.receipt ?? null, scopes: restoreResults, ...(error ? { error } : {}), - message: - outcome === 'updated' - ? 'Remote Hermes updated and every managed SSH profile is ready.' - : restoreOk - ? 'The remote update failed, but every managed SSH profile was restored.' - : 'The remote update transaction could not restore every managed SSH profile.' + ...managedUpdateMessage(outcome, restoreOk, proof?.receipt) } } @@ -1179,6 +1241,7 @@ export { managedSshUpdateAllRow, type ManagedUpdateDeps, type ManagedUpdateOutcome, + type ManagedUpdateOwedStep, type ManagedUpdateReceiptSummary, type ManagedUpdateScopeResult, markerIsClear, diff --git a/apps/desktop/src/global.d.ts b/apps/desktop/src/global.d.ts index e0b63fe03465..0159edf0902d 100644 --- a/apps/desktop/src/global.d.ts +++ b/apps/desktop/src/global.d.ts @@ -1232,6 +1232,8 @@ export interface DesktopManagedUpdateReceipt { preVersion?: string postVersion?: string stopReason?: string + followups?: Array<{ step: string; reason: string }> + userAction?: { step: string; reason: string } | null } export interface DesktopManagedConnectionUpdateResult { @@ -1244,6 +1246,8 @@ export interface DesktopManagedConnectionUpdateResult { exitCode: number | null receipt: DesktopManagedUpdateReceipt | null scopes: Array<{ profile: string; restored: boolean; error?: string }> + /** Post-commit steps a successful update still owes (named in `message`). */ + owed?: Array<{ step: string; reason: string }> error?: string message?: string } diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index cec308d4b0c9..f1d6cfead876 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -4232,10 +4232,10 @@ export const en: Translations = { restarting: 'Backend restarting to load the update…', notAvailable: 'Update not available for this backend.', failed: 'Backend update failed.', - noReturn: 'Backend didn’t come back online. The update may not have completed — check the backend host.' + noReturn: 'Backend didn’t come back online. The update may not have completed — check the backend host.', + owed: steps => `Backend updated, but still owed: ${steps}. Re-run \`hermes update\` to finish them.` }, - // Update-status overlay + version-details (mechanism-aware update UI). - appName: 'Hermes', + appName: 'Hermes', // Update-status overlay + version-details (mechanism-aware update UI). version: (value: string) => `Version ${value}`, versionUnavailable: 'Version unavailable', checkNow: 'Check now', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 7645d14cd8ff..9a88a48f8d2a 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -3521,9 +3521,9 @@ export interface Translations { notAvailable: string failed: string noReturn: string + owed: (steps: string) => string } - /** Update-status overlay + version-details (mechanism-aware update UI): - * the overlay reads these off t.updates directly. */ + /** Update-status overlay + version-details (mechanism-aware update UI), read off t.updates directly. */ appName: string version: (value: string) => string versionUnavailable: string diff --git a/apps/desktop/src/store/updates.test.ts b/apps/desktop/src/store/updates.test.ts index 173ff4c36a80..5b8811c54ea4 100644 --- a/apps/desktop/src/store/updates.test.ts +++ b/apps/desktop/src/store/updates.test.ts @@ -1487,6 +1487,100 @@ describe('applyBackendUpdate recovery', () => { expect(result.ok).toBe(false) expect($backendUpdateApply.get().stage).toBe('error') }, 10000) + + it('reports a committed update with owed follow-ups as success plus an owed-step warning', async () => { + updateHermesSpy.mockResolvedValue({ action_id: 'e'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + getActionStatusSpy.mockResolvedValue({ + exit_code: 0, + lines: ['Update complete!'], + name: 'hermes-update', + pid: null, + running: false, + receipt: { + action_id: 'e'.repeat(32), + outcome: 'success', + followups: [{ step: 'dependencies', reason: 'sync failed' }], + user_action: null + } + }) + notifySpy.mockClear() + + const promise = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + const result = await promise + + expect(result).toMatchObject({ ok: true }) + expect(result.message).toContain('dependencies (sync failed)') + expect(notifySpy).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'warning', message: expect.stringContaining('dependencies (sync failed)') }) + ) + }) + + it("names a partial update's owed user action verbatim, only from this action's own receipt", async () => { + const instruction = 'Your local changes are parked in stash@{0}; run `git stash pop` in ~/.hermes/hermes-agent.' + + const status = (actionId: string) => ({ + exit_code: 1, + lines: ['update parked local changes'], + name: 'hermes-update', + pid: null, + running: false, + receipt: { + action_id: actionId, + outcome: 'partial', + followups: [], + user_action: { step: 'local_changes', reason: instruction } + } + }) + + updateHermesSpy.mockResolvedValue({ action_id: 'c'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + getActionStatusSpy.mockResolvedValue(status('c'.repeat(32))) + const own = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + const result = await own + + // exit 1 stays a failed apply, but the committed run's debt is named, never as "re-run hermes update". + expect(result).toMatchObject({ ok: false, error: 'apply-failed' }) + expect(result.message).toContain(`local_changes: ${instruction}`) + expect(result.message).not.toMatch(/Re-run `hermes update`/) + expect($backendUpdateApply.get()).toMatchObject({ stage: 'error', message: result.message }) + + // The status route attaches the latest receipt when this action has none: another run's debt is not ours. + getActionStatusSpy.mockResolvedValue(status('d'.repeat(32))) + const foreign = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + + expect((await foreign).message).not.toContain('local_changes') + }) + + it("never lets another action's receipt certify this update", async () => { + const receipt = (actionId: string) => ({ + exit_code: null, + lines: [], + name: 'hermes-update', + pid: null, + running: false, + receipt: { + action_id: actionId, + outcome: 'success', + started_at: new Date().toISOString(), + finished_at: new Date().toISOString(), + followups: [], + user_action: null + } + }) + + updateHermesSpy.mockResolvedValue({ action_id: 'b'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + getActionStatusSpy.mockResolvedValue(receipt('a'.repeat(32))) + const foreign = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(6 * 60 * 1000 + 1500) + await expect(foreign).resolves.toMatchObject({ ok: false }) + + getActionStatusSpy.mockResolvedValue(receipt('b'.repeat(32))) + const own = applyBackendUpdate() + await vi.advanceTimersByTimeAsync(1500) + await expect(own).resolves.toMatchObject({ ok: true }) + }) }) describe('startUpdatePoller', () => { diff --git a/apps/desktop/src/store/updates.ts b/apps/desktop/src/store/updates.ts index bdcd35b8cb95..4b39552ae50e 100644 --- a/apps/desktop/src/store/updates.ts +++ b/apps/desktop/src/store/updates.ts @@ -3,6 +3,7 @@ * surfaces it as an ambient pill, and orchestrates the apply flow. */ +import { updateDebt } from '@hermes/shared' import { atom } from 'nanostores' import { connectionScoped, profileScoped } from '@/api/client' @@ -22,7 +23,7 @@ import { $connectionsRegistry, refreshConnectionsRegistry } from '@/store/connec import { reconnectGateway } from '@/store/gateway-reconnect' import { dismissNotification, notify } from '@/store/notifications' import { $connection } from '@/store/session' -import type { BackendUpdateCheckResponse } from '@/types/hermes' +import type { BackendUpdateCheckResponse, UpdateReceiptSummary } from '@/types/hermes' /** Keyed per retired-channel revision: a new retirement (or a revision bump on * the same channel) re-shows the notice, a plain re-check never does. */ @@ -747,7 +748,43 @@ const BACKEND_ACTION_POLL_MS = 1500 const BACKEND_ACTION_MAX_MS = 6 * 60 * 1000 const BACKEND_RETURN_MAX_MS = 4 * 60 * 1000 -function finishBackendApply(returned: boolean): DesktopUpdateApplyResult { +// C3: a committed update succeeds even when post-commit steps are still owed, +// and a partial one (committed, but the user must act) still owes them. Name +// every owed step of THIS action's committed run whatever the exit: follow-ups +// with the rerun remedy, a user action with the producer's own instruction, +// verbatim (a rerun does not restore a parked stash). +function owedMessage(receipt: UpdateReceiptSummary | undefined, actionId: string | undefined): string | null { + const debt = updateDebt(receipt, actionId) + + if (!debt) { + return null + } + + return [debt.followups && translateNow('updates.applyStatus.owed', debt.followups), debt.userAction] + .filter(Boolean) + .join(' ') +} + +// A failed apply whose receipt still proves a committed partial run (exit 1) +// names what that run owes instead of only "failed" / "did not come back". +function failBackendApply( + fallback: { shown: string; result: string }, + status: { receipt?: UpdateReceiptSummary } | null, + actionId: string | undefined +): DesktopUpdateApplyResult { + const owed = owedMessage(status?.receipt, actionId) + const message = owed ? `${translateNow('updates.applyStatus.failed')} ${owed}` : fallback.shown + + $backendUpdateApply.set({ ...$backendUpdateApply.get(), applying: false, stage: 'error', error: 'apply-failed', message }) + + return { ok: false, error: 'apply-failed', message: owed ? message : fallback.result } +} + +function finishBackendApply( + returned: boolean, + receipt?: UpdateReceiptSummary, + actionId?: string +): DesktopUpdateApplyResult { if (returned) { $backendUpdateApply.set(IDLE) setUpdateOverlayOpen(false) @@ -765,19 +802,22 @@ function finishBackendApply(returned: boolean): DesktopUpdateApplyResult { // affordance in remote mode targets the backend, so nothing ever told // them the app itself was stale). Nudge with a one-click client update. void maybeNudgeClientAfterBackendUpdate() + const owed = owedMessage(receipt, actionId) + + if (owed) { + notify({ durationMs: 0, kind: 'warning', message: owed }) + + return { ok: true, message: owed } + } return { ok: true, message: 'Backend update applied.' } } - $backendUpdateApply.set({ - ...$backendUpdateApply.get(), - applying: false, - stage: 'error', - error: 'apply-failed', - message: translateNow('updates.applyStatus.noReturn') - }) - - return { ok: false, error: 'apply-failed', message: 'Backend did not come back online.' } + return failBackendApply( + { shown: translateNow('updates.applyStatus.noReturn'), result: 'Backend did not come back online.' }, + { receipt }, + actionId + ) } function ingestBackendActionStatus(status: Awaited>): void { @@ -813,7 +853,11 @@ function completedAfterRestart( * run started at-or-after we kicked the update off counts — an older * receipt describes a previous update, and a still-running one proves * nothing yet. The 60s slack absorbs client/backend clock skew. */ -function receiptProvesOutcome(status: Awaited>, applyStartedAtMs: number): boolean { +function receiptProvesOutcome( + status: Awaited>, + applyStartedAtMs: number, + actionId: string | undefined +): boolean { const receipt = status.receipt if (!receipt || !receipt.finished_at || !receipt.started_at) { @@ -824,6 +868,12 @@ function receiptProvesOutcome(status: Awaited return false } + // A backend that records the writing action (null for a CLI run) proves + // identity directly: another action's success cannot certify this one. + if (actionId && receipt.action_id !== undefined) { + return receipt.action_id === actionId + } + const startedMs = Date.parse(receipt.started_at) return Number.isFinite(startedMs) && startedMs >= applyStartedAtMs - 60_000 @@ -929,15 +979,15 @@ async function runBackendUpdate(): Promise { } if (last.exit_code === 0 || (last.exit_code === null && completedAfterRestart(last, started.action_id))) { - return finishBackendApply(true) + return finishBackendApply(true, last.receipt, started.action_id) } // #91277 bullet 3: the backend now attaches the durable update // receipt to the status. A receipt whose run STARTED after we kicked // this update off is authoritative — read its outcome instead of // inferring from log markers or timing out across the restart gap. - if (last.exit_code === null && receiptProvesOutcome(last, applyStartedAtMs)) { - return finishBackendApply(last.receipt!.outcome === 'success') + if (last.exit_code === null && receiptProvesOutcome(last, applyStartedAtMs, started.action_id)) { + return finishBackendApply(last.receipt!.outcome === 'success', last.receipt, started.action_id) } if (!started.action_id && last.exit_code === null) { @@ -957,15 +1007,11 @@ async function runBackendUpdate(): Promise { } } - $backendUpdateApply.set({ - ...$backendUpdateApply.get(), - applying: false, - stage: 'error', - error: 'apply-failed', - message: translateNow('updates.applyStatus.failed') - }) - - return { ok: false, error: 'apply-failed', message: 'Backend update failed.' } + return failBackendApply( + { shown: translateNow('updates.applyStatus.failed'), result: 'Backend update failed.' }, + last, + started.action_id + ) } catch (error) { const message = error instanceof Error ? error.message : String(error) $backendUpdateApply.set({ diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 69b4cba8a977..81647d34e625 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -1557,6 +1557,11 @@ export interface UpdateReceiptSummary { post_sha: string | null post_version: string | null fleet_states: string[] + /** Post-commit steps a committed (successful) update still owes. */ + followups?: Array<{ step: string; reason: string }> + user_action?: { step: string; reason: string } | null + /** Dashboard action that wrote the receipt; null for a CLI-run update. */ + action_id?: string | null } export interface ActionStatusResponse { diff --git a/apps/shared/src/index.ts b/apps/shared/src/index.ts index 00144fdb1443..b67989b3607a 100644 --- a/apps/shared/src/index.ts +++ b/apps/shared/src/index.ts @@ -174,6 +174,7 @@ export { WINDOWS_GLASS_MIN_BUILD, type WindowsBackgroundMaterial } from './translucency' +export { type UpdateDebt, updateDebt, type UpdateDebtReceipt, type UpdateDebtStep } from './update-debt' export { buildHermesWebSocketUrl, type GatewayAuthMode, diff --git a/apps/shared/src/update-debt.ts b/apps/shared/src/update-debt.ts new file mode 100644 index 000000000000..3e03b162bd11 --- /dev/null +++ b/apps/shared/src/update-debt.ts @@ -0,0 +1,49 @@ +// Owed post-commit work of a dashboard-started `hermes update`, read from the +// receipt summary the action status route attaches. Shared by the web +// dashboard toast and the Desktop backend-apply store so both surfaces name +// the same debt with the same rules as the managed SSH result. + +export interface UpdateDebtStep { + step: string + reason?: string | null +} + +export interface UpdateDebtReceipt { + outcome?: string | null + action_id?: string | null + followups?: UpdateDebtStep[] | null + user_action?: UpdateDebtStep | null +} + +export interface UpdateDebt { + /** Follow-ups a rerun of `hermes update` finishes, as `step (reason); step`; '' when none. */ + followups: string + /** What only the user can do (e.g. re-apply a parked stash) as `step: `; + * '' when none. Never phrase this as "re-run hermes update": a rerun does not restore it. */ + userAction: string +} + +/** The debt of THIS action's committed run, or null when nothing is owed. + * + * Committed is `success` or `partial` (record_user_action turns a committed success into + * partial / exit 1), so the debt is read for every committed outcome, not only exit 0. The + * status route attaches the latest receipt when this action has none, so a receipt is only + * this action's when its `action_id` matches the id the start request returned. */ +export function updateDebt(receipt: UpdateDebtReceipt | null | undefined, actionId: string | undefined): UpdateDebt | null { + if (!receipt || !actionId || receipt.action_id !== actionId) { + return null + } + + if (receipt.outcome !== 'success' && receipt.outcome !== 'partial') { + return null + } + + const followups = (receipt.followups ?? []) + .map(step => (step.reason ? `${step.step} (${step.reason})` : step.step)) + .join('; ') + + const action = receipt.user_action + const userAction = action ? (action.reason ? `${action.step}: ${action.reason}` : action.step) : '' + + return followups || userAction ? { followups, userAction } : null +} diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 4fea926e603c..b7aa264b163c 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -136,8 +136,8 @@ The stage-by-stage contract: `website/docs/developer-guide/cli-internals.md` § `_ZIP_PRESERVED_NESTED` build outputs into the swap. - Restarts are fleet-wide and drain-first, bounded by the home each unit runs on (`update_fleet_scope.py`); another install's runtime is never restarted. -- A provably-stale gateway fails the update (exit 1); every run writes a receipt under - `logs/update_receipts/`. +- Post-commit failures are owed follow-ups, exit 0 (stash left 1, Ctrl-C 130); + receipts in ROOT `logs/update_receipts/`. - Nothing runs pulled code in the pre-pull interpreter. `update_handoff.py` and `update_serve_obligations.py` are a FROZEN compat surface (`tests/compat/old_updater_surface.json`): removing a name bricks releases mid-update. @@ -184,7 +184,7 @@ target; table-driven `_PREFLIGHT_CHECKS`; manifest `/gateway_migration. a re-run resumes from it; a named profile's `gateway install|start|run` refuse without `--force` via `gateway.py::_named_profile_refused_under_multiplexer`, dashboard twin `web_server_gateway.py::multiplexed_profile_refusal`); -`update_cmd_fleet._verify_fleet_after_update` calls `maybe_auto_migrate_after_update` on the success +`update_cmd_fleet_verify._verify_fleet_after_update` calls `maybe_auto_migrate_after_update` on the success path only; `gateway_migrate_guards.py` holds the auto-path-only refusals (table `_AUTO_MIGRATION_GUARDS`: other service domain / UNIX user / HERMES_HOME outside `profiles/` — notices for the explicit command, blockers for the hook) and the `gateway.auto_multiplex_migration` opt-out (#109954). Blockers reuse `GatewayRunner._adapter_credential_fingerprint` and `platform_binds_port`; diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index fafdf3a2249d..fa6e9a4c68c2 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -219,9 +219,8 @@ class LogSnapshot: def _primary_log_path(log_name: str) -> Optional[Path]: """Where *log_name* would live if present. Doesn't check existence.""" - from hermes_cli.logs import LOG_FILES - filename = LOG_FILES.get(log_name) - return (get_hermes_home() / "logs" / filename) if filename else None + from hermes_cli.logs import log_file_path + return log_file_path(log_name) # update/handoff: the ROOT home's, whatever profile is active # Logs written by a client process, invisible to a remote/docker/SSH backend running `debug diff --git a/hermes_cli/logs.py b/hermes_cli/logs.py index 42ea725f67d6..4b98c5389bfb 100644 --- a/hermes_cli/logs.py +++ b/hermes_cli/logs.py @@ -28,7 +28,7 @@ from pathlib import Path from typing import Optional, Sequence -from hermes_constants import get_hermes_home, display_hermes_home +from hermes_constants import display_hermes_home, get_default_hermes_root, get_hermes_home # Known log files (name → filename) LOG_FILES = { @@ -53,6 +53,20 @@ "mcp": "mcp-stderr.log", } +# Written to the ROOT home whatever profile is active: ``hermes update`` mutates the checkout every +# profile shares, and its mirror (main_dashboard) + the Desktop hand-off scripts write /logs. +ROOT_HOME_LOGS = frozenset({"update", "handoff"}) + + +def log_file_path(log_name: str) -> Optional[Path]: + """Where *log_name* lives (doesn't check existence); None for an unknown name.""" + filename = LOG_FILES.get(log_name) + if filename is None: + return None + home = get_default_hermes_root() if log_name in ROOT_HOME_LOGS else get_hermes_home() + return home / "logs" / filename + + # "2026-04-05 22:35:00[,123]" at the start of a line; update.log / # desktop-update-handoff.log stamp with the shell's ISO-8601 "T" shape # ("2026-09-29T21:36:18+08:00", "=== hermes update started 2026-09-29T21:36:18 ==="). @@ -174,7 +188,7 @@ def tail_log( print(f"Unknown log: {log_name!r}. Available: {', '.join(sorted(LOG_FILES))}") sys.exit(1) - log_path = get_hermes_home() / "logs" / filename + log_path = log_file_path(log_name) if not log_path.exists(): print(f"Log file not found: {log_path}") print("(Logs are created when Hermes runs — try 'hermes chat' first)") @@ -313,19 +327,23 @@ def _age_label(mtime: datetime) -> str: def list_logs() -> None: - """Print available log files with sizes.""" + """Print available log files with sizes (plus the root home's update logs under a profile).""" log_dir = get_hermes_home() / "logs" - if not log_dir.exists(): + root_logs = [path for name in sorted(ROOT_HOME_LOGS) + if (path := log_file_path(name)).parent != log_dir and path.is_file()] + if not log_dir.exists() and not root_logs: print(f"No logs directory at {display_hermes_home()}/logs/") return print(f"Log files in {display_hermes_home()}/logs/:\n") found = False - for entry in sorted(log_dir.iterdir()): + entries = sorted(log_dir.iterdir()) if log_dir.exists() else [] + for entry in [*entries, *root_logs]: if entry.is_file() and entry.suffix == ".log": st = entry.stat() age_str = _age_label(datetime.fromtimestamp(st.st_mtime)) - print(f" {entry.name:<25} {_size_label(st.st_size):>8} {age_str}") + label = entry.name if entry.parent == log_dir else f"{entry.name} (root)" + print(f" {label:<25} {_size_label(st.st_size):>8} {age_str}") found = True if not found: diff --git a/hermes_cli/main_dashboard.py b/hermes_cli/main_dashboard.py index 5a5ca614f417..0b884efd9059 100644 --- a/hermes_cli/main_dashboard.py +++ b/hermes_cli/main_dashboard.py @@ -515,10 +515,11 @@ def _install_hangup_protection(gateway_mode: bool = False): # Any failure here is non-fatal; we just skip the wrap. try: - # Late-bound import so tests can monkeypatch - # hermes_cli.config.get_hermes_home to simulate setup failure. - from hermes_cli.config import get_hermes_home as _get_hermes_home - logs_dir = _get_hermes_home() / "logs" + # Late-bound import so tests can monkeypatch it to simulate setup failure. The ROOT + # home, never a sticky profile's: the update mutates the shared checkout, and the + # Desktop and the hand-off scripts read /logs/update.log. + from hermes_constants import get_default_hermes_root as _get_root_home + logs_dir = _get_root_home() / "logs" logs_dir.mkdir(parents=True, exist_ok=True) log_file = open(logs_dir / "update.log", "a", buffering=1, encoding="utf-8") diff --git a/hermes_cli/source_build.py b/hermes_cli/source_build.py index 2559379ab2e9..240170e7d6b6 100644 --- a/hermes_cli/source_build.py +++ b/hermes_cli/source_build.py @@ -99,66 +99,80 @@ def source_frontends(project_root: Path) -> tuple[str, ...]: return tuple(name for name in ("ui-tui", "web") if (project_root / name / "package.json").is_file()) +class ProductBuildError(RuntimeError): + """One or more products failed; every independent product was still attempted.""" + + def __init__(self, failures: list[tuple[str, BaseException]]): + self.failures = failures + super().__init__("; ".join(f"{name}: {_failure_text(exc)}" for name, exc in failures)) + + +def _failure_text(exc: BaseException) -> str: + if isinstance(exc, subprocess.CalledProcessError): + return f"{' '.join(map(str, exc.cmd)) if isinstance(exc.cmd, (list, tuple)) else exc.cmd} exited {exc.returncode}" + return f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__ + + def build_update_products(project_root: Path, *, desktop: bool) -> None: - """Prepare the selected union once; a failed product aborts the update.""" + """Prepare the selected union once, attempting every independent product. + + A failed product never skips the next one (a broken web build must not leave the TUI or the + desktop app stale); the failures are raised together at the end as ``ProductBuildError``. + """ # Both current updates and historical takeover reach this in a fresh target # interpreter, never in the updater's pre-sync import graph. from hermes_cli.main_install_repair import _install_configured_features_missing_deps from hermes_cli.update_stage import publish_stage - _install_configured_features_missing_deps(project_root) + failures: list[tuple[str, BaseException]] = [] + + def attempt(name: str, step) -> bool: + try: + step() + return True + except Exception as exc: # health: allow BLE001 -- collected and raised after the independent steps + print(f" ⚠ {name} failed: {_failure_text(exc)}") + failures.append((name, exc)) + return False + + attempt("feature dependencies", lambda: _install_configured_features_missing_deps(project_root)) frontends = source_frontends(project_root) - if not frontends: - return - env = source_build_env(explicit=True) - workspaces = frontends + (("apps/desktop",) if desktop else ()) - publish_stage("Updating Node dependencies") - prepare_source_dependencies(project_root, workspaces, env=env, explicit=True) - # An update that changed no TUI/web input reuses the receipted output, as the - # launch path already does; recompiling it produces the same bytes. Desktop - # additionally needs the packaged app to name HEAD (its baked stamp carries the - # commit), so it is reused only when HEAD did not move: "Already up to date", - # a retried tail, a takeover re-entry. - if "ui-tui" in frontends: - if source_product_current(project_root, "tui", project_root / "ui-tui/dist"): - print(" ✓ TUI is up to date") - else: - publish_stage("Building the TUI") - build_source_tui(project_root, env=env) - if "web" in frontends: - if source_product_current(project_root, "web", project_root / "hermes_cli/web_dist"): - print(" ✓ Web UI is up to date") - else: - publish_stage("Building the web UI") - build_source_web(project_root, env=env) - if desktop: - from hermes_cli.main_desktop import ( - _packaged_desktop_current_for_head, _refresh_installed_desktop_apps, build_prepared_desktop) - - desktop_dir = project_root / "apps/desktop" - if _packaged_desktop_current_for_head(desktop_dir, project_root): - print(" ✓ Desktop app is up to date") - else: - publish_stage("Building the desktop app") - # The desktop build mutates checkout-scoped node_modules and - # apps/desktop/release; serialize it against a concurrent manual - # `hermes desktop` (#93940). The update path waits rather than exits: - # the in-flight build it queues behind produces the same fresh tree - # this update needs. - from hermes_cli.desktop_build_lock import DesktopBuildLock - - build_lock = DesktopBuildLock(project_root) - build_lock.acquire(wait=True) - try: - build_prepared_desktop( - desktop_dir, source_mode=False, - npm=shutil.which("npm", path=env["PATH"]), env=env, icons=project_root, - ) - finally: - build_lock.release() - # A current release/ can still sit beside a stale installed copy (an earlier - # update rebuilt but never installed); healing must not wait for the next build. - _refresh_installed_desktop_apps(desktop_dir) + if frontends: + desktop_built = False + env: dict = {} + workspaces = frontends + (("apps/desktop",) if desktop else ()) + publish_stage("Updating Node dependencies") + + def node_dependencies() -> None: + # Acquiring npm is part of this step: its failure must be reported like the install's. + env.update(source_build_env(explicit=True)) + prepare_source_dependencies(project_root, workspaces, env=env, explicit=True) + + # Every product compiles from these node_modules: without them there is nothing to build. + if attempt("Node dependencies", node_dependencies): + # An update that changed no TUI/web input reuses the receipted output, as the + # launch path already does; recompiling it produces the same bytes. Desktop + # additionally needs the packaged app to name HEAD (its baked stamp carries the + # commit), so it is reused only when HEAD did not move: "Already up to date", + # a retried tail, a takeover re-entry. + if "ui-tui" in frontends: + if source_product_current(project_root, "tui", project_root / "ui-tui/dist"): + print(" ✓ TUI is up to date") + else: + publish_stage("Building the TUI") + attempt("TUI build", lambda: build_source_tui(project_root, env=env)) + if "web" in frontends: + if source_product_current(project_root, "web", project_root / "hermes_cli/web_dist"): + print(" ✓ Web UI is up to date") + else: + publish_stage("Building the web UI") + attempt("web UI build", lambda: build_source_web(project_root, env=env)) + if desktop: + desktop_built = attempt("desktop app build", lambda: _build_desktop_product(project_root, env, publish_stage)) + if desktop and not desktop_built: + # One whole line the Desktop hand-off scripts match: the follow-up text is truncated + # and names whichever products failed first, so it cannot say if THIS app was rebuilt. + print(f" Desktop app build owed: {failures[-1][0]} failed") # A configured memory provider that no longer ships in core is installed from the # catalog for every profile home sharing this venv (config, data and tool names # unchanged). The update must finish even if the migration blows up. @@ -166,7 +180,7 @@ def build_update_products(project_root: Path, *, desktop: bool) -> None: from hermes_cli.memory_provider_migration import migrate_all_homes migrate_all_homes() - except Exception as exc: + except Exception as exc: # health: allow BLE001 -- post-commit boundary: printed as ⚠, never fails the committed update; every later update/tail re-runs it print(f" ⚠ Memory provider migration skipped: {exc}") # Same for a gateway platform / toolset that left core (Home Assistant): every home that used # it gets its catalog plugin (hermes_cli/left_core_migration.py). @@ -174,8 +188,40 @@ def build_update_products(project_root: Path, *, desktop: bool) -> None: from hermes_cli.left_core_migration import migrate_all_homes as migrate_left_core migrate_left_core() - except Exception as exc: + except Exception as exc: # health: allow BLE001 -- post-commit boundary: printed as ⚠, never fails the committed update; every later update/tail re-runs it print(f" ⚠ Plugin migration skipped: {exc}") + if failures: + raise ProductBuildError(failures) + + +def _build_desktop_product(project_root: Path, env: dict, publish_stage) -> None: + from hermes_cli.main_desktop import ( + _packaged_desktop_current_for_head, _refresh_installed_desktop_apps, build_prepared_desktop) + + desktop_dir = project_root / "apps/desktop" + if _packaged_desktop_current_for_head(desktop_dir, project_root): + print(" ✓ Desktop app is up to date") + else: + publish_stage("Building the desktop app") + # The desktop build mutates checkout-scoped node_modules and + # apps/desktop/release; serialize it against a concurrent manual + # `hermes desktop` (#93940). The update path waits rather than exits: + # the in-flight build it queues behind produces the same fresh tree + # this update needs. + from hermes_cli.desktop_build_lock import DesktopBuildLock + + build_lock = DesktopBuildLock(project_root) + build_lock.acquire(wait=True) + try: + build_prepared_desktop( + desktop_dir, source_mode=False, + npm=shutil.which("npm", path=env["PATH"]), env=env, icons=project_root, + ) + finally: + build_lock.release() + # A current release/ can still sit beside a stale installed copy (an earlier + # update rebuilt but never installed); healing must not wait for the next build. + _refresh_installed_desktop_apps(desktop_dir) if __name__ == "__main__": diff --git a/hermes_cli/source_completion.py b/hermes_cli/source_completion.py index f3df7d9cf479..40cdccfdd317 100644 --- a/hermes_cli/source_completion.py +++ b/hermes_cli/source_completion.py @@ -36,11 +36,13 @@ def complete_source_checkout( pre_update_version: str | None = None, completion_message: str | None = None, announce: str | None = None, + followups: list[tuple[str, str]] | None = None, ) -> bool: """Publish commands, build the products, then run post-build maintenance. - Returns the SQLite runtime verdict: a positive unsafe-runtime probe withholds - success here exactly as it does at the end of an update. + Every step runs even when an earlier one failed; each failure is printed as ``⚠``, + recorded on the open update receipt and appended to ``followups`` as ``(step, reason)``. + Returns True only when every step finished and the SQLite runtime is not unsafe. """ from hermes_cli.update_lock import UpdateLock, describe_holder @@ -65,7 +67,7 @@ def complete_source_checkout( root, desktop=desktop, assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, pre_update_version=pre_update_version, - completion_message=completion_message, announce=announce, + completion_message=completion_message, announce=announce, followups=followups, ) finally: lock.release() @@ -81,6 +83,7 @@ def _complete_locked( pre_update_version: str | None, completion_message: str | None, announce: str | None, + followups: list[tuple[str, str]] | None = None, ) -> bool: """The completion body; callers hold the update lock already.""" from hermes_cli.source_build import build_update_products @@ -95,19 +98,36 @@ def _complete_locked( expose_pm_git(root) except Exception as exc: # noqa: BLE001 — git-less steps below still complete print(f"⚠ Could not provide git for the source completion: {exc}", file=sys.stderr) - publish_launchers(root) - build_update_products(root, desktop=desktop) + owed: list[tuple[str, str]] = [] + + def step(name: str, run) -> None: + # Each step is independent: a failed launcher publish or web build must not skip the + # config migration, the maintenance, or (in an update) the gateway restart after it. + try: + run() + except (Exception, SystemExit) as exc: # health: allow BLE001 -- reported as an owed follow-up + from hermes_cli.update_receipt import record_followup + + reason = str(exc) or type(exc).__name__ + record_followup(name, reason) + owed.append((name, reason)) + + step("launchers", lambda: publish_launchers(root)) + step("build", lambda: build_update_products(root, desktop=desktop)) if announce: print(announce) - complete = _run_post_update_maintenance( + verdict: list[bool] = [] + step("maintenance", lambda: verdict.append(_run_post_update_maintenance( assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, had_desktop_app_before_update=desktop, pre_update_version=pre_update_version, completion_message=completion_message, - ) - if complete: + followups=owed, + ))) + tail_done = not owed + if tail_done: from hermes_cli.source_stamp import write_source_stamp try: @@ -115,7 +135,11 @@ def _complete_locked( except (OSError, ValueError) as exc: print(f"⚠ Source update completed, but the install stamp could not be written: {exc}", file=sys.stderr) - return complete + if followups is not None: + followups.extend(owed) + # The SQLite verdict is not tail work (re-running the tail cannot fix the interpreter); it + # is reported by the maintenance step itself and only withholds an install's success. + return tail_done and all(verdict) def _bootstrap_command(root: Path, argv: list[str]) -> list[str]: @@ -155,11 +179,14 @@ def main(argv: list[str] | None = None) -> int: # pre-handoff release cannot flip during `hermes update`, so its # update ends with the tree at HEAD and nothing built -- lands here # on the next ordinary startup. Same tail as an install, so the two - # states cannot drift apart. - ok = complete_source_checkout( + # states cannot drift apart. Only owed TAIL work keeps it pending: an + # unsafe SQLite runtime is reported, but re-running the tail cannot fix it. + owed: list[tuple[str, str]] = [] + complete_source_checkout( root, desktop=args.desktop, assume_yes=True, - completion_message=None, announce="\n✓ Code updated!", + completion_message=None, announce="\n✓ Code updated!", followups=owed, ) + ok = not owed else: ok = complete_source_checkout( root, desktop=args.desktop, assume_yes=not args.interactive, diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 13509f0674d9..c45e99ffc96c 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -61,11 +61,12 @@ _restart_systemd_gateway_units, _run_pending_fleet_restart, _service_restart_sec, _service_unit_supports_graceful_sigusr1_restart, _surviving_gateway_pids_after_failed_restart, - _systemctl, _systemctl_reset_and_restart, _verify_fleet_after_update, + _systemctl, _systemctl_reset_and_restart, _wait_for_service_active, _warn_gateway_restart_phase_aborted, _warn_incomplete_gateway_fleet_restart, _warn_pending_fleet_restart, _warn_pending_fleet_restart_on_startup, _write_fleet_restart_pending_marker, _write_gateway_update_exit_code) +from hermes_cli.update_cmd_fleet_verify import _verify_fleet_after_update # noqa: F401 from hermes_cli.update_cmd_zip import ( # noqa: F401 _ZIP_PRESERVED_TOP_LEVEL, _ZIP_STAGING_ARTIFACT_SUFFIXES, _abort_zip_update_if_dirty_tree, _atomic_replace_dir, _commit_staged_replacements, _discard_staged, @@ -544,7 +545,9 @@ def _log_only_write(text: str) -> None: log_file = getattr(stream, "_log", None) with suppress(Exception): if log_file is None: - log_path = get_hermes_home() / "logs" / "update.log" + from hermes_constants import get_default_hermes_root + + log_path = get_default_hermes_root() / "logs" / "update.log" # the root home's tee log_path.parent.mkdir(parents=True, exist_ok=True) with log_path.open("a", encoding="utf-8") as fallback: fallback.write(text) @@ -718,6 +721,39 @@ def _source_completion_request(opts, plan, snapshot_id, windows_resume, desktop, } +def _settle_windows_resume(request: dict) -> None: + """This run's post-commit resume attempt is over (the completion child's, or the A6 branch's). + + A failure is already the ``windows_resume`` follow-up, so neither the command's ``finally`` + nor the atexit net may replay it: a replay waits again and its error would fail a committed + update (C3). + """ + import atexit + + from hermes_cli import update_cmd_windows + + atexit.unregister(_m()._resume_windows_gateways_after_update) + atexit.unregister(update_cmd_windows._resume_windows_gateways_after_update) + request["windows_resume_settled"] = True + + +def _resume_paused_gateways_at_exit(token: dict | None, request: dict | None) -> None: + """The command's last resume of gateways it paused; a failure is reported, never raised. + + Raising here would replace the run's own exit (a committed update's 0, or the original + failure) with the restart's error. Skipped when the post-commit attempt already ran. + """ + if not token or not token.get("resume_needed") or (request or {}).get("windows_resume_settled"): + return + try: + _m()._resume_windows_gateways_after_update(token) + except Exception as exc: # health: allow BLE001 -- a restart failure is owed, not the update's status + from hermes_cli.update_receipt import owe_followup + + owe_followup(request["receipt"]["update_id"] if request else None, "windows_resume", + f"Windows gateway recovery failed: {exc}") + + def _complete_source_update(request: dict | None) -> None: # Never "Update complete!" while this run's local patches sit unrestored in the stash (#122557). unrestored = _unrestored_autostash_notice() @@ -736,21 +772,60 @@ def _complete_source_update(request: dict | None) -> None: # A head capture that came back empty must not arm an SHA-less record: it names no code the # fleet can be proven current on, so the warning could never clear (#125952). _write_fleet_restart_pending_marker(expected_sha=request.get("expected_sha") or _current_checkout_sha() or "") - result = run_completion(request) + # Pre-swap module (imported with run_completion above): never the replacement tree's. + from hermes_cli.update_completion import settle_lost_completion + lost = False + try: + result = run_completion(request) + except KeyboardInterrupt as interrupt: + # Ctrl-C after the commit point: the tree is new, so the run is interrupted, never "failed". + _completion_receipt.finalize_interrupted_update_receipt("KeyboardInterrupt: interrupted after the code was updated") + raise SystemExit(130) from interrupt + except Exception as exc: # health: allow BLE001 -- after the commit point a lost completion is owed, never a failed update (C3) + result, lost = settle_lost_completion(request, f"{type(exc).__name__}: {exc}"), True + if not lost and result.get("receipt") is None and result["exit_code"] == 130: + _completion_receipt.finalize_interrupted_update_receipt("the completion was interrupted after the code was updated") + elif not lost and result.get("receipt") is None and result["exit_code"]: + # The child crashed (OOM, SIGKILL) or never answered (run_completion maps every answer + # without a correlated terminal receipt to non-zero): the same owed tail (review P2). + result, lost = settle_lost_completion( + request, result.get("error") or f"the completion process exited {result['exit_code']}"), True _accept_completion_pm_receipt(result.get("pm_receipt"), request["receipt"]["update_id"]) + if result.get("receipt") is not None: + # The child closed the run: drop the stale pre-child context before anything below can + # write it (a later follow-up amends the terminal archive), and let the command boundary + # answer the gateway status from the receipt (C3, review regression 3). + _completion_receipt.adopt_terminal_receipt(result["receipt"]) + current = _completion_receipt._current.get() + if current is not None: + _completion_receipt._current.reset(current.current_token) token = request["windows_resume"] if token is not None and result.get("windows_resume") is not None: resumed = dict(result["windows_resume"]) token.clear() token.update(resumed) - if result.get("receipt") is not None: - current = _completion_receipt._current.get() - if current is not None: - _completion_receipt._current.reset(current.current_token) + _settle_windows_resume(request) + elif token and token.get("resume_needed") and not result["exit_code"]: + # Dependencies owed (A6) or a lost completion: no child resumed paused gateways; this process can. + try: + _m()._resume_windows_gateways_after_update(token) + except Exception as exc: # health: allow BLE001 -- the code is committed (C3) + from hermes_cli.update_receipt import owe_followup + + owe_followup(request["receipt"]["update_id"], "windows_resume", f"Windows gateway recovery failed: {exc}") + _settle_windows_resume(request) if result["exit_code"]: raise SystemExit(result["exit_code"]) - if adopt_retired_channel(request): - print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") + if lost: + return # a retired channel is adopted after a verified completion only; the next update adopts it + try: + if adopt_retired_channel(request): + print(f"→ Source subscription moved to {request['channel_retirement']['destination']}") + except Exception as exc: # health: allow BLE001 -- the code is committed (C3); the next update re-adopts + from hermes_cli.update_receipt import owe_followup + + owe_followup(request["receipt"]["update_id"], "channel_adoption", str(exc) or type(exc).__name__, + retry="the next `hermes update` adopts it again") def _reconcile_diverged_checkout(git_cmd, branch: str, pre_pull_sha, *, target_ref=None) -> None: @@ -1510,8 +1585,7 @@ def _cmd_update_impl(args, gateway_mode: bool): target_sha=release_sha, completion_request=completion_request, **({"target_repository": target_repository} if target_repository else {})) finally: - if _windows_gateway_resume and _windows_gateway_resume.get("resume_needed"): - _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + _resume_paused_gateways_at_exit(_windows_gateway_resume, completion_request) return @@ -1605,5 +1679,4 @@ def _cmd_update_impl(args, gateway_mode: bool): e, args, gateway_mode, had_desktop_app_before_update, target_sha=release_sha, target_repository=target_repository, completion_request=completion_request) finally: - if _windows_gateway_resume and _windows_gateway_resume.get("resume_needed"): - _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + _resume_paused_gateways_at_exit(_windows_gateway_resume, completion_request) diff --git a/hermes_cli/update_cmd_config.py b/hermes_cli/update_cmd_config.py index fd99c81a6b19..554165d9f1f4 100644 --- a/hermes_cli/update_cmd_config.py +++ b/hermes_cli/update_cmd_config.py @@ -182,6 +182,7 @@ def _check_and_apply_config_migration( print(" Run 'hermes config migrate' to check manually.") return + owed_error = None has_new_options = bool(missing_env or missing_config) version_bump_only = not has_new_options and current_ver < latest_ver needs_migration = has_new_options or current_ver < latest_ver @@ -205,6 +206,11 @@ def _check_and_apply_config_migration( except Exception as _mig_err: print(f" ⚠️ Config format update failed: {_mig_err}") print(" Run 'hermes config migrate' to retry.") + # Not a silent success: raised after the sibling migration and snapshot safety nets + # below (independent of the active write, and the snapshot id is only in hand now), so + # the caller's owed-step guard records ``config_migration`` and keeps the tail pending. + owed_error = RuntimeError(f"config format v{current_ver} → v{latest_ver} was not written: {_mig_err}") + owed_error.__cause__ = _mig_err elif needs_migration: print() # Show WHAT changed, not just a count, for an informed yes/no. @@ -241,6 +247,8 @@ def _check_and_apply_config_migration( print(f" ✓ Profile '{_name}': config format updated (v{_from_ver} → v{_to_ver})") _restore_snapshot_safety_nets(pre_update_snapshot_id) + if owed_error is not None: + raise owed_error # {profile: snapshot_id} from this run's pre-update backup, consumed by the per-profile diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index aec6e660123e..f37e91eb67a1 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -30,11 +30,6 @@ _FRESH_RESTART_SUPERVISORS = frozenset({"systemd", "launchd", "service", "s6"}) -# A supervisor can report a restarted unit active before the gateway finishes its -# bootstrap and publishes ``gateway_state.json``. Keep the readiness poll bounded, -# but allow the default systemd startup budget plus status-publication slack. -_FLEET_PROBE_SETTLE_TIMEOUT_SECONDS = 120.0 - _SYSTEMD_SCOPES = (("user", ["systemctl", "--user"]), ("system", ["systemctl"])) _LIST_GATEWAY_UNITS = [ "list-units", "hermes-gateway*", "hermes-serve*", "hermes-dashboard*", @@ -44,6 +39,11 @@ def _write_gateway_update_exit_code(ok: bool) -> None: from hermes_cli.update_cmd import get_hermes_home + from hermes_cli.update_receipt import committed_success + + # A run whose receipt already closed as a success is never reported failed (C3, review + # regression 3): the command boundary's catch-alls call this with False. + ok = ok or committed_success() path = get_hermes_home() / ".update_exit_code" with suppress(OSError): path.write_text("0" if ok else "1", encoding="utf-8") @@ -264,6 +264,7 @@ def _receipt_owed_gateways(receipt: dict, pending_manual: list[dict]) -> set[tup plan = receipt.get("plan") or {} entries: list[tuple[object, str | None]] = [(entry, None) for entry in plan.get("runtimes") or []] entries.extend((entry, None) for entry in receipt.get("pending_manual_serves") or []) + entries.extend((entry, None) for entry in receipt.get("carried_manual_serves") or []) entries.extend((entry, "gateway") for entry in receipt.get("fleet") or []) owed: set[tuple[str, str]] = set() unverified = False @@ -329,6 +330,11 @@ def _live_fleet_covers_receipt(expected_sha: str | None, receipt: dict, owed: se return False +# A gateway the restart phase stopped without knowing its profile: no live row can be matched to it, +# so only a live, current gateway of this checkout settles it, never an empty host (""=no profile). +_UNMAPPED_GATEWAY = ("gateway", "") + + def _marker_owed_gateways(inventory: object) -> set[tuple[str, str]] | None: """The ``("gateway", profile)`` set a marker's inventory owes; None when it recorded none. @@ -349,6 +355,9 @@ def _marker_owed_gateways(inventory: object) -> set[tuple[str, str]] | None: raise ValueError("fleet-restart inventory row is not an object") if runtime_outside_gateway_evidence(runtime): continue + if runtime.get("kind") == "gateway" and runtime.get("stopped_unmapped"): + owed.add(_UNMAPPED_GATEWAY) + continue profile = runtime.get("profile") if runtime.get("kind") != "gateway" or not isinstance(profile, str) or not profile.strip() or profile == "unknown": raise ValueError("fleet-restart inventory row is not an identified gateway") @@ -401,7 +410,7 @@ def _marker_only_restart_obsolete() -> bool: undischargeable on every host that runs a dashboard. A manual-serve row still needs its durable handoff (``defer_manual_serve``), and an unclassified backend stays fail-closed. Discharging here strands nobody: the same row is still accounted at update time by - ``update_inventory.report_unaccounted_runtimes``, which prints it and exits 1 when the restart + ``update_inventory.report_unaccounted_runtimes``, which prints it and records an owed restart when the restart phase never touched it — this marker only stops re-warning about it on every later startup. """ from hermes_cli.update_cmd_fleet_checkout import checkout_contains @@ -451,8 +460,13 @@ def _marker_only_restart_obsolete() -> bool: continue if row.get("state") != "current" or str(row.get("code_sha")) != target_sha: return False # stale / down / unknown-identity row still owes the restart - if owed is not None and not owed <= covered: + if owed is not None and not owed - {_UNMAPPED_GATEWAY} <= covered: return False # A gateway this marker owns is absent (down) or unidentifiable. + if owed and _UNMAPPED_GATEWAY in owed and not (_fleet_covered_gateways( + [row for row in fleet if not row_is_external(row)]) or set()) - owed: + # An unmapped stopped gateway has no identity to match, so only a local gateway that no + # mapped owed one explains can be its successor; another checkout's gateway never is. + return False _clear_fleet_restart_pending_marker() logger.debug( "Fleet-restart-pending marker discharged: %d gateway(s) already serve %s", @@ -1648,7 +1662,7 @@ def _recover_after_restart_phase_abort( from hermes_cli.update_abort_recovery import _owed_stale_serve_rows from hermes_cli.update_cmd import ( _abort_recovery_is_complete, _recover_gateway_restart_after_abort, _surviving_pre_update_serve_runtimes, - _warn_stale_serve_runtimes, _write_gateway_update_exit_code, + _warn_stale_serve_runtimes, ) logger.debug("Gateway restart during update failed: %s", e) out.phase_errors.append(str(e)) @@ -1708,11 +1722,10 @@ def _recover_after_restart_phase_abort( or _owed_stale_serve_rows(_stale_runtime_rows) or _serve_units_failed ): + # Owed restart (C3): verification records the follow-up; the /update marker stays committed. out.incomplete = True _warn_gateway_restart_phase_aborted(e, _surviving) _warn_stale_serve_runtimes(_stale_runtime_rows) - if gateway_mode: - _write_gateway_update_exit_code(False) out.record_receipt(phase_error=str(e), fresh_recovery=_recovery_result) @@ -1731,7 +1744,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): Never raises: a phase abort runs fresh-child recovery and fails closed unless every planned gateway is verifiably covered. """ - from hermes_cli.update_cmd import _m, _write_gateway_update_exit_code + from hermes_cli.update_cmd import _m # All bookkeeping is declared before the try so abort recovery and fleet reconciliation # can read it even if the phase raises early. ``pre_restart_gateway_pids`` stays empty # until we are about to stop/drain, so an early exception has nothing to fail closed on, @@ -1791,9 +1804,8 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): _restart_manual_gateways(out, _drain_budget) if out.failed_or_stale_units: + # Owed restart (C3): verification records the follow-up; the /update marker stays committed. out.incomplete = True - if gateway_mode: - _write_gateway_update_exit_code(False) _warn_incomplete_gateway_fleet_restart(out.failed_or_stale_units) out.record_receipt() _force_kill_stuck_gateways(out.killed_pids) @@ -1812,265 +1824,6 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): return out -def _print_legacy_units_warning() -> None: - """Legacy hermes.service fights hermes-gateway.service over the bot token; warn on - every update until migrated.""" - from hermes_cli.gateway import (has_legacy_hermes_units, _find_legacy_hermes_units, supports_systemd_services) - if not (supports_systemd_services() and has_legacy_hermes_units()): - return - print() - print("⚠ Legacy Hermes gateway unit(s) detected:") - for name, path, is_sys in _find_legacy_hermes_units(): - scope = "system" if is_sys else "user" - print(f" {path} ({scope} scope)") - print() - print(" These pre-rename units (hermes.service) fight the current") - print(" hermes-gateway.service for the bot token and cause SIGTERM") - print(" flap loops. Remove them with:") - print() - print(" hermes gateway migrate-legacy") - print() - print(" (add `sudo` if any are in system scope)") - - -def _collect_fleet_snapshot(restart, rows_expected: bool) -> list: - """Fleet version rows, polled over a bounded settle window when runtimes are expected. - - Gateways need time to rewrite gateway_state.json; Windows resumes DETACHED (~10s boot), - so a single 2s sleep reported "no rows" on healthy resumes. A "down" row may be a - detached replacement still booting: poll until none remain or the deadline passes. - Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (exit 1) - instead of no row at all. An ``unknown`` row whose pid is NOT a pre-restart pid is a successor - that has not published its code identity yet (a relaunched gateway can sit ~10s between process - start and its first runtime-status write, #112634) — keep polling; at the deadline it is flagged - ``identity_pending`` so the matrix does not call it a pre-stamping gateway. - """ - from hermes_cli.update_receipt import collect_fleet_versions - pending = getattr(restart, "self_restart_pending_pids", None) or None - if not rows_expected: - return collect_fleet_versions( - pre_restart_pids=restart.pre_restart_gateway_pids, self_restart_pending=pending) - pre_pids = restart.pre_restart_gateway_pids - _fleet_deadline = _time.monotonic() + _FLEET_PROBE_SETTLE_TIMEOUT_SECONDS - while True: - _time.sleep(2.0) - snapshot = collect_fleet_versions(pre_restart_pids=pre_pids, self_restart_pending=pending) - unstamped = [row for row in snapshot if _fleet_row_identity_pending(row, pre_pids)] - if snapshot and not unstamped and not any(row.get("state") == "down" for row in snapshot): - return snapshot - if _time.monotonic() >= _fleet_deadline or _restarted_units_gone( - getattr(restart, "restarted_scoped_units", ())): - for row in unstamped: - row["identity_pending"] = True - return snapshot - - -def _fleet_row_identity_pending(row: dict, pre_restart_pids) -> bool: - """An ``unknown`` row with no sha from a pid that did not exist at update start: a relaunched - gateway still booting, not a gateway that predates version stamping. A surviving pre-restart pid - (or no pid snapshot at all) is settled as-is — waiting cannot change what it publishes.""" - if row.get("state") != "unknown" or row.get("code_sha"): - return False - if pre_restart_pids is None: - return False - return row.get("pid") not in {int(p) for p in pre_restart_pids if isinstance(p, int)} - - -def _restarted_units_gone(scoped_units) -> bool: - """True when every restarted systemd unit is LOADED in its scope and neither active nor - activating: the successor died, nothing will publish a state stamp, so the settle poll should fail - closed now instead of at the deadline. Anything inconclusive keeps waiting: no units, systemctl - missing/slow, or ``LoadState=not-found`` — a unit name asked in a scope that does not own it - answers ``inactive`` exactly like a dead unit (#112466), so only a loaded unit can prove death.""" - if not scoped_units: - return False - scope_cmds = dict(_SYSTEMD_SCOPES) - for scoped in scoped_units: - scope, _, name = scoped.partition("/") - try: - stdout = _systemctl(scope_cmds[scope] + ["show", "-p", "LoadState,ActiveState", name], timeout=5).stdout - except (KeyError, FileNotFoundError, subprocess.TimeoutExpired): - return False - props = dict(line.split("=", 1) for line in stdout.splitlines() if "=" in line) - if props.get("LoadState") != "loaded": - return False - if props.get("ActiveState") in ("active", "activating", "reloading"): - return False - return True - - -def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: - """Profile -> gateway PIDs alive after the restart phase, from the fleet snapshot. - - Incarnation evidence for gateway reconciliation (``match_runtime_outcomes``'s - ``live_gateway_pids``): the plan identifies a gateway by the profile it SERVES while the restart - bookkeeping names the SERVICE, and a service can serve a profile its name does not encode — the - profile-scoped name matcher then never credits the planned runtime. A ``down`` row reports the - PRE-restart PID (nothing replaced it), so it is never a successor; rows without a usable - profile/PID are skipped. - """ - live: dict = {} - for row in fleet_rows: - pid, profile = row.get("pid"), row.get("profile") - if profile and isinstance(pid, int) and row.get("state") != "down": - live.setdefault(str(profile), set()).add(pid) - return live - - -def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, update_complete): - """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve - probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. - - Exits 1 (leaving ``fleet_restart_pending`` for the next catch-up) when any gateway - may still be stale; otherwise clears the marker. A failed SQLite verdict also - exits 1, without retaining a fulfilled fleet-restart obligation. - """ - from hermes_cli.update_cmd import ( - _m, _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes, - ) - from hermes_cli.update_cmd_maint import _refresh_dashboard_after_update - with _best_effort('Legacy unit check during update failed: %s'): - _print_legacy_units_warning() - - # Restart a managed dashboard via systemd or stop stale manual ones (raw-killing - # a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead). - # Already-restarted units aren't redone. - # A dashboard it stopped and could not bring back is a promised restart that did not happen. - _dashboards_down = _refresh_dashboard_after_update(already_restarted_units=set(restart.restarted_services)) - if _dashboards_down: - restart.incomplete = True - - # Success-path twin of the abort-recovery probe: the restart phase only touches - # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER - # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed - # reconciliation (survivor → exit 1); ``None`` = probe failed, stays fail-closed. - # Check if any pre-update serve/dashboard runtimes survived on pre-update code generations (#100479). - # This is the SUCCESS-path twin of the abort-recovery probe above: the restart phase only restarts - # units, so an sshd-spawned `serve --isolated` or a manual `hermes serve` (no unit) is left running its - # pre-update sys.modules graph — and its cron ticker keeps firing agent jobs that ImportError on every - # symbol added in the pulled range. The rows also feed the plan-vs-execution reconciliation below, so a - # survivor is escalated (exit 1) instead of merely printed. - _stale_serve_rows: "list | None" = None - with _best_effort('Failed to check for surviving serve runtimes: %s'): - _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) - if _stale_serve_rows: - _warn_stale_serve_runtimes(_stale_serve_rows) - - print() - print("Tip: You can now select a provider and model:") - print(" hermes model # Select provider and model") - - # Compare every live gateway's stamped code_sha against the fresh checkout - # instead of assuming the restart phase worked. - # Phase 1 (#91277): post-update fleet version verification. - _fleet_snapshot: list = [] - with _best_effort('Fleet version verification failed: %s'): - from hermes_cli.update_receipt import print_fleet_version_matrix - # Cross-platform "rows expected" signal: (restarted_services or killed_pids) - # never fires on Windows (pause/resume populates neither), so a healthy - # resumed gateway yielded zero rows and exit 0. - # See #93406. - # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design, - # so it must not count as an expected one — otherwise an update whose only live gateways - # were unmapped exits 1 with "no rows" after correctly stopping them. - _pre_restart, _killed = restart.fleet_probe_signals() - _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( - _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed, - ) - _fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected) - if print_fleet_version_matrix(_fleet_snapshot): - restart.incomplete = True - # A proven-stale survivor must not keep running (its ticker yields every tick and - # nothing else restarts it, #117275): hand it to the drain-first restart path. - from hermes_cli.update_cmd_stale_survivors import signal_stale_fleet_survivors - signal_stale_fleet_survivors(_fleet_snapshot, restart, _gateway_drain_budget()) - elif not _fleet_snapshot and _fleet_rows_expected: - # collect_fleet_versions() swallows every failure, so zero rows with - # expected runtimes is indistinguishable from health — fail (partial, exit 1). - print( - # Fleet probe returned zero rows even though at least one gateway runtime was (or may have - # been) live pre-update — POSIX restart bookkeeping, the pre-restart PID snapshot, the - # pre-update plan inventory, or the Windows pause/resume token all count as that signal. - # Every failure path inside collect_fleet_versions() is swallowed via logger.debug(), so an - # empty list is indistinguishable from a healthy fleet in the current output. Treat it as - # verification failure so the receipt records "partial" and the exit code is 1 (#93406). - "\n⚠ Fleet version check returned no rows even though" - " gateway runtimes were expected — verification incomplete." - ) - restart.incomplete = True - - # Every runtime the PLAN saw must appear in restart bookkeeping; an - # unaccounted one is a silent miss and escalates like a STALE/DOWN row. - with _best_effort('Runtime-outcome reconciliation failed: %s'): - # An unaccounted runtime is the silent-miss class (a platform branch re-discovered its own targets - # and skipped one the inventory knew about) — escalate it exactly like a STALE/DOWN fleet row. See - # #91277. - if _pre_update_plan is not None and _pre_update_plan.runtimes: - from hermes_cli.update_inventory import (match_runtime_outcomes, report_unaccounted_runtimes) - from hermes_cli.update_receipt import row_is_external - # Gateway incarnation evidence, from the post-restart fleet snapshot collected above: a - # service can serve a profile its own name does not encode (root-home launchd label + - # sticky active profile, hashed custom HERMES_HOME), so the bookkeeping's service names - # alone cannot credit the planned runtime. A profile the probe produced no row for simply - # has no evidence — that runtime stays on the name-matching path (and logs it). - _live_gateway_pids = _live_gateway_pids_from_fleet(_fleet_snapshot) - _runtime_outcomes = match_runtime_outcomes( - _pre_update_plan, - restarted_services=restart.restarted_services, - relaunched_profiles=restart.relaunched_profiles, - externally_supervised_profiles=restart.externally_supervised_profiles, - killed_pids=restart.killed_pids, - failed_units=restart.failed_or_stale_units, - # Serve/dashboard reconcile by incarnation liveness, not unit names. - # See #100479. - stale_serve_pids=( - {row.get("pid") for row in _stale_serve_rows} - if _stale_serve_rows is not None - else None - ), - failed_respawn_pids=_dashboards_down, - # A symlinked profile served by another install's checkout (#120240). - external_gateway_pids={row.get("pid") for row in _fleet_snapshot if row_is_external(row)}, - live_gateway_pids=_live_gateway_pids, - ) - from dataclasses import asdict - from hermes_cli.update_serve_obligations import defer_manual_serve - - for runtime, outcome in zip(_pre_update_plan.runtimes, _runtime_outcomes): - if outcome["outcome"] == "unaccounted" and defer_manual_serve(asdict(runtime), require_alive=True): - outcome["outcome"] = "deferred" - if report_unaccounted_runtimes(_runtime_outcomes): - restart.incomplete = True - with suppress(Exception): - import hermes_cli.update_receipt as _ur - _active = _ur._current.get() - if _active is not None: - _active.data["runtime_outcomes"] = _runtime_outcomes - - with _best_effort('Update receipt finalize failed: %s'): - from hermes_cli.update_receipt import finalize_update_receipt - _receipt_path = finalize_update_receipt( - "partial" if restart.incomplete or not update_complete else "success", - fleet=_fleet_snapshot, - ) - if _receipt_path is not None: - logger.info("Update receipt written: %s", _receipt_path) - - if restart.incomplete: - # Code updated but a gateway may still run stale modules: fail so automation - # doesn't treat the fleet as healthy; leave the pending marker for catch-up. - sys.exit(1) - _clear_fleet_restart_pending_marker() - if not update_complete: - # Fleet caught up, but the independently checked SQLite runtime is unsafe. - sys.exit(1) - # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing - # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later. - with _best_effort('Multiplex auto-migration after update failed: %s'): - from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update - maybe_auto_migrate_after_update() - - def _restart_phase_failure_is_incomplete(surviving, pre_restart_pids) -> bool: """Whether an escaped restart-phase exception must fail the update. diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py new file mode 100644 index 000000000000..6097bce476dd --- /dev/null +++ b/hermes_cli/update_cmd_fleet_verify.py @@ -0,0 +1,363 @@ +"""Post-restart fleet verification for ``hermes update`` (``update_cmd_fleet`` sibling). + +After the restart phase: compare every live gateway's stamped code against the checkout, reconcile +the pre-update plan against what the restart did, keep the restart owed (a ``gateway_restart`` +follow-up, never a failed update — contract C3) or clear the obligation, and finalize the receipt. +Restart-phase helpers stay in ``update_cmd_fleet`` and are read through it so their patches apply. +""" + +import json +import logging +import subprocess +import time as _time +from contextlib import suppress + +from hermes_cli.update_cmd_common import _best_effort + +# Log-record parity with the origin module. +logger = logging.getLogger("hermes_cli.update_cmd") + +# A supervisor can report a restarted unit active before the gateway finishes its +# bootstrap and publishes ``gateway_state.json``. Keep the readiness poll bounded, +# but allow the default systemd startup budget plus status-publication slack. +_FLEET_PROBE_SETTLE_TIMEOUT_SECONDS = 120.0 + + +def _print_legacy_units_warning() -> None: + """Legacy hermes.service fights hermes-gateway.service over the bot token; warn on + every update until migrated.""" + from hermes_cli.gateway import (has_legacy_hermes_units, _find_legacy_hermes_units, supports_systemd_services) + if not (supports_systemd_services() and has_legacy_hermes_units()): + return + print() + print("⚠ Legacy Hermes gateway unit(s) detected:") + for name, path, is_sys in _find_legacy_hermes_units(): + scope = "system" if is_sys else "user" + print(f" {path} ({scope} scope)") + print() + print(" These pre-rename units (hermes.service) fight the current") + print(" hermes-gateway.service for the bot token and cause SIGTERM") + print(" flap loops. Remove them with:") + print() + print(" hermes gateway migrate-legacy") + print() + print(" (add `sudo` if any are in system scope)") + + +def _collect_fleet_snapshot(restart, rows_expected: bool) -> list: + """Fleet version rows, polled over a bounded settle window when runtimes are expected. + + Gateways need time to rewrite gateway_state.json; Windows resumes DETACHED (~10s boot), + so a single 2s sleep reported "no rows" on healthy resumes. A "down" row may be a + detached replacement still booting: poll until none remain or the deadline passes. + Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (owed restart) + instead of no row at all. An ``unknown`` row whose pid is NOT a pre-restart pid is a successor + that has not published its code identity yet (a relaunched gateway can sit ~10s between process + start and its first runtime-status write, #112634) — keep polling; at the deadline it is flagged + ``identity_pending`` so the matrix does not call it a pre-stamping gateway. + """ + from hermes_cli.update_receipt import collect_fleet_versions + pending = getattr(restart, "self_restart_pending_pids", None) or None + if not rows_expected: + return collect_fleet_versions( + pre_restart_pids=restart.pre_restart_gateway_pids, self_restart_pending=pending) + pre_pids = restart.pre_restart_gateway_pids + _fleet_deadline = _time.monotonic() + _FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + while True: + _time.sleep(2.0) + snapshot = collect_fleet_versions(pre_restart_pids=pre_pids, self_restart_pending=pending) + unstamped = [row for row in snapshot if _fleet_row_identity_pending(row, pre_pids)] + if snapshot and not unstamped and not any(row.get("state") == "down" for row in snapshot): + return snapshot + if _time.monotonic() >= _fleet_deadline or _restarted_units_gone( + getattr(restart, "restarted_scoped_units", ())): + for row in unstamped: + row["identity_pending"] = True + return snapshot + + +def _fleet_row_identity_pending(row: dict, pre_restart_pids) -> bool: + """An ``unknown`` row with no sha from a pid that did not exist at update start: a relaunched + gateway still booting, not a gateway that predates version stamping. A surviving pre-restart pid + (or no pid snapshot at all) is settled as-is — waiting cannot change what it publishes.""" + if row.get("state") != "unknown" or row.get("code_sha"): + return False + if pre_restart_pids is None: + return False + return row.get("pid") not in {int(p) for p in pre_restart_pids if isinstance(p, int)} + + +def _restarted_units_gone(scoped_units) -> bool: + """True when every restarted systemd unit is LOADED in its scope and neither active nor + activating: the successor died, nothing will publish a state stamp, so the settle poll should fail + closed now instead of at the deadline. Anything inconclusive keeps waiting: no units, systemctl + missing/slow, or ``LoadState=not-found`` — a unit name asked in a scope that does not own it + answers ``inactive`` exactly like a dead unit (#112466), so only a loaded unit can prove death.""" + from hermes_cli import update_cmd_fleet as fleet + if not scoped_units: + return False + scope_cmds = dict(fleet._SYSTEMD_SCOPES) + for scoped in scoped_units: + scope, _, name = scoped.partition("/") + try: + stdout = fleet._systemctl(scope_cmds[scope] + ["show", "-p", "LoadState,ActiveState", name], timeout=5).stdout + except (KeyError, FileNotFoundError, subprocess.TimeoutExpired): + return False + props = dict(line.split("=", 1) for line in stdout.splitlines() if "=" in line) + if props.get("LoadState") != "loaded": + return False + if props.get("ActiveState") in ("active", "activating", "reloading"): + return False + return True + + +def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict: + """Profile -> gateway PIDs alive after the restart phase, from the fleet snapshot. + + Incarnation evidence for gateway reconciliation (``match_runtime_outcomes``'s + ``live_gateway_pids``): the plan identifies a gateway by the profile it SERVES while the restart + bookkeeping names the SERVICE, and a service can serve a profile its name does not encode — the + profile-scoped name matcher then never credits the planned runtime. A ``down`` row reports the + PRE-restart PID (nothing replaced it), so it is never a successor; rows without a usable + profile/PID are skipped. + """ + live: dict = {} + for row in fleet_rows: + pid, profile = row.get("pid"), row.get("profile") + if profile and isinstance(pid, int) and row.get("state") != "down": + live.setdefault(str(profile), set()).add(pid) + return live + + +def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None: + """Name the pre-update gateways on the obligation once their restart is owed. + + The pull arms the obligation before the restart phase knows what it owes. An inventory-less + record on a host whose gateway then died at boot is settled by the gateway-less discharge + (nothing live, nothing named), which would silence the owed-restart warning that replaces a + failed update under contract C3. A named inventory keeps it until those gateways run HEAD. + A gateway stopped with no known profile is recorded as an unmapped row for the same reason: it + has no ``gateway_state.json`` the gateway-less probe could see, so absence would settle it. + A readable inventory (a same-SHA catch-up keeps the standing record) only GROWS: rows this run + newly owes are appended, so a mapped successor cannot discharge a newly stopped unmapped one. + """ + from hermes_cli import update_cmd_fleet as fleet + armed = fleet._fleet_restart_obligation_armed() + fields = fleet._obligation_fields() if armed else {} + if fields is None: + return # unreadable terms stay fail-closed + recorded: list = [] + if fields.get("inventory") not in (None, "", "null"): + try: + inventory = json.loads(fields["inventory"]) + fleet._marker_owed_gateways(inventory) # validates; malformed/unsupported stays untouched + except ValueError: + return + recorded = inventory["runtimes"] + from dataclasses import asdict + rows = [ + asdict(runtime) for runtime in getattr(plan, "runtimes", ()) or () + if getattr(runtime, "kind", None) == "gateway" + and isinstance(getattr(runtime, "profile", None), str) + and runtime.profile.strip() and runtime.profile != "unknown" + ] + [{"kind": "gateway", "profile": None, "pid": pid, "stopped_unmapped": True} + for pid in sorted(stopped_unmapped_pids)] + rows = [row for row in rows if row not in recorded] + if rows: + # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the + # restart is owed. The SHA is the code the fleet must be proven current on. + sha = fields.get("expected_sha", "") if armed else (fleet._current_checkout_sha() or "") + fleet._write_fleet_restart_pending_marker(expected_sha=sha, runtimes=recorded + rows) + + +def _named_gateways_still_owed() -> bool: + """True when the armed obligation NAMES gateways (recorded by an earlier owed restart) and the + live fleet still does not prove them current. An inventory-less record keeps its old rules.""" + from hermes_cli import update_cmd_fleet as fleet + if not fleet._fleet_restart_obligation_armed(): + return False + fields = fleet._obligation_fields() + if fields is None or fields.get("inventory") in (None, "", "null"): + return False + return not fleet._marker_only_restart_obsolete() + + +def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, update_complete): + """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve + probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. + + Never fails the committed update (contract C3): when any gateway may still be stale it + records a ``gateway_restart`` follow-up and leaves ``fleet_restart_pending`` armed for the + next catch-up and the startup warning; otherwise clears the marker. + """ + from hermes_cli import update_cmd_fleet as fleet + from hermes_cli.update_cmd import ( + _m, _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes, + ) + from hermes_cli.update_cmd_maint import _refresh_dashboard_after_update + with _best_effort('Legacy unit check during update failed: %s'): + _print_legacy_units_warning() + + # Restart a managed dashboard via systemd or stop stale manual ones (raw-killing + # a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead). + # Already-restarted units aren't redone. + # A dashboard it stopped and could not bring back is a promised restart that did not happen. + _dashboards_down = _refresh_dashboard_after_update(already_restarted_units=set(restart.restarted_services)) + if _dashboards_down: + restart.incomplete = True + + # Success-path twin of the abort-recovery probe: the restart phase only touches + # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER + # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed + # reconciliation (survivor → owed restart); ``None`` = probe failed, stays fail-closed. + # Check if any pre-update serve/dashboard runtimes survived on pre-update code generations (#100479). + # This is the SUCCESS-path twin of the abort-recovery probe above: the restart phase only restarts + # units, so an sshd-spawned `serve --isolated` or a manual `hermes serve` (no unit) is left running its + # pre-update sys.modules graph — and its cron ticker keeps firing agent jobs that ImportError on every + # symbol added in the pulled range. The rows also feed the plan-vs-execution reconciliation below, so a + # survivor is escalated (an owed gateway_restart follow-up) instead of merely printed. + _stale_serve_rows: "list | None" = None + with _best_effort('Failed to check for surviving serve runtimes: %s'): + _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) + if _stale_serve_rows: + _warn_stale_serve_runtimes(_stale_serve_rows) + + print() + print("Tip: You can now select a provider and model:") + print(" hermes model # Select provider and model") + + # Compare every live gateway's stamped code_sha against the fresh checkout + # instead of assuming the restart phase worked. + # Phase 1 (#91277): post-update fleet version verification. + _fleet_snapshot: list = [] + with _best_effort('Fleet version verification failed: %s'): + from hermes_cli.update_receipt import print_fleet_version_matrix + # Cross-platform "rows expected" signal: (restarted_services or killed_pids) + # never fires on Windows (pause/resume populates neither), so a healthy + # resumed gateway yielded zero rows and exit 0. + # See #93406. + # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design, + # so it must not count as an expected one — otherwise an update whose only live gateways + # were unmapped reports "no rows" after correctly stopping them. + _pre_restart, _killed = restart.fleet_probe_signals() + _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( + _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed, + ) + _fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected) + if print_fleet_version_matrix(_fleet_snapshot): + restart.incomplete = True + # A proven-stale survivor must not keep running (its ticker yields every tick and + # nothing else restarts it, #117275): hand it to the drain-first restart path. + from hermes_cli.update_cmd_stale_survivors import signal_stale_fleet_survivors + signal_stale_fleet_survivors(_fleet_snapshot, restart, fleet._gateway_drain_budget()) + elif not _fleet_snapshot and _fleet_rows_expected: + # collect_fleet_versions() swallows every failure, so zero rows with + # expected runtimes is indistinguishable from health — keep the restart owed. + print( + # Fleet probe returned zero rows even though at least one gateway runtime was (or may have + # been) live pre-update — POSIX restart bookkeeping, the pre-restart PID snapshot, the + # pre-update plan inventory, or the Windows pause/resume token all count as that signal. + # Every failure path inside collect_fleet_versions() is swallowed via logger.debug(), so an + # empty list is indistinguishable from a healthy fleet in the current output. Treat it as + # verification failure so the receipt carries a gateway_restart follow-up (#93406). + "\n⚠ Fleet version check returned no rows even though" + " gateway runtimes were expected — verification incomplete." + ) + restart.incomplete = True + + # Every runtime the PLAN saw must appear in restart bookkeeping; an + # unaccounted one is a silent miss and escalates like a STALE/DOWN row. + with _best_effort('Runtime-outcome reconciliation failed: %s'): + # An unaccounted runtime is the silent-miss class (a platform branch re-discovered its own targets + # and skipped one the inventory knew about) — escalate it exactly like a STALE/DOWN fleet row. See + # #91277. + if _pre_update_plan is not None and _pre_update_plan.runtimes: + from hermes_cli.update_inventory import (match_runtime_outcomes, report_unaccounted_runtimes) + from hermes_cli.update_receipt import row_is_external + # Gateway incarnation evidence, from the post-restart fleet snapshot collected above: a + # service can serve a profile its own name does not encode (root-home launchd label + + # sticky active profile, hashed custom HERMES_HOME), so the bookkeeping's service names + # alone cannot credit the planned runtime. A profile the probe produced no row for simply + # has no evidence — that runtime stays on the name-matching path (and logs it). + _live_gateway_pids = _live_gateway_pids_from_fleet(_fleet_snapshot) + _runtime_outcomes = match_runtime_outcomes( + _pre_update_plan, + restarted_services=restart.restarted_services, + relaunched_profiles=restart.relaunched_profiles, + externally_supervised_profiles=restart.externally_supervised_profiles, + killed_pids=restart.killed_pids, + failed_units=restart.failed_or_stale_units, + # Serve/dashboard reconcile by incarnation liveness, not unit names. + # See #100479. + stale_serve_pids=( + {row.get("pid") for row in _stale_serve_rows} + if _stale_serve_rows is not None + else None + ), + failed_respawn_pids=_dashboards_down, + # A symlinked profile served by another install's checkout (#120240). + external_gateway_pids={row.get("pid") for row in _fleet_snapshot if row_is_external(row)}, + live_gateway_pids=_live_gateway_pids, + ) + from dataclasses import asdict + from hermes_cli.update_serve_obligations import defer_manual_serve + + for runtime, outcome in zip(_pre_update_plan.runtimes, _runtime_outcomes): + if outcome["outcome"] == "unaccounted" and defer_manual_serve(asdict(runtime), require_alive=True): + outcome["outcome"] = "deferred" + if report_unaccounted_runtimes(_runtime_outcomes): + restart.incomplete = True + with suppress(Exception): + import hermes_cli.update_receipt as _ur + _active = _ur._current.get() + if _active is not None: + _active.data["runtime_outcomes"] = _runtime_outcomes + + if not restart.incomplete: + with _best_effort('Owed-gateway check failed: %s'): + if _named_gateways_still_owed(): + # A gateway a previous run could not restart is still not serving HEAD (nothing + # live to restart this time): keep owing it rather than clearing the obligation. + print(" ⚠ A gateway owed a restart by a previous update is still not serving this checkout.") + restart.incomplete = True + if getattr(restart, "stopped_unmapped_pids", None): + # A gateway stopped with no successor is owed a restart, not "accounted for": keep the + # fleet obligation armed so the startup warning names it until someone restarts it. + restart.incomplete = True + if restart.incomplete: + # Code is committed (contract C3): a gateway that may still run stale modules is a + # follow-up, never a failed update. The fleet obligation stays armed, every CLI start + # warns about it, and the next `hermes update` retries the restart. + from hermes_cli.update_receipt import record_followup + stopped = sorted(getattr(restart, "stopped_unmapped_pids", None) or ()) + record_followup( + "gateway_restart", + "gateways may still run pre-update code or were stopped without a successor" + + (f" (stopped PIDs {', '.join(map(str, stopped))})" if stopped else "") + + "; recover with `hermes gateway restart`", + ) + with _best_effort('Fleet restart inventory not recorded: %s'): + _record_owed_gateway_inventory(_pre_update_plan, stopped) + with _best_effort('Update receipt finalize failed: %s'): + from hermes_cli.update_receipt import finalize_update_receipt + # A False ``update_complete`` (unsafe SQLite runtime, owed tail work) never fails the + # run: maintenance already reported it as a follow-up. It only vetoes migration below. + _receipt_path = finalize_update_receipt("success", fleet=_fleet_snapshot) + if _receipt_path is not None: + logger.info("Update receipt written: %s", _receipt_path) + + if restart.incomplete: + return + fleet._clear_fleet_restart_pending_marker() + if not update_complete: + # Fleet caught up, but the selected runtime's SQLite is unsafe or the tail is still owed + # (both reported by the completion): leave the topology alone until they are repaired. + return + # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing + # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later. + try: + from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update + maybe_auto_migrate_after_update() + except (Exception, SystemExit) as exc: # health: allow BLE001 -- a SystemExit here must not fail a committed update + logger.warning('Multiplex auto-migration after update failed: %s', exc) + print(f" ⚠ Gateway multiplex migration did not finish: {exc} (run `hermes gateway migrate` later)") diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py index 193ac25e27af..5504968d507c 100644 --- a/hermes_cli/update_cmd_maint.py +++ b/hermes_cli/update_cmd_maint.py @@ -289,7 +289,7 @@ def _refresh_dashboard_after_update(*, already_restarted_units: set[str] | None # Isolated like every sibling post-update step: a failure here (#112604) used to abort # the fleet matrix, reconciliation and the inner receipt finalize that follow it. A # dashboard/serve left on pre-update code is still caught by the survivor probe → - # reconciliation (exit 1). + # reconciliation (an owed gateway_restart follow-up). logger.warning("Post-update dashboard cleanup failed: %s", exc) _record_update_step("dashboard_cleanup", False, f"{type(exc).__name__}: {exc}") print() @@ -809,14 +809,26 @@ def _run_pre_update_backup(args) -> Optional[str]: def _sweep_bytecode_after_update(branch: str) -> None: """Clear stale ``__pycache__`` (else gateway restart ImportErrors on names absent from old - bytecode), re-stamp the fingerprint, refresh the bootstrap cache scripts.""" + bytecode), re-stamp the fingerprint, refresh the bootstrap cache scripts. + + Never fails the committed update: a failed sweep is a ``bytecode_sweep`` follow-up and the + fingerprint stays un-stamped, so the launch-time sweep (fingerprint mismatch) retries it. + """ from hermes_cli.update_cmd import _m + from hermes_cli.update_receipt import record_followup # Timestamp-based .pyc validation can accept old bytecode after the source swap. - removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) - if removed: - print(f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}") - _m()._record_bytecode_fingerprint() - _m()._refresh_bootstrap_cache_scripts(branch) + try: + removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) + if removed: + print(f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}") + _m()._record_bytecode_fingerprint() + except Exception as exc: # health: allow BLE001 -- owed to the next launch's fingerprint sweep + record_followup("bytecode_sweep", str(exc) or type(exc).__name__) + try: + _m()._refresh_bootstrap_cache_scripts(branch) + except Exception as exc: # health: allow BLE001 -- refreshed again by the next update + record_followup("bootstrap_scripts", str(exc) or type(exc).__name__, + retry="the next `hermes update` refreshes them") def _profile_skill_sync_status(r) -> str: @@ -967,12 +979,28 @@ def _migrate_relay_exporter_env() -> None: def _run_post_update_maintenance( *, assume_yes, gateway_mode, pre_update_snapshot_id, had_desktop_app_before_update, - pre_update_version, completion_message=None, + pre_update_version, completion_message=None, followups=None, ) -> bool: """Post-build housekeeping and completion, returning the SQLite runtime verdict. - Ancillary repairs and notices are best-effort; an unsafe runtime withholds success. + Ancillary repairs and notices are best-effort. The profile sync and the config migration run + after a failed build too and never fail the committed update. A config-migration failure is + printed as ``⚠``, recorded as a receipt follow-up and appended to ``followups`` so the tail + stays owed. Profile sync is best-effort per profile (``_sync_profiles_after_update`` prints + that profile's error and carries on); only a sync that escapes the step is owed. An unsafe + runtime withholds success (and is reported) but is not tail work. """ + from hermes_cli.update_receipt import record_followup + + def owed_step(name, run): + try: + run() + except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed; retry later + reason = str(exc) or type(exc).__name__ + record_followup(name, reason) + if followups is not None: + followups.append((name, reason)) + from hermes_cli.update_cmd import _check_and_apply_config_migration, _m # macOS TCC: Desktop bundles are re-signed each update, so old grants can go stale # (toggle ON, yet macOS re-prompts with no Allow button). Tell users how to re-grant. @@ -1030,11 +1058,11 @@ def _run_post_update_maintenance( print("→ Syncing bundled skills...") _print_bundled_skills_sync_report() - _sync_profiles_after_update() + owed_step("profile_sync", _sync_profiles_after_update) - _check_and_apply_config_migration( + owed_step("config_migration", lambda: _check_and_apply_config_migration( assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, - ) + )) print() update_complete = _print_verified_update_completion(completion_message or _update_complete_message(pre_update_version)) @@ -1045,5 +1073,11 @@ def _run_post_update_maintenance( for line in [*consume_rewritten_notice(), *recorded_standalone_warning_lines()]: print(line) - _print_post_update_notices_and_self_heals() + with _best_effort('Post-update notices failed: %s'): + _print_post_update_notices_and_self_heals() + # A non-✓ completion message (parked local changes) withholds success on its own; only a + # ✓ message that still came back False is the SQLite verdict. + if not update_complete and (completion_message or "✓").startswith("✓"): + record_followup("sqlite_runtime", "the selected Python links an unsafe SQLite runtime", + retry="run the installer again") return update_complete diff --git a/hermes_cli/update_cmd_stale_survivors.py b/hermes_cli/update_cmd_stale_survivors.py index e1716ff81438..d70c408117de 100644 --- a/hermes_cli/update_cmd_stale_survivors.py +++ b/hermes_cli/update_cmd_stale_survivors.py @@ -1,7 +1,7 @@ """Post-update escalation for gateways the fleet matrix proved stale. ``_verify_fleet_after_update`` compares every live gateway's stamped ``code_sha`` against the -fresh checkout. Until now a ``stale`` row only failed the update (exit 1) and left the process +fresh checkout. A ``stale`` row once only failed the update (exit 1) and left the process running pre-update modules: its cron ticker then yields every tick to the "fresh gateway" it assumes exists, and nothing ever restarts it (#117275). A proven-stale survivor is now handed to the same drain-first ``request_restart`` path (SIGUSR1) the restart phase uses — a supervised @@ -29,7 +29,7 @@ def signal_stale_fleet_survivors(fleet: list, restart, drain_budget: float) -> l """Drain-first restart every proven-stale gateway; returns the PIDs signalled. Bookkeeping lands in ``restart.killed_pids`` so the receipt and the survivor sweep see them. - Never raises: verification must still finalize the receipt and exit 1. + Never raises: verification must still finalize the receipt (with an owed restart). """ from hermes_cli.update_cmd_fleet import _drain_or_signal_gateway_for_update @@ -58,6 +58,11 @@ def signal_stale_fleet_survivors(fleet: list, restart, drain_budget: float) -> l logger.warning("Could not signal stale gateway PID %s: %s", pid, exc) print(f" ⚠ {label}: could not be signalled ({exc}) — restart it by hand") if manual: + # No successor: the receipt's gateway_restart follow-up names them and the fleet + # obligation stays armed, so the stop is never silent (codemap §6 V7). + stopped = getattr(restart, "stopped_unmapped_pids", None) + if isinstance(stopped, set): + stopped.update(manual) print(f" → Stopped {len(manual)} manual gateway process(es) that had no supervisor to respawn them") print(" Restart manually: hermes gateway run") if len(manual) > 1: diff --git a/hermes_cli/update_cmd_windows.py b/hermes_cli/update_cmd_windows.py index 2dd7c35d03ea..21b490525a1d 100644 --- a/hermes_cli/update_cmd_windows.py +++ b/hermes_cli/update_cmd_windows.py @@ -1331,15 +1331,15 @@ def _resume_windows_gateways_after_update(token: dict | None) -> None: def _resume_windows_gateways_and_merge_outcome(outcome, _windows_gateway_resume, gateway_mode: bool): """Resume gateways paused for a Windows update and fold the token into ``outcome``'s systemd/launchd-style bookkeeping so reconciliation never reports a healthy gateway as unaccounted. Must never abort the update.""" - from hermes_cli.update_cmd import _m, _write_gateway_update_exit_code + from hermes_cli.update_cmd import _m try: _m()._resume_windows_gateways_after_update(_windows_gateway_resume) except Exception as _windows_resume_exc: + # Owed restart, not a failed update (C3): verification records the ``gateway_restart`` + # follow-up; the gateway's /update marker keeps the committed result. outcome.incomplete = True outcome.phase_errors.append(str(_windows_resume_exc)) print(f" ⚠ Windows gateway service restart incomplete: {_windows_resume_exc}") - if gateway_mode: - _write_gateway_update_exit_code(False) if not isinstance(_windows_gateway_resume, dict): return diff --git a/hermes_cli/update_completion.py b/hermes_cli/update_completion.py index 011e57a89339..d51d4896fb6b 100644 --- a/hermes_cli/update_completion.py +++ b/hermes_cli/update_completion.py @@ -26,11 +26,11 @@ def _exit_status(code: int) -> int: return code if code >= 0 else 128 - code -def _failed_result(request: dict, result_path: Path, code: int) -> int: - code = _exit_status(code) or 1 +def _write_bootstrap_result(request: dict, result_path: Path, code: int, receipt: dict | None) -> int: + """The bootstrap child's answer; windows_resume stays the parent's token (and its atexit net).""" _write_json(result_path, { "schema": 1, "update_id": request["receipt"]["update_id"], "exit_code": code, - "receipt": None, "windows_resume": None, "pm_receipt": request.get("pm_receipt"), + "receipt": receipt, "windows_resume": None, "pm_receipt": request.get("pm_receipt"), }) return code @@ -56,7 +56,7 @@ def run_completion(request: dict) -> dict: request = {**request, "stdout_isatty": sys.stdout.isatty()} request["bytecode_cache"] = str(Path(directory) / "bytecode") _write_json(request_path, request) - command = [sys.executable, "-I", "-S", "-u", "-X", f"pycache_prefix={request['bytecode_cache']}", + command = [sys.executable, "-I", "-S", "-u", "-X", "utf8", "-X", f"pycache_prefix={request['bytecode_cache']}", str(root / "hermes_cli/update_completion.py"), str(request_path), str(result_path)] proc = subprocess.Popen( @@ -116,8 +116,9 @@ def run_completion(request: dict) -> dict: if code == 0 and receipt is None: raise ValueError("completion did not publish a terminal receipt") except (OSError, ValueError, KeyError, TypeError) as exc: - print(f"✗ Source update completion did not finish: {exc}") - return {"exit_code": code or 1, "receipt": None, "windows_resume": None} + # No correlated terminal result: the caller settles it (settle_lost_completion). + return {"exit_code": code or 1, "receipt": None, "windows_resume": None, + "error": f"{exc} (the completion process exited {code})"} return result @@ -132,13 +133,150 @@ def _resume_receipt(data: dict) -> None: def _read_terminal_receipt(request: dict) -> dict | None: - directory = Path(request["home"]) / "logs/update_receipts" - # Never latest.json: another profile/context may have finalized more recently. - for path in directory.glob(f"update_*_{request['receipt']['update_id']}.json"): - data = json.loads(path.read_text(encoding="utf-8-sig")) - if data.get("update_id") == request["receipt"]["update_id"] and data.get("finished_at"): - return data - return None + from hermes_cli import update_receipt + + update_id = request["receipt"]["update_id"] + found = update_receipt.read_run_record(update_id) + if found is not None and found[1].get("finished_at"): + return found[1] + # The store refused the terminal write after the commit point (loudly): this process still + # holds the correlated terminal record it finalized, so the exit status never turns into 1. + return update_receipt.finalized_receipt(update_id) + + +def _running_record(request: dict) -> dict | None: + from hermes_cli.update_receipt import read_run_record + + found = read_run_record(request["receipt"]["update_id"]) + if found is None or found[1].get("outcome") != "running": + return None + found[1].pop("writer_pid", None) + return found[1] + + +def _owed_user_action(request: dict) -> str | None: + """The unsettled-autostash notice (#122557) when this run parked the user's local changes. + + ``_complete_source_update`` hands it over as the completion message, and a completion message + that is not a ``✓`` line is never a success (``_print_verified_update_completion``). + """ + message = request.get("completion_message") or "" + return message if message and not message.startswith("✓") else None + + +def _record_owed_user_action(request: dict) -> bool: + """Land the parked local changes on the receipt; True when nothing is owed to the user.""" + from hermes_cli.update_receipt import record_user_action + + notice = _owed_user_action(request) + if notice: + record_user_action("local_changes", notice) + return notice is None + + +def _publish_gateway_success(request: dict) -> None: + """The gateway /update watcher's status: the code is committed (same as _complete_selected). + + Guarded like ``update_cmd_fleet._write_gateway_update_exit_code`` (which this stdlib-only + process cannot import): an unwritable home must not turn the committed run into an error + before its result is written (review P1). The watcher then falls back to its own timeout. + """ + if not request.get("gateway_mode"): + return + try: + (Path(request["home"]) / ".update_exit_code").write_text("0", encoding="utf-8") + except OSError as exc: + print(f" ⚠ Could not publish the gateway's update status: {exc}") + + +#: The one whole line the Desktop hand-off scripts match (scripts/desktop-update/posix.sh and +#: windows.ps1); ``source_build.build_update_products`` prints the same prefix when the build fails. +DESKTOP_BUILD_OWED = "Desktop app build owed:" + + +def _report_desktop_build_owed(request: dict, record: dict | None, reason: str) -> None: + """Name the owed Desktop build when this run never got the build to its own verdict (F6, review P4). + + The builder prints the line only when the build itself fails; a run that never reached it + (dependencies owed, the tail refused by the lock, a completion process lost) would otherwise end + as a plain "Update complete." in the hand-off. A ``build`` stage on the run's record means the + build ran and printed this line itself when the Desktop app failed. + """ + if not request.get("desktop"): + return + if any(isinstance(mark, dict) and mark.get("name") == "build" for mark in (record or {}).get("stages") or ()): + return + print(f" {DESKTOP_BUILD_OWED} {reason}", flush=True) + + +def settle_lost_completion(request: dict, reason: str) -> dict: + """The parent's side of C3: the code is committed, but the completion never answered (review P2). + + Reached when spawning or reading the completion failed (an OSError from the temporary + directory, the request file or Popen) or the child died without a correlated terminal result + (OOM, SIGKILL). The tail is owed, never a failed update: it is armed for the next launch, the + run's receipt is a success that names the ``completion`` follow-up, and the exit is 0 -- unless + the user's local changes are still parked (#122557: ``partial``, exit 1). Runs in the parent + (pre-swap) interpreter: everything it uses is already imported there. + """ + from hermes_cli import update_receipt + + print(f" ⚠ Source update completion did not finish: {reason}", flush=True) + receipt = _read_terminal_receipt(request) # the child finalized the run, then died + if receipt is None: + try: + from hermes_cli.venv_sync import arm_completion + + arm_completion(Path(request["source"])) + except Exception as exc: # health: allow BLE001 -- post-commit boundary: stale dependencies still make the next launch sync + print(f" ⚠ Could not record the owed source-update tail: {exc}") + _report_desktop_build_owed(request, _running_record(request), "the update completion did not finish") + update_receipt.resume_run_record() # the child's persisted stages, never this stale snapshot + update_receipt.record_followup("completion", reason, retry="the next launch or `hermes update` finishes it") + parked = not _record_owed_user_action(request) + if parked: + print(_owed_user_action(request)) # the completion child that prints it may never have run + update_receipt.finalize_pending_update_receipt(1 if parked else 0, "completion owed after the code was updated") + receipt = _read_terminal_receipt(request) + outcome = (receipt or {}).get("outcome") + code = 130 if outcome == "interrupted" else 1 if outcome == "partial" else 0 + if code == 0: + _publish_gateway_success(request) + return {"exit_code": code, "receipt": receipt, "windows_resume": None, "pm_receipt": None} + + +def _settle_after_commit(request: dict, result_path: Path, step: str, reason: str) -> int: + """The tree already moved, so a failure here is owed work, never a failed update (C3, A6). + + Runs in the bootstrap interpreter (stdlib + the new tree) when dependency preparation failed or + the prepared child died without a result. The tail obligation stays armed, so the next launch + syncs the dependencies and finishes the tail; the run's receipt is a success that names the + follow-up, so nothing reports "still on the previous version" while the tree is new. + """ + from hermes_cli import update_receipt + from hermes_cli.venv_sync import arm_completion + + try: + arm_completion(Path(request["source"])) + except OSError as exc: # the next launch still sees stale dependencies and syncs them + print(f" ⚠ Could not record the owed source-update tail: {exc}") + receipt = _read_terminal_receipt(request) # a prepared child that finalized, then died + if receipt is None: + # A prepared child that died mid-tail persisted stages the parent's snapshot lacks. + record = _running_record(request) or request["receipt"] + _report_desktop_build_owed(request, record, f"{step} did not finish") + _resume_receipt(record) + update_receipt.record_stage("deps" if step == "dependencies" else "build", "failed") + update_receipt.record_followup(step, reason, retry="dependencies not installed yet — the next launch retries" + if step == "dependencies" else "the next launch or `hermes update` retries it") + if not _record_owed_user_action(request): + print(_owed_user_action(request)) # the completion child that would print it never ran + update_receipt.finalize_pending_update_receipt(0, f"{step} owed after the code was updated") + receipt = _read_terminal_receipt(request) + code = 0 if receipt is not None and receipt.get("outcome") == "success" else 1 + if code == 0: + _publish_gateway_success(request) + return _write_bootstrap_result(request, result_path, code, receipt) def _prepare(request: dict, request_path: Path, result_path: Path) -> int: @@ -168,25 +306,39 @@ def _prepare(request: dict, request_path: Path, result_path: Path) -> int: finally: request["pm_receipt"] = receipt.last_for_update(update_id) _write_json(request_path, request) + # -X utf8 like the bootstrap child: -I ignores PYTHONIOENCODING, and this child prints ✓/⚠ + # (the follow-up protocol) into the parent's pipe, whatever the console code page. command = [str(project_python(root)), - "-I", "-S", "-u", "-X", f"pycache_prefix={request['bytecode_cache']}", + "-I", "-S", "-u", "-X", "utf8", "-X", f"pycache_prefix={request['bytecode_cache']}", str(root / "hermes_cli/update_completion.py"), str(request_path), str(result_path), "--prepared"] # A second interpreter is mandatory: PM may have selected a different Python # and dependency graph. No application maintenance runs in this bootstrap. code = _exit_status(subprocess.call(command, cwd=root, env=activation_environment(root))) if not result_path.exists(): - return _failed_result(request, result_path, code) + return _settle_after_commit(request, result_path, "completion", + f"the completion process exited {code} without a result") return code -def _complete_selected(request: dict) -> None: +def _complete_selected(request: dict) -> bool: + """Everything after the commit point. No step failure fails the update (contract C3). + + Each step is independent; a failed one prints ``⚠``, lands on the receipt as a follow-up + and keeps its own obligation armed (``source-completion-pending`` for the tail, the fleet + restart obligation for gateways), so the next launch or ``hermes update`` retries it. + Returns False only when the user's local changes were left parked in the stash: nothing + retries that, so the run is ``partial`` and exits 1 (#122557), never "Update complete". + """ from hermes_cli import main, update_cmd, update_cmd_config from hermes_cli.source_completion import complete_source_checkout from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan + from hermes_cli.update_receipt import ( + TAIL_FOLLOWUPS, record_build_stage, record_followup, record_skip, record_stage) root = Path(request["source"]) main.PROJECT_ROOT = root + complete = _record_owed_user_action(request) update_cmd_config._LAST_SIBLING_SNAPSHOTS = request["sibling_snapshots"] plan_data = request["plan"] plan = None if plan_data is None else UpdatePlan(**{ @@ -194,50 +346,59 @@ def _complete_selected(request: dict) -> None: update_cmd._sweep_bytecode_after_update(request["branch"]) # Launchers, products and post-build maintenance live in one place so an # install and an update cannot end in different states. - complete = complete_source_checkout( - root, desktop=request["desktop"], assume_yes=request["assume_yes"], - gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request["snapshot_id"], - pre_update_version=request["pre_update_version"], - completion_message=request.get("completion_message"), - announce=None if request.get("completion_message") else "\n✓ Code updated!") - from hermes_cli.update_receipt import record_stage - record_stage("build", "success" if complete else "failed") - if complete: + followups: list[tuple[str, str]] = [] + # Exit status and runtime safety are separate facts (R8): an unsafe SQLite runtime keeps the + # committed update at exit 0 (reported as the ``sqlite_runtime`` follow-up), but fleet + # verification still needs the real verdict so it never auto-migrates the gateway topology + # on a runtime that can corrupt sessions. Unknown (the tail raised) is not proven safe. + runtime_safe = False + try: + runtime_safe = complete_source_checkout( + root, desktop=request["desktop"], assume_yes=request["assume_yes"], + gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request["snapshot_id"], + pre_update_version=request["pre_update_version"], + completion_message=request.get("completion_message"), + announce=None if request.get("completion_message") else "\n✓ Code updated!", + followups=followups) + except (Exception, SystemExit) as exc: # health: allow BLE001 -- e.g. the shared update lock refused the tail + reason = str(exc) or type(exc).__name__ + # The tail's steps catch their own failures, so a raise here means the build never ran. + _report_desktop_build_owed(request, None, "the source-update tail did not run") + record_followup("completion", reason) + followups.append(("completion", reason)) + tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in followups) + record_build_stage(followups) + if not tail_owed: from hermes_cli.venv_sync import clear_completion clear_completion(root) # systemctl's KillMode=mixed fallback can kill this whole cgroup. Publish the - # gateway watcher's status BEFORE that operation, and demote on later failure. + # gateway watcher's status BEFORE that operation: the code is committed, so it is 0 unless + # the user's own changes are still parked. if request["gateway_mode"]: update_cmd._write_gateway_update_exit_code(complete) if request.get("no_gateway_restart", False): - from hermes_cli.update_receipt import record_skip - record_skip("gateway_restart", "--no-gateway-restart: deferred, marker kept") record_stage("restart", "skipped") print("→ Gateway restart deferred (--no-gateway-restart); restart gateways separately.") - if not complete: - raise SystemExit(1) - return + return complete skip = update_cmd._fleet_restart_skip_reason(plan) + if skip and update_cmd._pending_fleet_restart_needed(): + # A host already stamped "restarted" for this SHA whose fleet is still off the checkout + # (a stale sibling, a failed resume) gets the restart again instead of a dead end. + print(f" → Gateway restart not skipped ({skip}): gateways are still off the checkout code.") + skip = None if skip: - from hermes_cli.update_receipt import record_skip - record_skip("gateway_restart", skip) record_stage("restart", "skipped") print(f" ✓ Gateway restart skipped: {skip}.") - # Discharges the obligation this run armed when the live fleet vouches for it; a - # fleet still owing the restart fails closed exactly like a stale matrix would. - if update_cmd._pending_fleet_restart_needed(): - print(" ⚠ Gateways are still off the checkout code. Recover with: hermes gateway restart") - raise SystemExit(1) - if not complete: - raise SystemExit(1) - return + return complete restart = update_cmd._restart_gateway_fleet_after_update(plan, request["gateway_mode"]) record_stage("restart", "failed" if getattr(restart, "incomplete", False) else "success") update_cmd._resume_windows_gateways_and_merge_outcome(restart, request["windows_resume"], request["gateway_mode"]) update_cmd._verify_fleet_after_update( - restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], update_complete=complete) + restart, _pre_update_plan=plan, _windows_gateway_resume=request["windows_resume"], + update_complete=bool(runtime_safe) and complete) + return complete class _ForwardedOutput: @@ -253,6 +414,15 @@ def __getattr__(self, name): return getattr(self.stream, name) +def _report_unbuilt_desktop(request: dict) -> None: + """The tail raised out of ``_complete_selected``: before the build stage, the build never ran.""" + from hermes_cli import update_receipt + + current = update_receipt._current.get() + if current is not None: # a finalized run got past the build (verification finalizes it) + _report_desktop_build_owed(request, current.data, "the source-update tail did not finish") + + def _finish(request: dict, result_path: Path) -> int: from hermes_cli import update_receipt from pm.receipt import accept_worker_receipt @@ -262,30 +432,38 @@ def _finish(request: dict, result_path: Path) -> int: update_receipt.record_stage("deps", "success") # only a completed PM preparation reaches --prepared code, reason = 0, "source update completion" try: - _complete_selected(request) + if not _complete_selected(request): + code, reason = 1, "local changes left in the stash; re-apply them by hand" + except KeyboardInterrupt: + # An operator interrupt is not a step failure, and the code already moved: the run is + # ``interrupted`` (never "failed"), and the armed obligations finish the tail. + code, reason = 130, "KeyboardInterrupt: interrupted after the code was updated" + update_receipt.finalize_interrupted_update_receipt(reason, exit_code=code) except SystemExit as exc: - code = _exit_status(exc.code) if isinstance(exc.code, int) else 1 - reason = f"completion exited {code}" - except BaseException as exc: - code = _exit_status(exc.returncode) if isinstance(exc, subprocess.CalledProcessError) else 1 - reason = f"{type(exc).__name__}: {exc}" - print(f"✗ Source update completion failed: {reason}") + if exc.code not in (0, None): + _report_unbuilt_desktop(request) + update_receipt.record_followup("completion", f"completion exited {exc.code}") + except BaseException as exc: # noqa: BLE001 — after the commit point nothing fails the update + _report_unbuilt_desktop(request) + update_receipt.record_followup("completion", f"{type(exc).__name__}: {exc}") finally: - if code and request["gateway_mode"]: - from hermes_cli.update_cmd import _write_gateway_update_exit_code - _write_gateway_update_exit_code(False) # The new interpreter owns recovery too. The original parent's atexit # token is updated from the response; it acts only if this process dies. try: from hermes_cli.update_cmd import _resume_windows_gateways_after_update _resume_windows_gateways_after_update(request["windows_resume"]) except Exception as exc: - code, reason = 1, f"Windows gateway recovery failed: {exc}" - print(f"✗ {reason}") + update_receipt.owe_followup(request["receipt"]["update_id"], "windows_resume", + f"Windows gateway recovery failed: {exc}") update_receipt.finalize_pending_update_receipt(code, reason) terminal_receipt = _read_terminal_receipt(request) + if terminal_receipt and terminal_receipt.get("outcome") == "success": + code = 0 # an interrupt that landed after verification closed the run (review regression 3) + if code and request["gateway_mode"]: + from hermes_cli.update_cmd import _write_gateway_update_exit_code + _write_gateway_update_exit_code(False) if not terminal_receipt: - code = code or 1 + code = code or 1 # the parent settles a lost result (settle_lost_completion) _write_json(result_path, { "schema": 1, "update_id": request["receipt"]["update_id"], "exit_code": code, "receipt": terminal_receipt, "windows_resume": request["windows_resume"], @@ -310,14 +488,23 @@ def main() -> int: from pm.environments import activate_dependencies activate_dependencies(root) return _finish(request, result_path) + return _bootstrap(request, request_path, result_path) + + +def _bootstrap(request: dict, request_path: Path, result_path: Path) -> int: + """Dependency preparation in the parent's interpreter; its failure is a follow-up (A6).""" try: return _prepare(request, request_path, result_path) - except BaseException as exc: - # PM failed before application dependencies were ready. Leave the parent - # receipt and paused-gateway obligation intact for boundary recovery. - print(f"✗ Source update preparation failed: {exc}") - code = exc.returncode if isinstance(exc, subprocess.CalledProcessError) else 1 - return _failed_result(request, result_path, code) + except KeyboardInterrupt: + _resume_receipt(request["receipt"]) + from hermes_cli import update_receipt + update_receipt.finalize_interrupted_update_receipt( + "KeyboardInterrupt: interrupted while installing dependencies", exit_code=130) + return _write_bootstrap_result(request, result_path, 130, _read_terminal_receipt(request)) + except (Exception, SystemExit) as exc: # health: allow BLE001 -- after the commit point nothing fails the update + # The code is committed but its dependencies are not (A6): an owed follow-up, exit 0. + # Paused Windows gateways stay the parent's to resume (it has the dependencies). + return _settle_after_commit(request, result_path, "dependencies", str(exc) or type(exc).__name__) if __name__ == "__main__": diff --git a/hermes_cli/update_finish.py b/hermes_cli/update_finish.py index 32b3845dd0e1..2014b0a1b6cb 100644 --- a/hermes_cli/update_finish.py +++ b/hermes_cli/update_finish.py @@ -8,31 +8,62 @@ def finish_update(*, root, assume_yes, gateway_mode, pre_update_snapshot_id, had_desktop_app_before_update, pre_update_version, - plan, windows_resume) -> None: - """Finish the selected checkout; never fetch, switch branches or restore a stash.""" + plan, windows_resume, followups=None) -> None: + """Finish the selected checkout; never fetch, switch branches or restore a stash. + + Same contract as the current completion (C3): the code is committed, so a failed build, + maintenance step or fleet restart/verification is owed work, never a failed update. + ``followups`` carries the steps that already failed (the product build); while any is owed + the install stamp is withheld, the source-update tail stays pending for the next launch, and + the gateway topology is left alone. An unsafe SQLite runtime is reported by maintenance and + vetoes migration only. + """ from hermes_cli.update_cmd import ( _run_post_update_maintenance, _restart_gateway_fleet_after_update, _verify_fleet_after_update, _write_gateway_update_exit_code, _resume_windows_gateways_and_merge_outcome, ) + from hermes_cli import update_receipt + from hermes_cli.update_receipt import TAIL_FOLLOWUPS, record_build_stage, record_followup - complete = _run_post_update_maintenance( - assume_yes=assume_yes, gateway_mode=gateway_mode, - pre_update_snapshot_id=pre_update_snapshot_id, - had_desktop_app_before_update=had_desktop_app_before_update, - pre_update_version=pre_update_version, - ) - if complete: + owed = followups if followups is not None else [] + runtime_safe = False + try: + runtime_safe = _run_post_update_maintenance( + assume_yes=assume_yes, gateway_mode=gateway_mode, + pre_update_snapshot_id=pre_update_snapshot_id, + had_desktop_app_before_update=had_desktop_app_before_update, + pre_update_version=pre_update_version, followups=owed, + ) + except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed; retry later + reason = str(exc) or type(exc).__name__ + record_followup("maintenance", reason) + owed.append(("maintenance", reason)) + tail_owed = any(step in TAIL_FOLLOWUPS for step, _ in owed) + record_build_stage(owed) + if not tail_owed: from hermes_cli.source_stamp import write_source_stamp - - write_source_stamp(Path(root)) - # Restart can kill this process's gateway cgroup; record its result first. + from hermes_cli.venv_sync import clear_completion + + try: + write_source_stamp(Path(root)) + except (OSError, ValueError) as exc: + print(f"⚠ Source update completed, but the install stamp could not be written: {exc}", + file=sys.stderr) + clear_completion(Path(root)) + # Restart can kill this process's gateway cgroup; record its result first. The code is + # committed, so the watcher sees success; owed work is on the receipt. if gateway_mode: - _write_gateway_update_exit_code(complete) - restarted = _restart_gateway_fleet_after_update(plan, gateway_mode) - _resume_windows_gateways_and_merge_outcome(restarted, windows_resume, gateway_mode) - _verify_fleet_after_update(restarted, _pre_update_plan=plan, - _windows_gateway_resume=windows_resume, update_complete=complete) + _write_gateway_update_exit_code(True) + run = update_receipt._current.get() + update_id = run.data.get("update_id") if run is not None else None + try: + restarted = _restart_gateway_fleet_after_update(plan, gateway_mode) + _resume_windows_gateways_and_merge_outcome(restarted, windows_resume, gateway_mode) + _verify_fleet_after_update(restarted, _pre_update_plan=plan, _windows_gateway_resume=windows_resume, + update_complete=bool(runtime_safe) and not tail_owed) + except (Exception, SystemExit) as exc: # health: allow BLE001 -- the code is committed (C3): the fleet obligation stays armed + update_receipt.owe_followup(update_id, "gateway_restart", str(exc) or type(exc).__name__) def _restore_plan(data): @@ -80,6 +111,7 @@ def main(context: Path, result: Path) -> int: from hermes_cli.source_build import build_update_products from hermes_cli.update_lock import UpdateLock, describe_holder from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update + from hermes_cli.venv_sync import arm_completion cli.PROJECT_ROOT = root if restarting: @@ -105,13 +137,25 @@ def main(context: Path, result: Path) -> int: desktop_dir = root / "apps" / "desktop" desktop = (_desktop_packaged_executable(desktop_dir) is not None or _desktop_dist_exists(desktop_dir)) - build_update_products(root, desktop=desktop) + # The tail is owed from here until it finishes: a failure or a kill leaves the + # pending marker, so the next launch finishes it (same as the current completion). + try: + arm_completion(root) + except OSError as exc: # stale dependencies still trigger the next launch's sync + print(f" ⚠ Could not record the owed source-update tail: {exc}") + owed: list[tuple[str, str]] = [] + try: + build_update_products(root, desktop=desktop) + except (Exception, SystemExit) as exc: # health: allow BLE001 -- committed code: owed, not failed + reason = str(exc) or type(exc).__name__ + update_receipt.record_followup("build", reason) + owed.append(("build", reason)) finish_update( root=root, assume_yes=request["assume_yes"], gateway_mode=request["gateway_mode"], pre_update_snapshot_id=request.get("pre_update_snapshot_id"), had_desktop_app_before_update=desktop, pre_update_version=request.get("pre_update_version"), - plan=plan, windows_resume=token, + plan=plan, windows_resume=token, followups=owed, ) code = 0 except SystemExit as exc: @@ -120,27 +164,32 @@ def main(context: Path, result: Path) -> int: update_receipt.record_step("historical_completion", False, str(exc)) print(f"Update completion failed: {exc}", file=sys.stderr, flush=True) finally: + terminal = update_receipt.finalized_receipt(request["update_id"]) or {} if code and request.get("gateway_mode"): - # Even an application import failure must wake the gateway watcher. + # Even an application import failure must wake the gateway watcher. A run that already + # closed as a success (an interrupt after verification) stays 0 (review regression 3). from hermes_constants import get_hermes_home from hermes_cli.runtime_state import _atomic_bytes - _atomic_bytes(get_hermes_home() / ".update_exit_code", b"1") + _atomic_bytes(get_hermes_home() / ".update_exit_code", + b"0" if terminal.get("outcome") == "success" else b"1") if restarting and not cli_started: # Startup failed before the replacement command could own a # receipt. Preserve the original handoff, just like preparation. update_receipt.begin_update_receipt(previous=request.get("receipt"), correlation_id=request["update_id"]) begun = update_receipt._current.get() is not None + attempted = False if resume is not None: try: resume(token) - except Exception as exc: - code = 1 - print(f"Gateway recovery failed: {exc}", file=sys.stderr, flush=True) + except Exception as exc: # noqa: BLE001 — a restart failure is owed, never the exit status (C3) + attempted = True + update_receipt.owe_followup(request["update_id"], "windows_resume", f"Windows gateway recovery failed: {exc}") handled = cli_started or (begun and update_receipt._current.get() is None) written = update_receipt.finalize_pending_update_receipt(code, "historical takeover completion") result.write_text(json.dumps({ - "resume_handled": not token or not token.get("resume_needed"), + # An attempt that failed is reported here: the historical parent must not replay it. + "resume_handled": attempted or not token or not token.get("resume_needed"), "receipt_handled": handled or written is not None, }), encoding="utf-8") return code diff --git a/hermes_cli/update_handoff.py b/hermes_cli/update_handoff.py index c62e9e58e46a..4ff89ab049ed 100644 --- a/hermes_cli/update_handoff.py +++ b/hermes_cli/update_handoff.py @@ -30,10 +30,10 @@ def is_post_swap_child() -> bool: def write_handoff(payload: dict[str, Any]) -> Path: - """Persist the post-swap payload under HERMES_HOME; returns its path.""" - from hermes_constants import get_hermes_home + """Persist the post-swap payload beside the receipts (the ROOT home); returns its path.""" + from hermes_constants import get_default_hermes_root - directory = get_hermes_home() / "logs" / "update_receipts" + directory = get_default_hermes_root() / "logs" / "update_receipts" directory.mkdir(parents=True, exist_ok=True) path = directory / f"post_swap_{os.getpid()}.json" path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") diff --git a/hermes_cli/update_inventory.py b/hermes_cli/update_inventory.py index a5762a519ddd..fac8b12a26ca 100644 --- a/hermes_cli/update_inventory.py +++ b/hermes_cli/update_inventory.py @@ -542,7 +542,7 @@ def report_unaccounted_runtimes(outcomes: list[dict[str, Any]]) -> bool: """Print a loud warning for runtimes the restart phase never touched. Returns True when at least one planned runtime is unaccounted; the caller escalates like a - STALE/DOWN fleet row (exit 1) — a promised restart silently missed is the class this phase + STALE/DOWN fleet row (an owed ``gateway_restart`` follow-up) — a promised restart silently missed is the class this phase exists to kill. """ manual = [o for o in outcomes if o.get("outcome") == "deferred" and o.get("mechanism") == "respawn-argv"] diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index b1a6c72c7dc1..4682bff304c5 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -8,7 +8,8 @@ 1. **Update receipt** — a machine-readable JSON record of what one ``hermes update`` run discovered, did, skipped (and why), written to - ``/logs/update_receipts/``. Silent-failure classes this + the ROOT home's ``logs/update_receipts/`` (``get_default_hermes_root()``, + never a sticky profile's ``HERMES_HOME``). Silent-failure classes this makes visible: #88848 (helper died after "success" printed), #74973 (restart silently skipped), #85753 (restart phase never ran), #81193 (desktop shows failure for a successful update). @@ -33,6 +34,7 @@ import json import logging import os +import re import sys import time import uuid @@ -43,7 +45,9 @@ logger = logging.getLogger(__name__) -_RECEIPT_KEEP = 20 # keep the last N receipts per profile home +_RECEIPT_KEEP = 20 # keep the last N receipts in the root home's receipt directory +#: Terminal records finalized by THIS process, by update id (see ``finalized_receipt``). +_FINALIZED: dict[str, dict[str, Any]] = {} COMMAND_BOUNDARY_STOP_REASON = "completed at command boundary" # Receipt state is per-CONTEXT, not a module global: a nested @@ -54,18 +58,42 @@ _current: contextvars.ContextVar[Optional["UpdateReceipt"]] = contextvars.ContextVar( "update_receipt_current", default=None ) +#: The terminal outcome of the run the enclosing command scope finalized or adopted (from a +#: completion child): ``committed_success`` answers the command boundary's gateway status from it. +_scope_terminal: contextvars.ContextVar[Optional[dict[str, Any]]] = contextvars.ContextVar( + "update_receipt_scope_terminal", default=None +) @contextmanager def update_receipt_scope(): """Keep the command's finalization guard away from an enclosing update.""" token = _current.set(None) + terminal = _scope_terminal.set({}) try: yield finally: + _scope_terminal.reset(terminal) _current.reset(token) +def adopt_terminal_receipt(data: Any) -> None: + """A completion child finalized this command's run: its terminal outcome is this scope's too.""" + cell = _scope_terminal.get() + if cell is not None and isinstance(data, dict) and data.get("finished_at"): + cell["outcome"] = data.get("outcome") + + +def committed_success() -> bool: + """True when this command's run is closed and its receipt says ``success`` (C3). + + The gateway ``/update`` status written at the command boundary follows the receipt: an + interrupt or error that escapes after the run closed as a success never reports it failed. + """ + cell = _scope_terminal.get() + return _current.get() is None and bool(cell) and cell.get("outcome") == "success" + + def current_correlation_id() -> Optional[str]: """The update correlation id in force in this context, or None. @@ -128,6 +156,9 @@ def __init__(self) -> None: self.correlation_id = uuid.uuid4().hex self.data["update_id"] = self.correlation_id self.data["correlation_id"] = _launcher_correlation_id() + # The dashboard action that spawned this run: the receipt store is root-wide, so the + # action-status route certifies an outcome only from a receipt naming ITS action. + self.data["action_id"] = os.environ.get("HERMES_ACTION_ID", "").strip() or None def step(self, name: str, ok: bool, detail: str = "") -> None: self.data["steps"].append({"name": name, "ok": bool(ok), "detail": detail, "at": _utc_now_iso()}) @@ -142,6 +173,9 @@ def stage(self, name: str, outcome: str, **facts: str) -> None: def fact(self, key: str, value: Any) -> None: self.data[key] = value + def followup(self, step: str, reason: str) -> None: + self.data.setdefault("followups", []).append({"step": step, "reason": reason, "at": _utc_now_iso()}) + def gateway_restart_result( self, *, restarted_services: list | None = None, relaunched_profiles: list | None = None, externally_supervised_profiles: list | None = None, killed_pids: list | None = None, @@ -184,6 +218,8 @@ def gateway_restart_result( self.data["gateway_restart"] = result def finalize(self, outcome: str) -> None: + if outcome == "success" and self.data.get("user_action"): + outcome = "partial" # committed, but the user still has to act (record_user_action) self.data["outcome"] = outcome self.data["finished_at"] = _utc_now_iso() self.data["post_update"] = _code_identity(refresh=True) @@ -192,14 +228,129 @@ def finalize(self, outcome: str) -> None: def _receipt_dir() -> Path: # ``hermes_constants`` (stdlib-only), never ``hermes_cli.config``: the receipt must be # writable from the refused/failed paths where config loading itself may be what broke - # (#112465, #112558). + # (#112465, #112558). The ROOT home, never a sticky profile's: an update mutates the + # checkout every profile shares, and the Desktop and the hand-off scripts read the root. + from hermes_constants import get_default_hermes_root + + return get_default_hermes_root() / "logs" / "update_receipts" + + +def _receipt_dirs() -> list[Path]: + """The root store, then the profile's own when a named profile makes them differ. ``pm/`` + writes its sync receipts to ``get_hermes_home()`` and receipts from before the root move live + there too, so readers take both and writers mirror ``latest.json`` into it (``hermes -p + pm status`` reads only that one).""" from hermes_constants import get_hermes_home - return get_hermes_home() / "logs" / "update_receipts" + root, profile = _receipt_dir(), get_hermes_home() / "logs" / "update_receipts" + return [root] if profile.resolve() == root.resolve() else [root, profile] + + +def _write_latest(payload: bytes) -> None: + """Replace ``latest.json`` in every store a reader of this home consults.""" + from hermes_cli.runtime_state import _atomic_bytes + + for directory in _receipt_dirs(): + directory.mkdir(parents=True, exist_ok=True) + _atomic_bytes(directory / "latest.json", payload) + + +def _run_file(directory: Path, data: dict[str, Any]) -> Path: + """One archive file per run, named at begin so the running and terminal records coincide.""" + stamp = re.sub(r"[^0-9]", "", str(data.get("started_at") or ""))[:14] or time.strftime("%Y%m%d%H%M%S") + return directory / f"update_{stamp[:8]}_{stamp[8:]}_{data.get('pid') or os.getpid()}_{data.get('update_id')}.json" + + +def read_run_record(update_id: str) -> Optional[tuple[Path, dict[str, Any]]]: + """The run's own archive ``(path, record)`` in the root store, or None. Never raises. + + The ONE lookup of a run's archive (named at begin, so its running and terminal records share + one file); never ``latest.json``, which another profile or run may have replaced since. A file + that cannot be read, is not a JSON object, or names another run is not this run's record: + callers treat all of those exactly like a missing archive. + """ + with suppress(OSError): + for path in _receipt_dir().glob(f"update_*_{update_id}.json"): + with suppress(OSError, ValueError): + record = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(record, dict) and record.get("update_id") == update_id: + return path, record + return None + + +def _persist_running(data: dict[str, Any]) -> None: + """Write the open run to disk: a killed update still leaves its own record. Never raises.""" + with suppress(Exception): + from hermes_cli.runtime_state import _atomic_bytes + + directory = _receipt_dir() + directory.mkdir(parents=True, exist_ok=True) + path = _run_file(directory, data) + # A completion child can finalize while its parent still holds the pre-child + # context. That stale running snapshot must never undo the terminal receipt. + stored = read_run_record(str(data.get("update_id"))) + if stored is not None and stored[1].get("finished_at"): + return + from hermes_cli.process_identity import _process_create_time + + payload = (json.dumps({**data, "writer_pid": os.getpid(), + "writer_create_time": _process_create_time(os.getpid())}, + indent=2, default=str) + "\n").encode("utf-8") + _atomic_bytes(path, payload) + _write_latest(payload) + + +def _owner_alive(record: dict[str, Any]) -> bool: + from hermes_cli.process_identity import _pid_alive_matches + + identities = {} + for key, time_key in (("pid", "pid_create_time"), ("writer_pid", "writer_create_time")): + pid = record.get(key) + if isinstance(pid, int) and pid > 0: + # Older receipts recorded no writer incarnation. Do not let a duplicate + # bare PID override the original owner's proven creation-time mismatch. + create_time = record.get(time_key) + if pid not in identities or identities[pid] is None: + identities[pid] = create_time + return any(_pid_alive_matches(pid, create_time) is not False + for pid, create_time in identities.items()) + + +def reconcile_interrupted_runs() -> list[dict[str, Any]]: + """Mark ``running`` records whose processes are gone as ``interrupted``; returns them. + + The start-of-run reclaimer for the durable running receipt: the next update says plainly + that the previous one was killed and at which stage, instead of reporting its outcome as + whatever ran before it. Never raises. + """ + interrupted: list[dict[str, Any]] = [] + with suppress(Exception): + from hermes_cli.runtime_state import _atomic_bytes + + directory = _receipt_dir() + latest = read_latest_receipt() or {} + for path in sorted(directory.glob("update_*.json"), key=lambda p: p.stat().st_mtime)[-_RECEIPT_KEEP:]: + with suppress(Exception): + record = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(record, dict) or record.get("outcome") != "running" or _owner_alive(record): + continue + stages = record.get("stages") or [] + last = stages[-1].get("name") if stages and isinstance(stages[-1], dict) else None + record.update(outcome="interrupted", interrupted_detected_at=_utc_now_iso(), + stop_reason=f"process exited during the update (last stage: {last or 'start'})") + payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8") + _atomic_bytes(path, payload) + if latest.get("update_id") == record.get("update_id"): + _write_latest(payload) + interrupted.append(record) + for record in interrupted: + print(f"⚠ The previous update ({record.get('started_at')}) was interrupted before it finished " + f"({record.get('stop_reason')}); this update completes the work it still owed.") + return interrupted def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | None = None) -> None: - """Start recording a new update receipt. + """Start recording a new update receipt, durably ``running`` until it finalizes. Nested updates are safe: the previous receipt (if any) is preserved behind the ContextVar token and comes back when this one finalizes — @@ -213,10 +364,107 @@ def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | receipt.data.update(update_id=receipt.correlation_id, outcome="running", finished_at=None) # A handoff receipt from an older interpreter may predate the field. receipt.data["correlation_id"] = receipt.data.get("correlation_id") or _launcher_correlation_id() + with suppress(Exception): + from hermes_cli.process_identity import _process_create_time + + receipt.data.setdefault("pid_create_time", _process_create_time(receipt.data["pid"])) + if not previous: + reconcile_interrupted_runs() + # The running record is about to replace latest.json: snapshot the previous record's + # manual serve rows (raw, no side effects) for finalize's carry-forward. A handed-off + # receipt from an older interpreter predates the field, so it needs the same snapshot. + if not previous or "carried_manual_serves" not in previous: + with suppress(Exception): + prior = read_latest_receipt() or {} + rows = list((prior.get("plan") or {}).get("runtimes") or []) + rows += list(prior.get("pending_manual_serves") or []) + rows += list(prior.get("carried_manual_serves") or []) # a run killed before finalize + receipt.data["carried_manual_serves"] = [ + row for row in rows if isinstance(row, dict) and row.get("supervisor") == "manual-serve"] except Exception as exc: # pragma: no cover - defensive logger.debug("Could not start update receipt: %s", exc) return receipt.current_token = _current.set(receipt) + _persist_running(receipt.data) + + +def persist_running_receipt() -> None: + """Re-assert the open run on disk (the completion child calls this after resuming it).""" + current = _current.get() + if current is not None and current.data.get("outcome") == "running": + _persist_running(current.data) + + +#: Follow-up steps whose failure leaves the source-update tail owed (``source-completion-pending``). +TAIL_FOLLOWUPS = frozenset({"dependencies", "launchers", "build", "maintenance", "profile_sync", + "config_migration", "completion"}) + +#: Follow-ups that mean the BUILD stage did not succeed (C3: the receipt names what actually failed): +#: the products or the launchers the build publishes failed, or the tail raised before proving them. +#: Owed dependencies, config migration or maintenance keep the tail armed but say nothing about the +#: build; they are reported as their own follow-ups. +BUILD_FOLLOWUPS = frozenset({"launchers", "build", "completion"}) + + +def record_build_stage(followups) -> None: + """Mark the build stage from the owed ``(step, reason)`` follow-ups: failed only for a build one.""" + record_stage("build", "failed" if any(step in BUILD_FOLLOWUPS for step, _ in followups) else "success") + + +def record_followup(step: str, reason: str, *, retry: str = "the next launch or `hermes update` retries it") -> None: + """A post-commit step failed: print ⚠, keep the run a success, and say what is still owed. + + The code already committed, so the update does not fail; the step's own obligation (the + source tail, the fleet restart, the launch-time bytecode sweep) stays armed and the next + launch or ``hermes update`` retries it. Never raises. The printed line is a protocol: the + Desktop hand-offs (scripts/desktop-update/posix.sh, windows.ps1) parse ``Update follow-up + '' did not finish:`` to report every owed step, so keep it one whole line. + """ + reason = " ".join(str(reason).split())[:500] or "failed" + print(f" ⚠ Update follow-up '{step}' did not finish: {reason} ({retry})", flush=True) + current = _current.get() + if current is None: + return + _record("followup", f"update followup {step}", step, reason) + current = _current.get() + if current is not None: + _persist_running(current.data) + + +def record_user_action(step: str, reason: str) -> None: + """The code committed, but something only the user can do is still owed. Never raises. + + Unlike a follow-up nothing retries it (a stash whose restore conflicted stays parked until the + user re-applies it), so the run can never be a plain success: it finalizes ``partial`` and + ``hermes update`` exits 1, as #122557 established for an unrestored autostash. + """ + _record("fact", f"update user action {step}", "user_action", {"step": step, "reason": " ".join(str(reason).split())[:500]}) + persist_running_receipt() + + +def amend_terminal_followup(update_id: str, step: str, reason: str) -> None: + """A follow-up that failed after the run's receipt was finalized lands on that receipt. Never raises.""" + found = read_run_record(update_id) + if found is None: + return + path, record = found + record.setdefault("followups", []).append({"step": step, "reason": reason, "at": _utc_now_iso()}) + payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8") + with suppress(Exception): # an unwritable store: the printed follow-up line still names it + from hermes_cli.runtime_state import _atomic_bytes + + _atomic_bytes(path, payload) + if (read_latest_receipt() or {}).get("update_id") == update_id: + _write_latest(payload) + + +def owe_followup(update_id: Optional[str], step: str, reason: str, **retry: str) -> None: + """A post-commit step failed: record it on the run while its receipt is open, else amend the + run's closed receipt. Amending an OPEN run would append a second copy beside the one + ``record_followup`` just persisted. Never raises.""" + record_followup(step, reason, **retry) + if _current.get() is None and update_id: + amend_terminal_followup(update_id, step, reason) def _record(method: str, what: str, *args: Any, **kwargs: Any) -> None: @@ -254,6 +502,8 @@ def record_skip(name: str, reason: str) -> None: def record_stage(name: str, outcome: str, **facts: str) -> None: """Mark the END of a pipeline stage (``success``/``failed``/``skipped``) with a timestamp.""" _record("stage", f"update stage {name}", name, outcome, **facts) + # Each stage boundary refreshes the durable running record (a kill names its last stage). + persist_running_receipt() def record_fact(key: str, value: Any) -> None: @@ -293,13 +543,21 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason receipt.finalize(outcome) if stop_reason: receipt.data["stop_reason"] = stop_reason + # The terminal facts exist before the store is touched; a store that refuses the write + # (after the commit point) must not erase them for this process's own caller. + _FINALIZED[str(receipt.data.get("update_id"))] = receipt.data + adopt_terminal_receipt(receipt.data) if fleet is not None: receipt.data["fleet"] = fleet # Manual serve restart obligations outlive one receipt rotation: carry the previous # receipt's still-pending rows forward so the startup warning survives (see # update_serve_obligations). from hermes_cli.update_serve_obligations import retain_receipt_manual_serves - pending = retain_receipt_manual_serves(read_latest_receipt() or {}) + if "carried_manual_serves" in receipt.data: + prior = {"pending_manual_serves": receipt.data.pop("carried_manual_serves") or []} + else: # a receipt begun before the running record existed + prior = read_latest_receipt() or {} + pending = retain_receipt_manual_serves(prior) if pending: receipt.data["pending_manual_serves"] = pending # EMBED the pm sync sections (the settled receipts contract): the @@ -335,14 +593,12 @@ def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason # latest.json pointer (no torn readers). from hermes_cli.runtime_state import _atomic_bytes - path = directory / ( - f"update_{time.strftime('%Y%m%d_%H%M%S')}_{os.getpid()}_" - f"{receipt.correlation_id}.json" - ) + # The run's own file (named at begin): the durable running record becomes terminal. + path = _run_file(directory, receipt.data) payload = (json.dumps(receipt.data, indent=2, default=str) + "\n").encode("utf-8") _atomic_bytes(path, payload) with suppress(Exception): # stable pointer for the dashboard/desktop - _atomic_bytes(directory / "latest.json", payload) + _write_latest(payload) _prune_old_receipts(directory) _publish_shared_metrics(receipt.data) return path @@ -417,13 +673,25 @@ def _publish_shared_metrics(data: dict[str, Any]) -> None: record_update_receipt(data) +def finalized_receipt(update_id: str) -> Optional[dict[str, Any]]: + """The terminal record this process finalized for ``update_id``, published or not. + + The completion child answers its parent from this when the receipt store refused the + terminal write: the run is still correlated and terminal, only its archive is missing + (already reported by ``⚠ Update receipt not written``). + """ + data = _FINALIZED.get(str(update_id)) + return copy.deepcopy(data) if data is not None else None + + def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason: str = "") -> Optional[Path]: """Command-boundary safety net: persist a still-open receipt, if any. Never raises. ``hermes update`` has many early ``sys.exit`` paths (preflight refusals, venv-holder refusal, fetch failure) predating the inner finalize calls; finalizing here means refused/failed runs — where a receipt matters most — leave a record. Exit 0/None → ``success``, exit 2 → ``refused`` - (preflight convention), else → ``failed``. + (preflight convention), else → ``failed`` (``partial`` when the run committed and only owes a + user action, see ``record_user_action``). No-op when no receipt is open (the inner paths already finalized — exactly-once via the popped per-context receipt) or when recording was never started. See #91283. @@ -431,7 +699,8 @@ def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason current = _current.get() if current is None: return None - outcome = "success" if exit_code in (0, None) else "refused" if exit_code == 2 else "failed" + outcome = ("success" if exit_code in (0, None) else "refused" if exit_code == 2 + else "partial" if current.data.get("user_action") else "failed") if exit_code is not None: with suppress(Exception): clone = copy.copy(current) @@ -441,6 +710,54 @@ def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason return finalize_update_receipt(outcome, stop_reason=stop_reason) +def resume_run_record() -> Optional[Path]: + """Carry the open run forward from its own archive when another process took it further. + + The completion child resumes this process's snapshot and persists stages (or closes the run) + the snapshot never saw; closing the run from that stale snapshot would erase them. Returns the + archive when the run is already closed there (the open receipt is dropped: nothing is left to + finalize), else None. Never raises. + """ + current = _current.get() + found = read_run_record(str(current.data.get("update_id"))) if current is not None else None + if found is None: + return None + path, record = found + if record.get("finished_at"): + try: + _current.reset(current.current_token) + except ValueError: # token from another context: pop THIS context only + _current.set(None) + return path + record.pop("writer_pid", None) + clone = copy.copy(current) + clone.data = record + _current.set(clone) + return None + + +def finalize_interrupted_update_receipt(stop_reason: str, *, exit_code: int = 130) -> Optional[Path]: + """Close a run the operator interrupted AFTER the commit point as ``interrupted``. Never raises. + + Not ``failed``: the code already moved, so "still on the previous version" would be false; the + armed obligations finish the rest. The run's own on-disk record is preferred when it is further + along (the completion child persisted stages this process never saw). + """ + print("⚠ Interrupted after the code was updated: the new code is in place; its remaining steps are owed " + "and the next launch or `hermes update` finishes them.", flush=True) + if _current.get() is None: + return None + closed = resume_run_record() + if closed is not None: # the completion child already closed the run + return closed + with suppress(Exception): + clone = copy.copy(_current.get()) + clone.data = copy.deepcopy(clone.data) + clone.data["exit_code"] = int(exit_code) + _current.set(clone) + return finalize_update_receipt("interrupted", stop_reason=stop_reason) + + def _prune_old_receipts(directory: Path) -> None: with suppress(Exception): receipts = (p for p in directory.glob("update_*.json") if p.is_file()) @@ -459,7 +776,7 @@ def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> b matrix as the receipt's post-restart ``fleet`` (and un-flagging ``gateway_restart``) is what lets the stale-runtime readers see the recovery. ``discharges(settled_receipt)`` decides on the in-memory copy; ``latest.json`` is rewritten only when it answers True, so a catch-up - that still exits 1 leaves the receipt byte-identical. Only the ``latest.json`` pointer is + that still owes the restart leaves the receipt byte-identical. Only the ``latest.json`` pointer is rewritten; the archived per-run file keeps the original outcome. Never raises. """ try: @@ -476,7 +793,7 @@ def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> b receipt["gateway_restart"] = gateway_restart if not discharges(receipt): return False - path.write_text(json.dumps(receipt, indent=2, default=str), encoding="utf-8") + _write_latest((json.dumps(receipt, indent=2, default=str) + "\n").encode("utf-8")) return True except Exception as exc: logger.debug("Could not settle latest update receipt from the live fleet: %s", exc) @@ -486,14 +803,33 @@ def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> b def read_latest_receipt() -> Optional[dict[str, Any]]: """Read the most recent update receipt, or None. Never raises.""" with suppress(Exception): - path = _receipt_dir() / "latest.json" - if not path.is_file(): + # The newest pointer wins (the root on a tie): the last writer of either store, exactly + # as when updater and pm shared one folder. + points = [d / "latest.json" for d in _receipt_dirs() if (d / "latest.json").is_file()] + if not points: return None + path = max(points, key=lambda p: p.stat().st_mtime) payload = json.loads(path.read_text(encoding="utf-8-sig")) return payload if isinstance(payload, dict) else None return None +def read_receipt_for_action(action_id: str) -> Optional[dict[str, Any]]: + """Newest receipt written by the dashboard action ``action_id`` (``latest.json`` first, then + the retained archive), or None. Never raises.""" + latest = read_latest_receipt() + if latest and latest.get("action_id") == action_id: + return latest + with suppress(Exception): + archives = (p for d in _receipt_dirs() for p in d.glob("update_*.json")) + for path in sorted(archives, key=lambda p: p.stat().st_mtime, reverse=True): + with suppress(Exception): + payload = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(payload, dict) and payload.get("action_id") == action_id: + return payload + return None + + def _profile_homes() -> list[tuple[str, Path]]: """``(profile, home)`` for the default home plus every valid named profile dir, sorted.""" from hermes_cli.profiles import _get_default_hermes_home, _get_profiles_root, _PROFILE_ID_RE @@ -784,12 +1120,11 @@ def print_fleet_version_matrix(fleet: list[dict[str, Any]]) -> bool: print(" ⚠ Stale gateways keep serving pre-update code until restarted.") if "down" in states: print(" ⚠ Down gateways stopped serving messaging entirely.") - # ``✓ Update complete!`` was already printed before the restart phase (the exit code - # must land before a systemd restart can kill this process), so this verdict has to - # supersede it explicitly — otherwise the output says success while the exit code is 1. + # The code is committed (the update exits 0), but the fleet is not on it yet: say what + # is still owed after ``✓ Update complete!``; every CLI start repeats it until restarted. print() print( - f"✗ Update not complete: {stale_or_down} gateway(s) still running the old code (or stopped).") + f"⚠ Gateway restart still owed: {stale_or_down} gateway(s) still running the old code (or stopped).") print(" Run `hermes gateway restart` (or `hermes -p gateway restart` for a named") print(" profile), then `hermes gateway status` to confirm.") return stale_or_down > 0 diff --git a/hermes_cli/update_serve_obligations.py b/hermes_cli/update_serve_obligations.py index 8d5b71a40e68..e27cb8e9da67 100644 --- a/hermes_cli/update_serve_obligations.py +++ b/hermes_cli/update_serve_obligations.py @@ -60,9 +60,15 @@ def defer_manual_serve(runtime: dict, *, require_alive: bool = False) -> bool: def retain_receipt_manual_serves(receipt: dict) -> list[dict]: - """Return transfers still owed so receipt rotation cannot discard failed writes.""" + """Return transfers still owed so receipt rotation cannot discard failed writes. + + ``carried_manual_serves`` is the durable running record's copy of the previous receipt's rows + (update_receipt.begin): read it too, or the warning vanishes while an update runs and after a + run killed before its ``plan`` stage. + """ plan = receipt.get("plan") or {} rows = list(plan.get("runtimes") or []) + list(receipt.get("pending_manual_serves") or []) + rows += list(receipt.get("carried_manual_serves") or []) pending = [] for row in rows: if not isinstance(row, dict) or row.get("kind") not in ("serve", "dashboard") or row.get("supervisor") != "manual-serve": diff --git a/hermes_cli/web_routers/actions.py b/hermes_cli/web_routers/actions.py index 966662c56935..661379ac1cd5 100644 --- a/hermes_cli/web_routers/actions.py +++ b/hermes_cli/web_routers/actions.py @@ -53,6 +53,7 @@ def _server_path(name: str) -> Path: _ACTION_LOG_TAIL_MAX_CHUNK_BYTES = 64 * 1024 _UPDATE_ACTION_COMPLETED_RE = re.compile(r"^=== hermes-update completed ([0-9a-f]{32}) ===$") +_UPDATE_ACTION_STARTED_RE = re.compile(r"^=== hermes-update started .* ([0-9a-f]{32}) ===$") _MANAGED_EXTERNALLY_MESSAGE = "Hermes updates are managed outside this dashboard in containerized environments." @@ -137,6 +138,25 @@ def _durable_completed_update_action_id(lines: List[str]) -> Optional[str]: return completed_action_id if completed_action_id and last_completed > last_start else None +def _latest_spawned_update_action_id(lines: List[str]) -> Optional[str]: + """Action id named by the latest ``hermes-update started`` header of THIS dashboard's log.""" + for line in reversed(lines): + if line.startswith("=== hermes-update started "): + match = _UPDATE_ACTION_STARTED_RE.fullmatch(line.strip()) + return match.group(1) if match else None + return None + + +def _persisted_action_id(log_dir: Path, name: str) -> Optional[str]: + """Action id of the latest spawn of ``name`` from its sidecar (survives the log's tail + bound and rotation). A visible start header wins: a dashboard predating the sidecar wrote + only the header, and the newest header IS the latest spawn.""" + with contextlib.suppress(OSError, UnicodeDecodeError): + value = (log_dir / f"{name}.action_id").read_text(encoding="utf-8-sig").strip() + return value if re.fullmatch(r"[0-9a-f]{32}", value) else None + return None + + @router.post("/api/gateway/restart") async def restart_gateway(profile: Optional[str] = None): """Kick off a ``hermes gateway restart`` in the background.""" @@ -336,9 +356,9 @@ def _completed_exit_code( if durable_action_id: return 0 if receipt is not None and receipt.get("outcome") in ("success", "partial"): - # No in-memory result and no log marker (e.g. log rotated), but the - # receipt proves a completed run: report its outcome rather than a - # null clients time out on. ``partial`` maps to exit 1 like the CLI. + # No in-memory result and no log marker (e.g. log rotated), but THIS + # action's receipt proves a completed run: report its outcome rather than + # a null clients time out on. ``partial`` maps to exit 1 like the CLI. return 0 if receipt["outcome"] == "success" else 1 return None @@ -355,9 +375,19 @@ async def get_action_status(name: str, lines: int = 200): tail = _tail_lines(log_dir / log_file_name, requested_lines) durable_update_action_id = None - update_receipt_summary = None + update_receipt_summary = action_receipt_summary = None if name == "hermes-update": - durable_update_action_id = _durable_completed_update_action_id(_tail_lines(log_dir / "update.log", 2000)) + # ``hermes update`` mirrors to the ROOT home's update.log (main_dashboard), never this + # dashboard's profile home: read it where it is written. + from hermes_cli.logs import log_file_path + + durable_update_action_id = _durable_completed_update_action_id(_tail_lines(log_file_path("update"), 2000)) + spawned_action_id = (_latest_spawned_update_action_id(_tail_lines(log_dir / log_file_name, 2000)) + or _persisted_action_id(log_dir, name) or _ACTION_IDS.get(name)) + if durable_update_action_id != spawned_action_id: + # The root log is shared by every profile: another profile's (or an older) run's + # completion never certifies the action this dashboard started. + durable_update_action_id = None if durable_update_action_id: marker = f"=== hermes-update completed {durable_update_action_id} ===" if marker not in tail: @@ -367,14 +397,17 @@ async def get_action_status(name: str, lines: int = 200): # dashboard restarting itself mid-action). Surface it so clients READ # the outcome instead of inferring it from liveness probes. # See #81193, #87359, #91277. - update_receipt_summary = _latest_update_receipt_summary() + # The store is root-wide (every profile's runs): show THIS action's receipt when one exists. + update_receipt_summary = _latest_update_receipt_summary(spawned_action_id) + if spawned_action_id and (update_receipt_summary or {}).get("action_id") == spawned_action_id: + action_receipt_summary = update_receipt_summary proc = _ACTION_PROCS.get(name) if proc is None: result = _ACTION_RESULTS.get(name) running = False pid = result.get("pid") if result else None - exit_code = _completed_exit_code(result, durable_update_action_id, update_receipt_summary) + exit_code = _completed_exit_code(result, durable_update_action_id, action_receipt_summary) else: exit_code = proc.poll() running = exit_code is None @@ -401,25 +434,36 @@ def _read_latest_receipt() -> Optional[Dict[str, Any]]: return None -def _latest_update_receipt_summary() -> Optional[Dict[str, Any]]: - """Compact summary of the latest receipt (written by EVERY ``hermes update`` run, - incl. refused/failed), or None; never raises. Steps/skips stay in the full endpoint. +def _latest_update_receipt_summary(action_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Compact summary of dashboard action ``action_id``'s receipt when one exists, else of the + latest receipt (written by EVERY ``hermes update`` run, incl. refused/failed), or None; never + raises. Steps/skips stay in the full endpoint. ``action_id`` names the writer so a client + never credits another action's run. Phase-1 bullet 3 (#91277): the receipt (written by EVERY ``hermes update`` run since #91283, including refused and failed ones, with a ``latest.json`` pointer) is the durable success signal the Desktop and dashboard should read instead of inferring outcomes from liveness probes across the update's stop/start gap (#81193, #87359). """ - receipt = _read_latest_receipt() + receipt = None + if action_id: + from hermes_cli.update_receipt import read_receipt_for_action + receipt = read_receipt_for_action(action_id) + receipt = receipt or _read_latest_receipt() if not receipt: return None try: post = receipt.get("post_update") or {} return { - **{k: receipt.get(k) for k in ("outcome", "started_at", "finished_at")}, + **{k: receipt.get(k) for k in ("outcome", "started_at", "finished_at", "action_id")}, "pre_sha": (receipt.get("pre_update") or {}).get("sha"), "post_sha": post.get("sha"), "post_version": post.get("version"), "fleet_states": sorted({str(e.get("state")) for e in receipt.get("fleet") or [] if isinstance(e, dict)}), + # C3: a committed run is ``success`` even while post-commit steps are owed; carry them + # so a client reports the owed step and its remedy instead of plain success. + "followups": [{"step": str(f.get("step")), "reason": str(f.get("reason") or "")} + for f in receipt.get("followups") or [] if isinstance(f, dict)], + "user_action": receipt.get("user_action") if isinstance(receipt.get("user_action"), dict) else None, } except Exception: return None diff --git a/hermes_cli/web_server_gateway.py b/hermes_cli/web_server_gateway.py index 232d04524647..2aa9d6a61460 100644 --- a/hermes_cli/web_server_gateway.py +++ b/hermes_cli/web_server_gateway.py @@ -457,7 +457,16 @@ def _spawn_hermes_action( from hermes_cli.web_server import PROJECT_ROOT _ACTION_LOG_DIR.mkdir(parents=True, exist_ok=True) log_file = open(_ACTION_LOG_DIR / _ACTION_LOG_FILES[name], "ab", buffering=0) - log_file.write(f"\n=== {name} started {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n".encode()) + # The header carries the action id durably: the update's completion marker lands in the ROOT + # update.log every profile shares, and only this id ties one to the action THIS log started. + action_id = (env_overrides or {}).get("HERMES_ACTION_ID") + stamp = time.strftime('%Y-%m-%d %H:%M:%S') + (f" {action_id}" if action_id else "") + log_file.write(f"\n=== {name} started {stamp} ===\n".encode()) + if action_id: + # ...and a sidecar keeps it once build output pushes the header past the status route's + # bounded tail, or the log rotates: a restarted dashboard still knows which action it ran. + from hermes_cli.runtime_state import _atomic_bytes + _atomic_bytes(_ACTION_LOG_DIR / f"{name}.action_id", action_id.encode("ascii")) from hermes_cli._launchers import runtime_command cmd = runtime_command(PROJECT_ROOT, subcommand) @@ -494,7 +503,6 @@ def _spawn_hermes_action( _ACTION_RESULTS.pop(name, None) _ACTION_COMMANDS[name] = tuple(subcommand) _ACTION_PROCS[name] = proc - action_id = (env_overrides or {}).get("HERMES_ACTION_ID") if action_id: _ACTION_IDS[name] = action_id else: diff --git a/locales/_keys.desktop.json b/locales/_keys.desktop.json index b0babc71d73a..67e80ac0a435 100644 --- a/locales/_keys.desktop.json +++ b/locales/_keys.desktop.json @@ -4697,6 +4697,7 @@ "updates.applyStatus.failed", "updates.applyStatus.noReturn", "updates.applyStatus.notAvailable", + "updates.applyStatus.owed", "updates.applyStatus.preparing", "updates.applyStatus.pulling", "updates.applyStatus.restarting", diff --git a/scripts/desktop-update/posix.sh b/scripts/desktop-update/posix.sh index b1c86d26ed70..76e7cc8f311e 100755 --- a/scripts/desktop-update/posix.sh +++ b/scripts/desktop-update/posix.sh @@ -88,10 +88,17 @@ STARTED_AT="$(date +%s)" # the shim's elapsed clock; see serve-ui.py UI_SERVER_PID="" UI_BROWSER_PID="" UI_PANEL_PID="" UI_PROFILE_DIR="" FINAL_CODE=1 FINAL_MSG="update did not complete" -DONE_NOTE="" # set when the update succeeded but the app will NOT reopen itself +DONE_NOTE="" # set when the update succeeded but the user must act (reopen, reinstall, rebuild) +APP_REBUILD_FAILED=0 # 1 = the code committed but the Desktop app build is an owed follow-up log() { echo "$(date +%Y-%m-%dT%H:%M:%S%z) $1" | tee -a "$LOG" 2>/dev/null; } +owed_followup_steps() { # $OUT -> the distinct owed steps, space-separated, in print order + # hermes_cli/update_receipt.record_followup prints each as one whole line. + printf '%s\n' "$OUT" | sed -n "s/^.*Update follow-up '\([A-Za-z0-9_]*\)' did not finish: .*$/\1/p" \ + | awk '!seen[$0]++' | tr '\n' ' ' | sed 's/ *$//' +} + # Keep a durable signal breadcrumb. A detached hand-off used to leave only the # generic FINAL_MSG when it was terminated while the updater child was running, # which erased the one fact needed to diagnose the failure. @@ -400,7 +407,9 @@ mac_swap() { # Transactional swap: stage a full copy, move the old bundle aside, move # the copy in. Every step checked; a failed final move ROLLS BACK so the # user always has a launchable app, and the result file tells the truth. - if [ "$FINAL_CODE" -eq 0 ] && [ -n "$rebuilt" ] && [ -d "$RELAUNCH_TARGET" ] && [ "$rebuilt" != "$RELAUNCH_TARGET" ]; then + # A failed rebuild left release/ holding the OLD build: nothing new to install. + if [ "$FINAL_CODE" -eq 0 ] && [ "$APP_REBUILD_FAILED" -eq 0 ] && [ -n "$rebuilt" ] \ + && [ -d "$RELAUNCH_TARGET" ] && [ "$rebuilt" != "$RELAUNCH_TARGET" ]; then publish_stage "Installing the new app" rm -rf "$RELAUNCH_TARGET.new" "$RELAUNCH_TARGET.old" 2>/dev/null || true if ! /usr/bin/ditto "$rebuilt" "$RELAUNCH_TARGET.new"; then @@ -434,7 +443,7 @@ deliver_outcome() { # the truth-determining half: swap bundles / gate the relaun else linux_gate if [ "$GATE" != "relaunch" ] && [ "$FINAL_CODE" -eq 0 ]; then - DONE_NOTE="$GATE_MSG" + DONE_NOTE="${DONE_NOTE:+$DONE_NOTE }$GATE_MSG" log "no relaunch ($GATE): $GATE_MSG" fi fi @@ -505,9 +514,10 @@ finish() { fi if [ -n "$DONE_NOTE" ]; then - if [ "$(uname)" = "Darwin" ]; then + if [ "$(uname)" = "Darwin" ] || [ "$GATE" = "relaunch" ]; then # mac DONE_NOTE = swap failed but the PREVIOUS bundle was kept/rolled - # back — bring it back up; the note still tells the user to re-run. + # back, or (any OS) the Desktop rebuild is owed — bring the kept app + # back up; the note still tells the user what to do. # A gated linux outcome (skew/manual) skips the launch BY DESIGN. if ! launch_app; then # Even the kept bundle didn't come back: the durable message must @@ -885,6 +895,27 @@ if [ "$LEGACY_INSTALL" -eq 1 ] && [ "$CODE" -eq 0 ] && printf '%s' "$OUT" | grep fi if [ "$CODE" -eq 0 ]; then FINAL_CODE=0 FINAL_MSG="Update complete." + # Contract C3: a Desktop build that fails after the code committed is an owed + # follow-up (hermes update exits 0 and prints one whole "Desktop app build + # owed:" line for it; the follow-up text itself is truncated). The user is on + # the new Hermes, but this app was not rebuilt: say so and say how to fix it, + # never "finished OK" and never "still on the previous version". + if printf '%s\n' "$OUT" | grep -Eq '^[[:space:]]*Desktop app build owed: '; then + APP_REBUILD_FAILED=1 + DONE_NOTE="Hermes was updated, but the Desktop app could not be rebuilt, so it still runs its old build. Run hermes desktop --force-build in a terminal to rebuild it; the update log has the build error." + log "desktop app build is an owed follow-up of the committed update" + fi + # Every other owed follow-up (a gateway still on the old code, a Windows + # resume, a lost completion, channel adoption, maintenance...) prints one + # whole "Update follow-up '' did not finish:" line. None of them may + # end as a plain "Update complete." (review regression 1). + OWED_STEPS="$(owed_followup_steps)" + if [ -n "$OWED_STEPS" ]; then + case " $OWED_STEPS " in *" gateway_restart "*) OWED_HINT=" Run hermes gateway restart to move the messaging gateway onto the new code now." ;; *) OWED_HINT="" ;; esac + if [ -n "$DONE_NOTE" ]; then DONE_NOTE="$DONE_NOTE Follow-up steps still owed: $OWED_STEPS.$OWED_HINT" + else DONE_NOTE="Hermes was updated, but some follow-up steps did not finish ($OWED_STEPS). The next launch or hermes update retries them; the update log has the details.$OWED_HINT"; fi + log "owed follow-ups of the committed update: $OWED_STEPS" + fi else FINAL_CODE="$CODE" FINAL_MSG="Update failed (exit $CODE). Run hermes debug share in a terminal to send a report." # The bricked-venv class is fixable and must not read as a generic exit 1: diff --git a/scripts/desktop-update/windows.ps1 b/scripts/desktop-update/windows.ps1 index 2bda29767dcd..e50f058f1d07 100644 --- a/scripts/desktop-update/windows.ps1 +++ b/scripts/desktop-update/windows.ps1 @@ -1731,6 +1731,27 @@ try { } } + # Contract C3: a Desktop build that failed after the code committed is an owed follow-up + # (exit 0); the CLI prints one whole "Desktop app build owed:" line for it. The user is on + # the new Hermes but this app was not rebuilt: a manual outcome, never plain success. + if ($res.Code -eq 0 -and -not $desktopBuildFailed -and $res.Output -match '(?m)^\s*Desktop app build owed: ') { + $manualAction = $true + $manualMsg = ("Hermes was updated, but the Desktop app could not be rebuilt, so it still runs its old build. Run ``hermes desktop --force-build`` in a terminal to rebuild it; the update log has the build error. " + $manualMsg).Trim() + Write-HandoffLog $manualMsg + } + + # Every other owed follow-up of the committed update (a gateway still on the old code, a + # Windows resume, a lost completion...) prints one whole "Update follow-up '' did not + # finish:" line (hermes_cli/update_receipt.record_followup): never a plain success either. + $owedSteps = @([regex]::Matches(($res.Output -join "`n"), "Update follow-up '([A-Za-z0-9_]+)' did not finish: ") | + ForEach-Object { $_.Groups[1].Value } | Select-Object -Unique) + if ($res.Code -eq 0 -and -not $desktopBuildFailed -and $owedSteps.Count -gt 0) { + $manualAction = $true + $owedHint = if ($owedSteps -contains 'gateway_restart') { ' Run `hermes gateway restart` to move the messaging gateway onto the new code now.' } else { '' } + $manualMsg = ($manualMsg + " Hermes was updated, but some follow-up steps did not finish (" + ($owedSteps -join ', ') + "). The next launch or ``hermes update`` retries them; the update log has the details." + $owedHint).Trim() + Write-HandoffLog $manualMsg + } + if ($res.Code -eq 0 -and -not $desktopBuildFailed) { $finalCode = 0 $finalMsg = "Update complete." diff --git a/tests/e2e/core/upgrade/git/test_hostile_post.py b/tests/e2e/core/upgrade/git/test_hostile_post.py new file mode 100644 index 000000000000..ccaa2cdb1946 --- /dev/null +++ b/tests/e2e/core/upgrade/git/test_hostile_post.py @@ -0,0 +1,671 @@ +"""Once ``hermes update`` has committed the new code, the update never fails (contract C3). + +One real HEAD install (``_git_world``: the real ``scripts/install.sh`` over real smart HTTP, every +command in a ``bwrap`` sandbox) is shared by the cells. Each cell publishes a hostile upstream +release and breaks something that runs AFTER the checkout moved, for real: + +(a) the web UI build fails: the release's own ``scripts/build/web.mjs`` exits 1 while + ``$HOME/.e2e-break-web-build`` exists, and the release changes a web build input (``web.mjs`` + itself lives under ``scripts/build``, plus a new file under ``web/``) so ``freshness.mjs`` reports + the web UI stale and the updater must build it. A config migration is owed too + (``_config_version`` lowered by one): one failed step must not skip later independent ones. +(b) a gateway that refuses to restart: a credential-free, platform-less (cron-only) ``hermes + gateway run`` lives in the same PID namespace as the updater (``handoff._nshost``). The release + makes ``hermes_cli.gateway.run_gateway`` exit 1 before booting while + ``$HOME/.e2e-break-gateway-boot`` exists. The updater drains the OLD gateway (pre-update code, + no fault) and its relaunch boots the NEW code, which refuses to start: the restart fails for real, + no gateway serves the new commit. Removing the toggle and updating again must retry the owed + restart. +(c) SIGKILL during the post-commit tail: the release's ``web.mjs`` writes + ``$HOME/.e2e-web-build-started`` and sleeps 120 s while ``$HOME/.e2e-slow-web-build`` exists. The + update runs in the shared namespace; once the slow build started, every process born after the + update was spawned is SIGSTOPped then SIGKILLed (that is the whole update tree, including a + completion child in its own session or reparented to the namespace init). +(d) a sticky profile (``hermes profile use e2epost``) must not move update.log / update receipts off + the root hermes home, and ``hermes logs update`` under that profile reads the root's update.log; + pm's own sync receipts stay where pm writes them and ``hermes pm status`` there still shows them. +(e) the dependency sync fails after the tree moved (contract amendment A6): the release's + ``pm.client.ensure_tools_for_sync`` raises while ``$HOME/.e2e-break-deps-sync`` exists; it runs in + the completion bootstrap from the NEW tree, after the checkout moved. Exit 0, a ``dependencies`` + follow-up, the tail obligation armed; the next update completes it. +(f) Ctrl-C after the commit point: SIGINT (what a terminal's Ctrl-C delivers to the foreground + ``hermes update``; the completion child runs in its own session) while the slow web build runs. + The user must read that the new code is in place and the rest is owed, never a failure. + +What is asserted is what the user sees: the exit code, ``⚠`` lines, the receipt at +``/logs/update_receipts/latest.json`` (``outcome``, ``followups``), the +``source-completion-pending`` obligation in the install-state dir, the config version, the +owed-restart warning a later command prints, and which gateway (by its control-socket ``identify``) +serves the checkout. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import shutil +import signal +import tempfile +import time +from pathlib import Path +from typing import Iterator + +import pytest + +import hermes_yaml as yaml +from tests.e2e.core.upgrade import _helpers as H +from tests.e2e.core.upgrade import _install_helpers as I +from tests.e2e.core.upgrade.git import _git_world as G +from tests.e2e.core.upgrade.handoff._nshost import NamespaceHost + +pytestmark = G.PYTESTMARK + +WARN = "⚠" +BREAK_WEB = ".e2e-break-web-build" +SLOW_WEB = ".e2e-slow-web-build" +WEB_STARTED = ".e2e-web-build-started" +BREAK_GATEWAY = ".e2e-break-gateway-boot" +BREAK_DEPS = ".e2e-break-deps-sync" +TOGGLES = (BREAK_WEB, SLOW_WEB, WEB_STARTED, BREAK_GATEWAY, BREAK_DEPS) +DEPS_FAULT_TEXT = "e2e: dependency sync fails on purpose" +DEPS_OWED = "dependencies not installed yet" # update_completion._settle_after_commit +INTERRUPTED_AFTER_COMMIT = "Interrupted after the code was updated" # update_receipt +WEB_FAULT_TEXT = "e2e: the web build fails on purpose" +GATEWAY_FAULT_TEXT = "e2e: gateway boot refused on purpose" +OWED_RESTART = "did not restart running gateways" # update_cmd_fleet._warn_pending_fleet_restart +PROFILE = "e2epost" +UPDATE_TIMEOUT = 1500 +_WEB_MARK = "// e2e (test_hostile_post): hostile build toggles" +_GATEWAY_MARK = "# e2e (test_hostile_post): hostile gateway boot toggle" +_DEPS_MARK = "# e2e (test_hostile_post): hostile dependency sync toggle" + +_HOSTILE_WEB = f"""{_WEB_MARK}, keyed on files in $HOME. +import {{ existsSync as e2eExists, writeFileSync as e2eWrite }} from 'node:fs' +import {{ homedir as e2eHomedir }} from 'node:os' +const e2eHomes = [...new Set([process.env.HOME, e2eHomedir()].filter(Boolean))] +const e2eToggle = name => e2eHomes.some(home => e2eExists(`${{home}}/${{name}}`)) +if (e2eToggle('{BREAK_WEB}')) {{ + console.error('{WEB_FAULT_TEXT} ({BREAK_WEB})') + process.exit(1) +}} +if (e2eToggle('{SLOW_WEB}')) {{ + for (const home of e2eHomes) {{ try {{ e2eWrite(`${{home}}/{WEB_STARTED}`, String(process.pid)) }} catch {{}} }} + console.log('e2e: slow web build started ({SLOW_WEB})') + await new Promise(resolve => setTimeout(resolve, 120000)) +}} +""" + +_HOSTILE_GATEWAY = f""" {_GATEWAY_MARK} + if __import__("os").path.exists(__import__("os").path.join(__import__("os").path.expanduser("~"), "{BREAK_GATEWAY}")): + print("{GATEWAY_FAULT_TEXT} ({BREAK_GATEWAY})", file=__import__("sys").stderr, flush=True) + raise SystemExit(1) +""" + +_HOSTILE_DEPS = f""" {_DEPS_MARK} + if __import__("os").path.exists(__import__("os").path.join(__import__("os").path.expanduser("~"), "{BREAK_DEPS}")): + raise RuntimeError("{DEPS_FAULT_TEXT} ({BREAK_DEPS})") +""" + + +# -- the shared install -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def w() -> Iterator[G.World]: + """A SHORT root (the gateway's AF_UNIX control socket lives under ``$HERMES_HOME``; pytest's deep + ``tmp_path`` overflows ``sun_path``). Set ``HERMES_E2E_KEEP_ROOT=1`` to keep it for a post-mortem.""" + base = os.environ.get("TMPDIR") or tempfile.gettempdir() + root = Path(tempfile.mkdtemp(prefix="hp", dir=base)) + try: + with G.world(root, base=I.head_sha()) as world: + _edit_config(world, lambda cfg: cfg.update(updates={**(cfg.get("updates") or {}), "check": False})) + yield world + finally: + if not os.environ.get("HERMES_E2E_KEEP_ROOT"): + shutil.rmtree(root, ignore_errors=True) + + +def _config_path(w: G.World) -> Path: + return w.sb.hermes_home / "config.yaml" + + +def _load_config(w: G.World) -> dict: + p = _config_path(w) + data = yaml.safe_load(p.read_text(encoding="utf-8-sig")) if p.is_file() else None + return data if isinstance(data, dict) else {} + + +def _edit_config(w: G.World, edit) -> None: + cfg = _load_config(w) + edit(cfg) + _config_path(w).write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") + + +def _config_version(w: G.World) -> int | None: + v = _load_config(w).get("_config_version") + return int(v) if isinstance(v, int) or (isinstance(v, str) and v.isdigit()) else None + + +def _toggle(w: G.World, name: str, on: bool) -> None: + p = w.sb.home / name + if on: + p.write_text("on\n", encoding="utf-8") + else: + p.unlink(missing_ok=True) + + +def _clear_toggles(w: G.World) -> None: + for name in TOGGLES: + _toggle(w, name, False) + + +def _pending_markers(w: G.World) -> list[str]: + """The ``source-completion-pending`` obligation(s) in the install-state dir(s).""" + installs = w.sb.hermes_home / "installs" + return sorted(str(p) for p in installs.glob("**/source-completion-pending")) if installs.is_dir() else [] + + +def _followups(rec: dict) -> list[dict]: + items = rec.get("followups") or [] + return [f for f in items if isinstance(f, dict)] + + +def _followup_steps(rec: dict) -> set[str]: + return {str(f.get("step")) for f in _followups(rec)} + + +def _update_log(w: G.World) -> Path: + return w.sb.hermes_home / "logs" / "update.log" + + +def _read(p: Path) -> str: + return p.read_text(encoding="utf-8-sig", errors="replace") if p.is_file() else "" + + +def _read_json(path: Path) -> dict: + try: + data = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def _warn_lines(text: str) -> list[str]: + return [line.strip() for line in text.splitlines() if WARN in line] + + +def _obligation(w: G.World) -> str: + """The host fleet-restart obligation record(s) anywhere in the sandbox (diagnostics only).""" + found = [] + for base, dirs, files in os.walk(w.sb.root): + dirs[:] = [d for d in dirs if d not in ("node_modules", ".git", "venv", ".venv", "hermes-agent")] + if "host-update-restart.json" in files: + path = Path(base) / "host-update-restart.json" + found.append(f"{path}: {_read(path)[:600]}") + return "; ".join(found) or "none" + + +def _facts(w: G.World, cp=None, **extra) -> str: + """A one-glance summary heading every failure message.""" + rec = w.receipt() + parts = [f"rc={cp.returncode}" if cp is not None else "rc=-", + f"head={w.head()[:12]}", + f"receipt.outcome={rec.get('outcome')!r} update_id={str(rec.get('update_id'))[:12]} " + f"pid={rec.get('pid')} followups={_followups(rec)}", + f"pending={_pending_markers(w)}", + f"config_version={_config_version(w)}", + f"obligation={_obligation(w)}"] + if cp is not None: + parts.append(f"⚠ lines={_warn_lines(G.output(cp))[:8]}") + parts += [f"{k}={v}" for k, v in extra.items()] + return "\n".join(parts) + "\n" + + +# -- hostile releases ---------------------------------------------------------------------------- + + +def _head_file(w: G.World, rel: str) -> str: + return w.git("show", f"HEAD:{rel}") + "\n" + + +def _hostile_web_mjs(w: G.World) -> str: + """The checkout's ``scripts/build/web.mjs`` with the toggle block after its imports.""" + text = _head_file(w, "scripts/build/web.mjs") + if _WEB_MARK in text: + return text + lines = text.splitlines(keepends=True) + last_import = max(i for i, line in enumerate(lines) if line.startswith("import ")) + while " from " not in lines[last_import] and last_import + 1 < len(lines): # a multi-line import + last_import += 1 + out = "".join(lines[:last_import + 1]) + _HOSTILE_WEB + "".join(lines[last_import + 1:]) + assert out.count(_WEB_MARK) == 1 + return out + + +def _publish_web_release(w: G.World, tag: str) -> str: + """A release whose web build obeys the toggles and whose web inputs changed (web UI stale).""" + return w.publish(f"release: e2e hostile post web {tag}", { + "scripts/build/web.mjs": _hostile_web_mjs(w), + f"web/e2e-hostile-post-{tag}.txt": f"web input changed by release {tag}\n", + }) + + +def _publish_gateway_fault_release(w: G.World, tag: str) -> str: + """A release whose ``hermes gateway run`` refuses to boot while the toggle exists.""" + text = _head_file(w, "hermes_cli/gateway.py") + if _GATEWAY_MARK not in text: + m = re.search(r"^def run_gateway\(.*?\):\n \"\"\".*?\"\"\"\n", text, re.S | re.M) + assert m, "premise: hermes_cli/gateway.py has no run_gateway() with a docstring to inject after" + text = text[:m.end()] + _HOSTILE_GATEWAY + text[m.end():] + return w.publish(f"release: e2e hostile post gateway {tag}", { + "hermes_cli/gateway.py": text, + f"docs/e2e-hostile-post-{tag}.txt": f"release {tag}\n", + }) + + +def _publish_deps_fault_release(w: G.World, tag: str) -> str: + """A release whose dependency preparation (``ensure_tools_for_sync``) fails while the toggle exists.""" + text = _head_file(w, "pm/client.py") + if _DEPS_MARK not in text: + m = re.search(r"^def ensure_tools_for_sync\(\) -> None:\n \"\"\".*?\"\"\"\n", text, re.S | re.M) + assert m, "premise: pm/client.py has no ensure_tools_for_sync() with a docstring to inject after" + text = text[:m.end()] + _HOSTILE_DEPS + text[m.end():] + return w.publish(f"release: e2e hostile post deps {tag}", { + "pm/client.py": text, + f"docs/e2e-hostile-post-{tag}.txt": f"release {tag}\n", + }) + + +# -- one shared PID namespace (gateway / killable update) ---------------------------------------- + + +@contextlib.contextmanager +def _namespace(w: G.World) -> Iterator[NamespaceHost]: + """Every process of the cell in ONE sandbox, as on a user's machine: the updater's process scans + see the gateway it restarts. Closing kills the namespace init, which takes everything with it.""" + host = NamespaceHost(w.sb.root, w.sb.env) + try: + yield host + finally: + leaked = host.close() + assert not leaked, f"processes outlived the sandbox: {leaked}" + + +def _ns_cli(host: NamespaceHost, w: G.World, *args: str, timeout: float = 600): + cp = host.run([w.sb.hermes, *args], timeout=timeout) + w.transcripts.append(f"$ hermes {' '.join(args)} -> rc={cp.returncode}\n{(cp.stdout or '')[-6000:]}\n" + f"{(cp.stderr or '')[-3000:]}") + return cp + + +def _ns_update(host: NamespaceHost, w: G.World, *extra: str): + return _ns_cli(host, w, "update", "--yes", "--branch", "main", *extra, timeout=UPDATE_TIMEOUT) + + +def _identify(w: G.World) -> dict | None: + from gateway.control_socket import identify_gateway + + with contextlib.suppress(Exception): + return identify_gateway(w.sb.hermes_home, timeout=5.0) + return None + + +def _serving(w: G.World, sha: str) -> dict | None: + ident = _identify(w) + return ident if ident and ident.get("code_sha") == sha else None + + +def _kill_born_after(host: NamespaceHost, before: set[int]) -> list[int]: + """SIGSTOP then SIGKILL every process born after ``before`` was taken (by PID, namespace view): + the update, its re-exec'd interpreter, build children and any detached completion child (own + session, or reparented to the namespace init). Repeats until none is left.""" + killed: list[int] = [] + for _ in range(10): + live = [p["pid"] for p in host.procs() if p["pid"] not in before and p["state"] not in ("Z", "X")] + if not live: + break + for pid in live: + host.kill(pid, signal.SIGSTOP) + for pid in live: + host.kill(pid, signal.SIGKILL) # windows-footgun: ok - linux-only (bwrap) module + if pid not in killed: + killed.append(pid) + time.sleep(0.5) + return killed + + +def _gateway_logs(w: G.World) -> str: + logs = w.sb.hermes_home / "logs" + chunks = [_read(p)[-4000:] for p in sorted(logs.glob("*.log"))] if logs.is_dir() else [] + chunks += [_read(p)[-4000:] for p in sorted(w.sb.root.glob("gateway-*.log"))] + return "\n".join(chunks) + + +# -- cells --------------------------------------------------------------------------------------- + + +def test_failed_web_build_after_commit_is_a_followup_not_a_failed_update(w): + w.reset_clean() + _clear_toggles(w) + latest = _config_version(w) + if latest is None: + from hermes_cli.config_defaults import DEFAULT_CONFIG + + latest = int(DEFAULT_CONFIG["_config_version"]) + owed = latest - 1 + _edit_config(w, lambda cfg: cfg.update(_config_version=owed)) + target = _publish_web_release(w, "a") + log0 = len(_read(_update_log(w))) + + _toggle(w, BREAK_WEB, True) + try: + cp = w.update("--no-gateway-restart") + finally: + _toggle(w, BREAK_WEB, False) + out = G.output(cp) + log = _read(_update_log(w))[log0:] + rec = w.receipt() + facts = _facts(w, cp, config_version_lowered_to=owed, latest_config_version=latest) + diag = facts + w.diag(cp) + + assert w.head() == target, f"premise: the release never committed, so this is no post-commit cell:\n{diag}" + assert WEB_FAULT_TEXT in out + log, f"premise: the hostile web build never ran (web UI not seen stale?):\n{diag}" + assert cp.returncode == 0, f"the code committed, yet a failed web build failed `hermes update`:\n{diag}" + assert _warn_lines(out), f"the failed web build was not surfaced with a ⚠ line:\n{diag}" + assert rec.get("outcome") == "success", f"receipt outcome is not success after the commit:\n{diag}" + assert "build" in _followup_steps(rec), f"receipt followups do not name the failed `build` step:\n{diag}" + assert _pending_markers(w), f"the failed build disarmed the source-completion obligation:\n{diag}" + assert (_config_version(w) or 0) >= latest, ( + f"the failed web build skipped the independent config migration (_config_version still " + f"{_config_version(w)}, latest {latest}):\n{diag}") + + # The fault clears; the next plain update (no new release) pays the owed tail. + cp = w.update() + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + assert cp.returncode == 0 and w.head() == target, f"the healing update failed:\n{diag}" + assert not _followups(rec), f"the healing update still reports followups:\n{diag}" + assert not _pending_markers(w), f"the healing update left the completion obligation armed:\n{diag}" + assert (w.checkout / "hermes_cli" / "web_dist" / "index.html").is_file(), \ + f"the healing update did not build the web UI:\n{diag}" + + +def test_gateway_that_cannot_reboot_is_an_owed_restart_not_a_failed_update(w): + """Fault: the RELEASE's gateway cannot boot (``run_gateway`` exits 1 while the toggle exists). The + old gateway drains/stops normally; the relaunch runs the new code and dies at boot, so no gateway + serves the new commit. A SIGSTOPped old gateway would not do: the survivor sweep SIGKILLs it and + the relaunch then succeeds.""" + w.reset_clean() + _clear_toggles(w) + with _namespace(w) as host: + try: + gw = host.spawn([w.sb.hermes, "gateway", "run"], log=w.sb.root / "gateway-b.log") # sandbox-writable + before_sha = w.head() + try: + ident = H.wait_for(lambda: _serving(w, before_sha), timeout=240, interval=1.0, + what="the cron-only gateway's control-socket identify") + except AssertionError as exc: + raise AssertionError(f"premise: the gateway never came up: {exc}\n{_gateway_logs(w)[-4000:]}\n" + f"{host.ps_text()}") from None + target = _publish_gateway_fault_release(w, "b") + _toggle(w, BREAK_GATEWAY, True) + cp = _ns_update(host, w) + out = G.output(cp) + rec = w.receipt() + obligation_after_update = _obligation(w) + status = _ns_cli(host, w, "status", timeout=300) + facts = _facts(w, cp, gateway_before=f"pid {ident.get('pid')} sha {before_sha[:12]} (spawned {gw})", + identify_after=_identify(w), obligation_after_update=obligation_after_update) + diag = (facts + w.diag(cp) + "\n--- gateway logs ---\n" + _gateway_logs(w)[-6000:] + + "\n--- sandbox ---\n" + host.ps_text() + "\n--- hermes status ---\n" + G.output(status)[-3000:]) + + assert w.head() == target, f"premise: the release never committed:\n{diag}" + assert not _serving(w, target), f"premise: a gateway serves the new commit; the restart fault never fired:\n{diag}" + assert cp.returncode == 0, f"the code committed, yet a failed gateway restart failed `hermes update`:\n{diag}" + assert _warn_lines(out), f"the failed gateway restart was not surfaced with a ⚠ line:\n{diag}" + assert rec.get("outcome") == "success", f"receipt outcome is not success after the commit:\n{diag}" + assert "gateway_restart" in _followup_steps(rec), \ + f"receipt followups do not name the failed `gateway_restart` step:\n{diag}" + assert OWED_RESTART in G.output(status), \ + f"a later command does not warn that a gateway restart is still owed:\n{diag}" + + # The fault clears; the next plain update retries the owed restart. This gateway is a + # manual `gateway run` that died at boot: no supervisor and no live process, so the + # retry has nothing it can relaunch. It must still exit 0 and must NOT silently discharge + # the obligation (no gateway serves HEAD yet). + _toggle(w, BREAK_GATEWAY, False) + cp = _ns_update(host, w) + status = _ns_cli(host, w, "status", timeout=300) + diag = (_facts(w, cp, identify_after=_identify(w)) + w.diag(cp) + "\n--- gateway logs ---\n" + + _gateway_logs(w)[-6000:] + "\n--- sandbox ---\n" + host.ps_text() + + "\n--- hermes status ---\n" + G.output(status)[-3000:]) + assert cp.returncode == 0, f"the retrying update failed:\n{diag}" + assert OWED_RESTART in G.output(status), \ + f"the retry discharged the owed restart while no gateway serves HEAD:\n{diag}" + + # The operator restarts it (the remedy the warning names); once a gateway serves HEAD the + # obligation discharges and the warning goes away. + host.spawn([w.sb.hermes, "gateway", "run"], log=w.sb.root / "gateway-b2.log") + served = None + with contextlib.suppress(AssertionError): + served = H.wait_for(lambda: _serving(w, target), timeout=240, interval=1.0, what="a gateway on HEAD") + status = _ns_cli(host, w, "status", timeout=300) + diag = (_facts(w, cp, identify_after=_identify(w)) + "\n--- gateway logs ---\n" + + _gateway_logs(w)[-6000:] + "\n--- sandbox ---\n" + host.ps_text() + + "\n--- hermes status ---\n" + G.output(status)[-3000:]) + assert served, f"premise: the restarted gateway never served {target[:12]}:\n{diag}" + assert OWED_RESTART not in G.output(status), \ + f"the owed-restart warning survived a gateway serving HEAD:\n{diag}" + finally: + _toggle(w, BREAK_GATEWAY, False) + for p in host.procs(): + if p["pid"] > 2 and "gateway" in p["cmdline"]: + host.kill(p["pid"], signal.SIGKILL) # windows-footgun: ok - linux-only (bwrap) module + # The namespace (and its gateways) is gone; its PID file names a PID of a dead namespace. + (w.sb.hermes_home / "gateway.pid").unlink(missing_ok=True) + + +def test_sigkill_during_post_commit_tail_leaves_a_running_receipt(w): + w.reset_clean() + _clear_toggles(w) + prev = w.receipt() + target = _publish_web_release(w, "c") + log0 = len(_read(_update_log(w))) + started = w.sb.home / WEB_STARTED + _toggle(w, SLOW_WEB, True) + killed: list[int] = [] + try: + with _namespace(w) as host: + before = {p["pid"] for p in host.procs()} + spawn_log = w.sb.root / "update-c.log" # sandbox-writable + upd = host.spawn([w.sb.hermes, "update", "--yes", "--branch", "main", "--no-gateway-restart"], + log=spawn_log) + + def _slow_build_running(): + if started.is_file(): + return True + if not host.alive(upd): + raise AssertionError(f"premise: the update exited before the slow web build started:\n" + f"{_read(spawn_log)[-6000:]}\n{w.diag()}") + return False + + H.wait_for(_slow_build_running, timeout=1200, interval=1.0, what="the slow web build to start") + with contextlib.suppress(AssertionError): + H.wait_for(lambda: re.search(r"web UI|slow web build", _read(_update_log(w))[log0:]), + timeout=30, interval=0.5, what="update.log to show the web build") + log_shows_build = bool(re.search(r"web UI|slow web build", _read(_update_log(w))[log0:])) + killed = _kill_born_after(host, before) + rec = w.receipt() + facts = _facts(w, killed=killed, update_spawn_pid=upd, log_shows_web_build=log_shows_build, + previous_update_id=prev.get("update_id")) + diag = facts + w.diag() + "\n--- spawned update output ---\n" + _read(spawn_log)[-6000:] + finally: + _toggle(w, SLOW_WEB, False) + started.unlink(missing_ok=True) + + assert killed, f"premise: nothing was killed:\n{diag}" + assert w.head() == target, f"premise: the kill landed before the release committed:\n{diag}" + assert rec.get("outcome") == "running" and rec.get("update_id") and \ + rec.get("update_id") != prev.get("update_id"), \ + f"latest.json is not a `running` record of the killed update:\n{json.dumps(rec, default=str)[:3000]}\n{diag}" + assert rec.get("pid") in killed, f"the running receipt names pid {rec.get('pid')}, not a killed process:\n{diag}" + killed_id = rec["update_id"] + + cp = w.update("--no-gateway-restart") + rec = w.receipt() + diag = _facts(w, cp, killed_update_id=killed_id) + w.diag(cp) + assert "interrupted" in G.output(cp), f"the next update did not report the interrupted run:\n{diag}" + assert cp.returncode == 0, f"the update after the interrupted one failed:\n{diag}" + assert rec.get("outcome") == "success" and rec.get("update_id") != killed_id, \ + f"latest.json is not this run's terminal success:\n{diag}" + + +def test_sticky_profile_update_logs_and_receipts_land_in_the_root_home(w): + w.reset_clean() + _clear_toggles(w) + root = w.sb.hermes_home + prof_receipts = root / "profiles" / PROFILE / "logs" / "update_receipts" + if not (root / "profiles" / PROFILE).is_dir(): + cp = w.sb.cli("profile", "create", PROFILE, "--no-alias", timeout=300) + assert cp.returncode == 0, f"premise: profile create failed:\n{H.describe(cp)}" + cp = w.sb.cli("profile", "use", PROFILE, timeout=300) + assert cp.returncode == 0, f"premise: profile use failed:\n{H.describe(cp)}" + try: + assert (root / "active_profile").read_text(encoding="utf-8-sig").strip() == PROFILE, \ + "premise: the sticky profile was not recorded" + prof_before = {p.name for p in prof_receipts.iterdir()} if prof_receipts.is_dir() else set() + log0 = _update_log(w).stat().st_size if _update_log(w).is_file() else 0 + prev = w.receipt() + target = w.publish("release: e2e hostile post profile d", {"docs/e2e-hostile-post-d.txt": "d\n"}) + + cp = w.update("--no-gateway-restart") + + prof_after = {p.name for p in prof_receipts.iterdir()} if prof_receipts.is_dir() else set() + log1 = _update_log(w).stat().st_size if _update_log(w).is_file() else 0 + rec = w.receipt() + diag = _facts(w, cp, profile_receipts_new=sorted(prof_after - prof_before), root_update_log_bytes=f"{log0}->{log1}", + previous_update_id=prev.get("update_id")) + w.diag(cp) + assert w.head() == target, f"premise: the release never committed:\n{diag}" + assert cp.returncode == 0, f"the sticky-profile update failed:\n{diag}" + # The update's receipts never follow the sticky profile: the root dir is what the Desktop + # and the hand-off scripts read. pm's own sync receipts stay where pm writes them (the + # active home; pm is not the updater's). pm's reader shows the newest receipt of any kind, + # as when both shared one folder: the updater finalizes after pm's sync, so that is this + # run's update receipt, mirrored into the profile store pm reads. + assert not [n for n in prof_after - prof_before if n.startswith("update_")], \ + f"the update wrote update receipts into the profile home {prof_receipts}:\n{diag}" + assert [n for n in prof_after - prof_before if n.startswith("pm_") and "-sync-" in n], \ + f"pm's sync receipt for this update is missing from the profile home {prof_receipts}:\n{diag}" + pm_cp = w.sb.cli("pm", "status", timeout=300) + assert pm_cp.returncode == 0 and json.loads(pm_cp.stdout).get("update_id") == rec.get("update_id"), \ + f"`hermes pm status` under the profile does not show this run's newest receipt:\n{H.describe(pm_cp)}\n{diag}" + assert log1 > log0, f"the root update.log did not grow:\n{diag}" + assert rec.get("update_id") != prev.get("update_id") and rec.get("outcome") == "success", \ + f"root latest.json is not this run's success:\n{diag}" + # The readers follow the writers: `hermes logs update` under the profile shows the root log. + logs_cp = w.sb.cli("logs", "update", "-n", "400", timeout=300) + assert logs_cp.returncode == 0 and "hermes update started" in G.output(logs_cp), \ + f"`hermes logs update` under the sticky profile does not show the root update.log:\n{H.describe(logs_cp)}\n{diag}" + finally: + w.sb.cli("profile", "use", "default", timeout=300) + + +def test_dependency_sync_failure_after_commit_is_a_followup_not_a_failed_update(w): + w.reset_clean() + _clear_toggles(w) + target = _publish_deps_fault_release(w, "e") + _toggle(w, BREAK_DEPS, True) + try: + cp = w.update("--no-gateway-restart") + finally: + _toggle(w, BREAK_DEPS, False) + out = G.output(cp) + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + + assert w.head() == target, f"premise: the release never committed, so this is no post-commit cell:\n{diag}" + assert DEPS_FAULT_TEXT in out, f"premise: the hostile dependency sync never ran:\n{diag}" + assert cp.returncode == 0, f"the code committed, yet a failed dependency sync failed `hermes update`:\n{diag}" + assert any(DEPS_OWED in line for line in _warn_lines(out)), \ + f"no ⚠ line says the dependencies are not installed yet:\n{diag}" + assert rec.get("outcome") == "success", f"receipt outcome is not success after the commit:\n{diag}" + assert "dependencies" in _followup_steps(rec), f"receipt followups do not name `dependencies`:\n{diag}" + assert _pending_markers(w), f"the failed dependency sync disarmed the source-completion obligation:\n{diag}" + + # The fault clears; the next plain update completes the owed dependencies and tail. + cp = w.update("--no-gateway-restart") + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + assert cp.returncode == 0 and w.head() == target, f"the completing update failed:\n{diag}" + assert not _followups(rec), f"the completing update still reports followups:\n{diag}" + assert not _pending_markers(w), f"the completing update left the completion obligation armed:\n{diag}" + + +def _foreground_group(host: NamespaceHost, upd: int) -> list[int]: + """What a terminal's Ctrl-C reaches: the update and its descendants, minus the completion child's + subtree (it runs in its own session, so the terminal's SIGINT never reaches it).""" + procs = {p["pid"]: p for p in host.procs() if p["state"] not in ("Z", "X")} + group, frontier = [], [upd] + while frontier: + pid = frontier.pop() + if pid not in procs or "update_completion.py" in " ".join(procs[pid]["cmdline"]): + continue + group.append(pid) + frontier += [p for p, row in procs.items() if row["ppid"] == pid] + return group + + +def test_ctrl_c_after_commit_reports_the_new_code_not_a_failed_update(w): + w.reset_clean() + _clear_toggles(w) + prev = w.receipt() + target = _publish_web_release(w, "f") + started = w.sb.home / WEB_STARTED + _toggle(w, SLOW_WEB, True) + try: + with _namespace(w) as host: + spawn_log = w.sb.root / "update-f.log" # sandbox-writable + upd = host.spawn([w.sb.hermes, "update", "--yes", "--branch", "main", "--no-gateway-restart"], + log=spawn_log) + + def _slow_build_running(): + if started.is_file(): + return True + if not host.alive(upd): + raise AssertionError(f"premise: the update exited before the slow web build started:\n" + f"{_read(spawn_log)[-6000:]}\n{w.diag()}") + return False + + H.wait_for(_slow_build_running, timeout=1200, interval=1.0, what="the slow web build to start") + interrupted = _foreground_group(host, upd) + for pid in interrupted: + host.kill(pid, signal.SIGINT) + exited = True + try: + H.wait_for(lambda: not host.alive(upd), timeout=120, interval=0.5, what="the interrupted update to exit") + except AssertionError: + exited = False + out = _read(spawn_log) + rec = w.receipt() + diag = (_facts(w, sigint=interrupted, update_exited=exited, previous_update_id=prev.get("update_id")) + + w.diag() + "\n--- sandbox ---\n" + host.ps_text() + + "\n--- interrupted update output ---\n" + out[-6000:]) + finally: + _toggle(w, SLOW_WEB, False) + started.unlink(missing_ok=True) + + assert w.head() == target, f"premise: the SIGINT landed before the release committed:\n{diag}" + assert exited, f"the update did not exit after Ctrl-C:\n{diag}" + assert INTERRUPTED_AFTER_COMMIT in out, \ + f"after Ctrl-C the user is not told the new code is in place and the rest is owed:\n{diag}" + assert rec.get("update_id") != prev.get("update_id") and rec.get("outcome") == "interrupted", \ + f"latest.json is not this run's `interrupted` record (a `failed` one reads as the previous version):\n{diag}" + assert _pending_markers(w), f"the interrupt disarmed the source-completion obligation:\n{diag}" + + cp = w.update("--no-gateway-restart") + rec = w.receipt() + diag = _facts(w, cp) + w.diag(cp) + assert cp.returncode == 0 and rec.get("outcome") == "success", f"the update after Ctrl-C failed:\n{diag}" + assert not _pending_markers(w), f"the update after Ctrl-C left the completion obligation armed:\n{diag}" diff --git a/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py b/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py index 0edc3e265537..8dd99f0b8a22 100644 --- a/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py +++ b/tests/e2e/core/upgrade/pm/test_stamps_and_doctor.py @@ -13,8 +13,10 @@ * real drift is caught and healed: a user uninstalls fastapi (the ``web`` extra the dashboard imports) from the selected environment. ``hermes doctor`` must say so, and ``hermes pm repair`` must bring the dashboard's import back; -* a dependency update that cannot resolve fails loudly, ``hermes pm status`` reports it as failed, - and the previous generation stays selected and working. +* a dependency update that cannot resolve after the tree moved is an owed follow-up (contract C3, + A6): ``hermes update`` exits 0 but says loudly that the dependencies are not installed yet, the + receipt names the ``dependencies`` follow-up, the source-update tail stays armed for the next + launch, and the previous generation stays selected and working. """ from __future__ import annotations @@ -149,20 +151,29 @@ def test_pm_repair_heals_the_drift(drifted): f"`hermes pm repair` exited 0 but the dashboard still cannot import: {imports}\n" + P.diagnostics(sb, rp)) -def test_failed_dependency_update_is_reported_as_failed(updated): - """A release whose uv.lock uv rejects: the update must fail loudly, the receipt must say so, and - the install must keep running on the generation it had.""" +def test_failed_dependency_update_is_an_owed_followup(updated): + """A release whose uv.lock uv rejects, synced after the tree moved: the code is committed, so + the update exits 0 (A6) but must say loudly that the dependencies are not installed yet, the + receipt must name the owed follow-up, the tail must stay armed, and the install must keep + running on the generation it had.""" sb = updated["sb"] before = P.selected_generation(sb) origin, scratch = updated["origin"], updated["root"] lock = I.git("show", "main:uv.lock", cwd=origin) + '\n[[package]]\nname = "e2e-broken"\n' - I.publish_commit(origin, scratch, "release: e2e broken lockfile", {"uv.lock": lock}) + target = I.publish_commit(origin, scratch, "release: e2e broken lockfile", {"uv.lock": lock}) up = P.update(sb) receipt = json.loads(P.ok(sb.cli("pm", "status")).stdout) - assert up.returncode != 0, "`hermes update` exited 0 on a release whose uv.lock uv rejects\n" + P.diagnostics(sb, up) - assert receipt.get("outcome") not in ("ok", "success"), ( - f"`hermes pm status` reports the failed update as a success: {receipt}\n" + P.diagnostics(sb, up)) - assert P.selected_generation(sb) == before, "a failed update switched the selected generation" + assert up.returncode == 0, ( + "`hermes update` failed although the code committed; a dependency sync after the tree moved " + "is an owed follow-up (A6)\n" + P.diagnostics(sb, up)) + assert I.git("rev-parse", "HEAD", cwd=sb.checkout) == target, "the code update did not land\n" + P.diagnostics(sb, up) + assert re.search(r"⚠ Update follow-up 'dependencies' did not finish: .*dependencies not installed yet", + up.stdout + up.stderr), "the owed dependency sync is not reported loudly\n" + P.diagnostics(sb, up) + assert receipt.get("outcome") == "success" and "dependencies" in [ + f.get("step") for f in receipt.get("followups") or []], ( + f"the receipt does not name the owed dependency sync: {receipt}\n" + P.diagnostics(sb, up)) + assert P.pending_marker(sb).exists(), "the owed source-update tail is not armed for the next launch" + assert P.selected_generation(sb) == before, "a failed dependency sync switched the selected generation" imports = P.managed_imports(sb, "pydantic", "openai") assert set(imports.values()) == {"ok"}, f"the install no longer works after a failed update: {imports}" cp = sb.cli("--version") diff --git a/tests/fixtures/resolution_allowlist.json b/tests/fixtures/resolution_allowlist.json index 41f37f2824f7..30a0d3d46163 100644 --- a/tests/fixtures/resolution_allowlist.json +++ b/tests/fixtures/resolution_allowlist.json @@ -356,7 +356,7 @@ }, { "path": "hermes_cli/source_build.py", - "symbol": "build_update_products", + "symbol": "_build_desktop_product", "kind": "bare_which" }, { diff --git a/tests/hermes_cli/test_debug_share_update_logs.py b/tests/hermes_cli/test_debug_share_update_logs.py index 211cfe7ebe3d..449f71698cac 100644 --- a/tests/hermes_cli/test_debug_share_update_logs.py +++ b/tests/hermes_cli/test_debug_share_update_logs.py @@ -67,6 +67,23 @@ def test_snapshots_include_update_and_handoff_keys(self, home_with_update_logs): assert "Desktop GUI build failed" in snaps["update"].tail_text assert "ELIFECYCLE" in snaps["handoff"].tail_text + def test_snapshots_read_the_root_home_update_logs_under_a_profile(self, tmp_path, monkeypatch): + """``hermes update`` and the hand-off scripts write the ROOT home's logs, whatever profile.""" + from hermes_cli.debug import _capture_default_log_snapshots + + root = tmp_path / "root" + profile = root / "profiles" / "coder" + (root / "logs").mkdir(parents=True) + (profile / "logs").mkdir(parents=True) + (root / "logs" / "update.log").write_text(UPDATE_LOG_BODY) + (root / "logs" / "desktop-update-handoff.log").write_text(HANDOFF_LOG_BODY) + monkeypatch.setenv("HERMES_HOME", str(profile)) + + snaps = _capture_default_log_snapshots(100) + + assert "Desktop GUI build failed" in snaps["update"].tail_text + assert "ELIFECYCLE" in snaps["handoff"].tail_text + def test_missing_files_report_absence_without_raising( self, home_without_update_logs ): diff --git a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py index 1a0929113024..4664dde8b8d4 100644 --- a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py +++ b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py @@ -13,6 +13,7 @@ import pytest import hermes_cli.update_cmd_fleet as fleet_mod +import hermes_cli.update_cmd_fleet_verify as fleet_verify import hermes_cli.update_receipt as ur OLD = "a" * 40 @@ -77,7 +78,7 @@ def test_restart_phase_records_accepted_self_restart_and_verify_exits_clean(monk assert pending == {ancestor} _fleet_homes(monkeypatch, tmp_path, {"default": {"pid": ancestor, "gateway_state": "running", "code_sha": OLD}}) - monkeypatch.setattr(fleet_mod, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda *a, **k: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) monkeypatch.setattr(fleet_mod._time, "sleep", lambda s: None) @@ -90,15 +91,18 @@ def test_restart_phase_records_accepted_self_restart_and_verify_exits_clean(monk self_restart_pending_pids=pending, ) with contextlib.redirect_stdout(io.StringIO()) as out: - fleet_mod._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) # a SystemExit(1) here is the #119597 symptom assert "restart pending" in out.getvalue() assert "Update not complete" not in out.getvalue() assert cleared == [True] - with contextlib.redirect_stdout(io.StringIO()), pytest.raises(SystemExit) as exc: - restart.self_restart_pending_pids = set() # same fleet, identity not threaded → STALE, exit 1 - fleet_mod._verify_fleet_after_update( + with contextlib.redirect_stdout(io.StringIO()) as out: + restart.self_restart_pending_pids = set() # same fleet, identity not threaded → STALE + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) - assert exc.value.code == 1 + # Contract C3: STALE is an owed restart (was SystemExit(1)): flagged, follow-up, marker kept. + assert restart.incomplete + assert "follow-up 'gateway_restart'" in out.getvalue() + assert cleared == [True] diff --git a/tests/hermes_cli/test_logs.py b/tests/hermes_cli/test_logs.py index 3ac6b1d65b1f..9643f7217be6 100644 --- a/tests/hermes_cli/test_logs.py +++ b/tests/hermes_cli/test_logs.py @@ -228,3 +228,26 @@ def test_every_log_file_writes_a_stamp_hermes_logs_since_can_read(): # gateway.error.log (launchd stderr, not in LOG_FILES) uses the shared stamper. from hermes_cli.stderr_timestamp import stamp_line assert _parse_line_timestamp(stamp_line("raw gateway stderr")) is not None + + +def test_update_logs_are_read_from_the_root_home_under_a_profile(tmp_path, monkeypatch, capsys): + # ``hermes update`` mirrors to /logs/update.log and the hand-off scripts write the root + # too; `hermes logs update` under a sticky profile read the profile's (absent) copy. + from hermes_cli.logs import list_logs, log_file_path, tail_log + + root = tmp_path / "root" + profile = root / "profiles" / "coder" + (root / "logs").mkdir(parents=True) + (profile / "logs").mkdir(parents=True) + (root / "logs" / "update.log").write_text("=== hermes update started 2026-10-03T10:00:00 ===\n✓ Update complete!\n") + (profile / "logs" / "agent.log").write_text("2026-10-03 10:00:00,000 INFO run_agent: hi\n") + monkeypatch.setenv("HERMES_HOME", str(profile)) + + assert log_file_path("update") == root / "logs" / "update.log" + assert log_file_path("handoff") == root / "logs" / "desktop-update-handoff.log" + assert log_file_path("agent") == profile / "logs" / "agent.log" + tail_log("update", num_lines=5) + assert "✓ Update complete!" in capsys.readouterr().out + list_logs() + listing = capsys.readouterr().out + assert "agent.log" in listing and "update.log (root)" in listing diff --git a/tests/hermes_cli/test_manual_serve_deferral.py b/tests/hermes_cli/test_manual_serve_deferral.py index 05c14404544f..eb8ae394b0ee 100644 --- a/tests/hermes_cli/test_manual_serve_deferral.py +++ b/tests/hermes_cli/test_manual_serve_deferral.py @@ -7,6 +7,7 @@ from hermes_cli import process_identity from hermes_cli import update_cmd_fleet as fleet +from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli import update_receipt from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan from hermes_cli.update_serve_obligations import defer_manual_serve, retain_receipt_manual_serves @@ -20,7 +21,7 @@ def test_manual_deferral_survives_receipt_rotation(monkeypatch, capsys, kind, co plan = UpdatePlan(runtimes=[runtime]) monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: True) monkeypatch.setattr(process_identity, "ledger_entries", lambda: [{"pid": 900, "purpose": kind, "create_time": 1000.0}]) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd._finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None) monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **k: []) @@ -37,13 +38,17 @@ def test_manual_deferral_survives_receipt_rotation(monkeypatch, capsys, kind, co restart.failed_or_stale_units.append("hermes-serve-work.service") restart.incomplete = True if condition != "alive": - with pytest.raises(SystemExit) as exc: - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) - assert exc.value.code == 1 + # Contract C3: a restart that is still owed no longer fails the committed update (was + # SystemExit(1) + "partial") ... + fleet_verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + # ... the obligation stays armed (unchanged) ... assert fleet._fleet_restart_obligation_armed() - assert update_receipt.read_latest_receipt()["outcome"] == "partial" + receipt = update_receipt.read_latest_receipt() + # ... and the receipt is a success naming the owed restart. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] return - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + fleet_verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) receipt = update_receipt.read_latest_receipt() assert receipt["runtime_outcomes"][0]["outcome"] == "deferred" assert not fleet._fleet_restart_obligation_armed() @@ -204,6 +209,34 @@ def replace(src, dst, *args, **kwargs): assert "900" not in capsys.readouterr().err +def test_unsaved_manual_restart_warning_survives_a_running_and_a_killed_update(monkeypatch, capsys): + """The durable running latest.json holds the prior rows as ``carried_manual_serves`` (no plan + yet): the startup warning must read them while an update runs, after one is killed before its + ``plan`` stage, and the next run must carry them forward again.""" + manual = asdict(RuntimeRecord(kind="serve", profile="work", pid=900, supervisor="manual-serve", restart_via="respawn-argv", detail={"create_time": 1000.0})) + root = get_hermes_home() / "logs" / "update_receipts" + root.mkdir(parents=True, exist_ok=True) + (root / "latest.json").write_text(json.dumps({"outcome": "partial", "plan": {"runtimes": [manual]}})) + (get_hermes_home() / "serve_restart_pending").write_text("not a directory") # the reminder can't be filed + monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: True) + + update_receipt.begin_update_receipt() # running record replaces latest.json + running = json.loads((root / "latest.json").read_text()) + assert running["outcome"] == "running" and "plan" not in running + fleet._warn_pending_fleet_restart_on_startup() + assert "serve [work] pid 900" in capsys.readouterr().err + + # Killed before the plan stage: the in-memory receipt is gone, latest.json stays the running record. + update_receipt._current.set(None) + fleet._warn_pending_fleet_restart_on_startup() + assert "serve [work] pid 900" in capsys.readouterr().err + + update_receipt.begin_update_receipt() # the next run reconciles the killed one and carries again + update_receipt.finalize_update_receipt("success", fleet=[]) + fleet._warn_pending_fleet_restart_on_startup() + assert "serve [work] pid 900" in capsys.readouterr().err + + @pytest.mark.parametrize("kind", ["serve", "dashboard"]) @pytest.mark.parametrize("alive", [True, None, False]) def test_unreadable_create_time_discharges_only_a_proven_dead_pid(monkeypatch, kind, alive): diff --git a/tests/hermes_cli/test_old_updater_takeover.py b/tests/hermes_cli/test_old_updater_takeover.py index 465e5ec5c148..32870b23557b 100644 --- a/tests/hermes_cli/test_old_updater_takeover.py +++ b/tests/hermes_cli/test_old_updater_takeover.py @@ -54,10 +54,12 @@ def test_historical_payload_maps_to_takeover_request_schema(tmp_path, desktop, r # checkout is the whole tree the child sees (CI has no editable finder for the source). shutil.copy2(source / "hermes_cli/update_handoff.py", package / "update_handoff.py") # write_handoff resolves the home through hermes_constants; the child tree is the whole - # sys.path (CI has no editable finder), so give it the one name the hand-off reads. + # sys.path (CI has no editable finder), so give it the names the hand-off reads (the + # payload lands beside the receipts in the ROOT home, which is HERMES_HOME here). (root / "hermes_constants.py").write_text( "import os\nfrom pathlib import Path\n" - "def get_hermes_home():\n return Path(os.environ['HERMES_HOME'])\n", encoding="utf-8", + "def get_hermes_home():\n return Path(os.environ['HERMES_HOME'])\n" + "get_default_hermes_root = get_hermes_home\n", encoding="utf-8", ) program = root / "historical.py" program.write_text( diff --git a/tests/hermes_cli/test_restart_plan_reconciliation.py b/tests/hermes_cli/test_restart_plan_reconciliation.py index b74d533df871..f69a5eb77070 100644 --- a/tests/hermes_cli/test_restart_plan_reconciliation.py +++ b/tests/hermes_cli/test_restart_plan_reconciliation.py @@ -494,7 +494,7 @@ def test_live_gateway_pids_from_fleet_skips_down_and_unusable_rows(): PRE-restart PID (nothing replaced it) so it must never count as a successor; rows without a usable profile/PID are skipped; a ``stale`` row is a live successor (the fleet matrix escalates staleness on its own).""" - from hermes_cli.update_cmd_fleet import _live_gateway_pids_from_fleet + from hermes_cli.update_cmd_fleet_verify import _live_gateway_pids_from_fleet rows = [ {"profile": "coder", "pid": 76796, "state": "current"}, diff --git a/tests/hermes_cli/test_source_build.py b/tests/hermes_cli/test_source_build.py index 9c3c9e85d9c6..afe7de03449f 100644 --- a/tests/hermes_cli/test_source_build.py +++ b/tests/hermes_cli/test_source_build.py @@ -299,21 +299,55 @@ def products(): @pytest.mark.platforms("linux") @pytest.mark.parametrize("step", ["tui", "web", "desktop"]) def test_update_failure_raises_without_retries_or_replacing_live_app(source_products, step): - from hermes_cli.source_build import build_update_products + from hermes_cli.source_build import ProductBuildError, build_update_products root, acquired = source_products app = root / "apps/desktop/release/linux-unpacked/hermes" app.parent.mkdir(parents=True) app.write_text("previous app") (root / f"fail-{step}").touch() - with pytest.raises(subprocess.CalledProcessError): + with pytest.raises(ProductBuildError) as failure: build_update_products(root, desktop=True) - assert app.read_text() == "previous app" - assert not list((root / "apps/desktop").glob(".staging-*")) - order = ["deps", "tui", "web", "desktop"] - assert [event["step"] for event in _events(root)] == order[:order.index(step) + 1] + # The failure is still raised, naming the one product that failed (no retries) ... + assert len(failure.value.failures) == 1 + assert isinstance(failure.value.failures[0][1], subprocess.CalledProcessError) + # ... but it no longer skips the independent products after it (was order[:index + 1]). + assert [event["step"] for event in _events(root)] == ["deps", "tui", "web", "desktop"] assert acquired == ["npm"] - assert not (Path(os.environ["HERMES_HOME"]) / "desktop-build-stamp.json").exists() + assert not list((root / "apps/desktop").glob(".staging-*")) + if step == "desktop": + # A failed desktop build never replaces the live app nor stamps it. + assert app.read_text() == "previous app" + assert not (Path(os.environ["HERMES_HOME"]) / "desktop-build-stamp.json").exists() + + +@pytest.mark.platforms("linux") +@pytest.mark.parametrize("failure", ["npm-unavailable", "desktop-after-long-feature-failure"]) +def test_unbuilt_desktop_is_named_on_one_whole_line(source_products, monkeypatch, capsys, failure): + # The Desktop hand-off keys on this line: the receipt follow-up is truncated and leads with + # whichever product failed first, and an npm acquisition failure used to escape un-named. + import hermes_cli.main_install_repair as install_repair + from hermes_cli.source_build import ProductBuildError, build_update_products + + root, _ = source_products + + def fail(error): + def raiser(*_args, **_kwargs): + raise error + return raiser + + if failure == "npm-unavailable": + monkeypatch.setattr(pm, "ensure", fail(pm.InstallError("npm", "unavailable"))) + expected = "Desktop app build owed: Node dependencies failed" + else: + monkeypatch.setattr(install_repair, "_install_configured_features_missing_deps", + fail(RuntimeError("pip install failed: " + "x" * 600))) + (root / "fail-desktop").touch() + expected = "Desktop app build owed: desktop app build failed" + with pytest.raises(ProductBuildError): + build_update_products(root, desktop=True) + lines = [line.strip() for line in capsys.readouterr().out.splitlines()] + assert [line for line in lines if line.startswith("Desktop app build owed:")] == [expected] @pytest.mark.platforms("linux") diff --git a/tests/hermes_cli/test_source_channel_integration.py b/tests/hermes_cli/test_source_channel_integration.py index 5695dc06c058..10d8704e1df8 100644 --- a/tests/hermes_cli/test_source_channel_integration.py +++ b/tests/hermes_cli/test_source_channel_integration.py @@ -125,11 +125,9 @@ def completion(request): monkeypatch.setattr(update_cmd, "_write_fleet_restart_pending_marker", lambda **kw: None) flags = ["--channel", name] if outcome == "transient" else [] args = source.parser.parse_args(["update", "--yes", *flags]) - if outcome == "failed": - with pytest.raises(SystemExit): - update_cmd._cmd_update_impl(args, False) - else: - update_cmd._cmd_update_impl(args, False) + # A completion that answered no terminal receipt ("failed") was SystemExit: after the commit + # point it is an owed completion, exit 0 (review P2) -- and still never adopts the destination. + update_cmd._cmd_update_impl(args, False) expected = {"success": "stable", "already-current": "stable", "failed": name, "transient": name, "concurrent": "my-new-choice"}[outcome] assert saved(source)["channel"] == expected @@ -430,11 +428,13 @@ def test_retirement_waits_for_correlated_completion_process(source, monkeypatch, "receipt": deepcopy(update_receipt._current.get().data), "channel_retirement": {"original": original, "destination": "stable"}} monkeypatch.setattr(update_cmd, "_write_fleet_restart_pending_marker", lambda **kw: None) - if outcome == "success": - update_cmd._complete_source_update(request) - else: + if outcome == "failure": # the child itself answered a correlated failure with pytest.raises(SystemExit): update_cmd._complete_source_update(request) + else: + # An uncorrelated or missing answer was SystemExit: after the commit point it is an owed + # completion, exit 0 (review P2). Adoption still waits for a verified completion. + update_cmd._complete_source_update(request) assert saved(source)["channel"] == ("stable" if outcome == "success" else name) child_request = json.loads((source.home / "child-request.json").read_text()) assert child_request["channel_retirement"]["original"] == original diff --git a/tests/hermes_cli/test_source_completion_stamp.py b/tests/hermes_cli/test_source_completion_stamp.py index c445aafcd374..6ca887dae08f 100644 --- a/tests/hermes_cli/test_source_completion_stamp.py +++ b/tests/hermes_cli/test_source_completion_stamp.py @@ -52,12 +52,30 @@ def maintenance(**_kwargs): def test_failed_source_completion_does_not_publish_identity(tmp_path, monkeypatch): root = _repo(tmp_path) - _completion_dependencies(monkeypatch, lambda **_kwargs: False) + _completion_dependencies(monkeypatch, lambda **_kwargs: True) - assert not complete_source_checkout(root, desktop=False, assume_yes=True) + def broken_build(root, *, desktop): + raise RuntimeError("web UI build: npm exited 1") + + monkeypatch.setattr("hermes_cli.source_build.build_update_products", broken_build) + owed: list = [] + assert not complete_source_checkout(root, desktop=False, assume_yes=True, followups=owed) + # The tail is owed, so the checkout identity is not published. + assert [step for step, _ in owed] == ["build"] assert not (root / "install-stamp.json").exists() +def test_unsafe_sqlite_withholds_success_but_not_the_finished_tail(tmp_path, monkeypatch): + # Was: an unsafe SQLite verdict also withheld the stamp, so the tail stayed owed forever + # (re-running it cannot fix the interpreter, codemap §8 V1). The install still reports failure. + root = _repo(tmp_path) + _completion_dependencies(monkeypatch, lambda **_kwargs: False) + owed: list = [] + assert not complete_source_checkout(root, desktop=False, assume_yes=True, followups=owed) + assert owed == [] + assert (root / "install-stamp.json").is_file() + + def _sandboxed_marker(tmp_path, monkeypatch): """Point every update-lock reader at a marker inside this test's sandbox.""" marker = tmp_path / ".hermes-update-in-progress" diff --git a/tests/hermes_cli/test_update_completion_process.py b/tests/hermes_cli/test_update_completion_process.py index 91158c51c39b..7648225b8062 100644 --- a/tests/hermes_cli/test_update_completion_process.py +++ b/tests/hermes_cli/test_update_completion_process.py @@ -155,6 +155,22 @@ def git(*args): "_current = contextvars.ContextVar('receipt', default=None)\n" "class UpdateReceipt: pass\n" "def record_stage(*args, **kwargs): pass\n" + "def record_build_stage(*args, **kwargs): pass\n" + "def record_skip(*args, **kwargs): pass\n" + "TAIL_FOLLOWUPS = frozenset({'dependencies', 'launchers', 'build', 'maintenance', 'profile_sync',\n" + " 'config_migration', 'completion'})\n" + "finalized_receipt = lambda update_id: None\n" + "def record_followup(step, reason, **kwargs):\n" + " print(f' ⚠ Update follow-up {step!r} did not finish: {reason}')\n" + " r = _current.get()\n" + " if r is not None: r.data.setdefault('followups', []).append({'step': step, 'reason': reason})\n" + "def amend_terminal_followup(*args): pass\n" + "def record_user_action(*args): pass\n" + "def _receipt_dir():\n" + " return pathlib.Path(os.environ['HERMES_HOME']) / 'logs/update_receipts'\n" + "def read_run_record(update_id):\n" + " for path in _receipt_dir().glob(f'update_*_{update_id}.json'):\n" + " return path, json.loads(path.read_text(encoding='utf-8-sig'))\n" "def finalize_pending_update_receipt(code, reason):\n" " r = _current.get()\n" " if r is None: return\n" @@ -235,13 +251,11 @@ def test_old_process_new_git_tree_completes_in_fresh_python(transition, tmp_path assert [e["name"] for e in events].index("exit_marker") < [e["name"] for e in events].index("restart") -@pytest.mark.parametrize("code", [0, 23]) -def test_missing_child_result_fails_boundary_receipt_and_releases_lock(transition, monkeypatch, code): +def _run_cmd_update(monkeypatch, request, capsys): + """``hermes update --gateway`` whose impl hands this request to the real post-commit completion.""" from types import SimpleNamespace - from hermes_cli import main, update_cmd, update_receipt, update_lock + from hermes_cli import main, update_cmd, update_receipt - root, git, old, new, request = transition - (root / "hermes_cli/update_completion.py").write_text(f"import os\nos._exit({code})\n") monkeypatch.setenv("HERMES_HOME", request["home"]) monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) @@ -253,20 +267,129 @@ def complete(args, gateway_mode): update_cmd._complete_source_update(request) monkeypatch.setattr(update_cmd, "_cmd_update_impl", complete) - with pytest.raises(SystemExit) as error: + try: main.cmd_update(SimpleNamespace(gateway=True)) - assert error.value.code == (code or 1) - receipt = update_receipt.read_latest_receipt() - assert receipt["outcome"] == "failed" - assert receipt["exit_code"] == (code or 1) + code = 0 + except SystemExit as exc: + code = exc.code + return code, update_receipt.read_latest_receipt(), capsys.readouterr().out + + +@pytest.mark.parametrize("code", [0, 23, -9]) +def test_missing_child_result_is_an_owed_completion_and_releases_lock(transition, monkeypatch, capsys, code): + """Review P2 (invariant 3): the tree already moved, so a completion that died without a result + (exit 0/23 with no answer, SIGKILL/OOM) is an owed ``completion`` follow-up, never "Update failed". + + Was exit ``code or 1``, receipt ``failed``, ``.update_exit_code`` 1 and the paused gateways left + paused: that pinned the failure invariant 3 forbids. + """ + from hermes_cli import update_lock + + root, git, old, new, request = transition + die = f"os._exit({code})" if code >= 0 else "os.kill(os.getpid(), 9)" + (root / "hermes_cli/update_completion.py").write_text(f"import os\n{die}\n", encoding="utf-8") + exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) + assert exit_code == 0, out assert receipt["update_id"] == request["receipt"]["update_id"] - assert request["windows_resume"]["resume_needed"] is True - assert (Path(request["home"]) / ".update_exit_code").read_text().strip() == "1" + assert receipt["outcome"] == "success" + assert receipt["exit_code"] == 0 + assert [row["step"] for row in receipt["followups"]] == ["completion"] + assert "Update failed" not in out + # The Desktop hand-off scripts must not read this as a plain "Update complete." (review P4). + assert "Desktop app build owed: the update completion did not finish" in out + # This process resumes what the lost child never did, and the gateway watcher hears success. + assert request["windows_resume"]["resume_needed"] is False + assert (Path(request["home"]) / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" lock = update_lock.UpdateLock() assert lock.acquire() lock.release() +def test_completion_spawn_failure_after_commit_is_owed(transition, monkeypatch, capsys): + """Review P2: an OSError while starting the completion (temporary directory, request file, + Popen) after the commit point is the same owed ``completion`` follow-up and exit 0.""" + import tempfile + + root, git, old, new, request = transition + + def no_temporary_directory(*args, **kwargs): + raise FileNotFoundError(2, "No usable temporary directory found") + + monkeypatch.setattr(tempfile, "TemporaryDirectory", no_temporary_directory) + exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) + assert exit_code == 0, out + assert receipt["outcome"] == "success" + assert [row["step"] for row in receipt["followups"]] == ["completion"] + assert "No usable temporary directory" in receipt["followups"][0]["reason"] + assert (Path(request["home"]) / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" + + +def test_lost_completion_with_parked_local_changes_stays_partial(transition, monkeypatch, capsys): + """The one documented exception survives the owed path: parked user changes still exit 1.""" + from hermes_cli import update_cmd + + root, git, old, new, request = transition + (root / "hermes_cli/update_completion.py").write_text("import os\nos._exit(0)\n", encoding="utf-8") + notice = "⚠ Your local changes are parked in stash@{0}; re-apply them by hand." + monkeypatch.setattr(update_cmd, "_unrestored_autostash_notice", lambda: notice) + exit_code, receipt, out = _run_cmd_update(monkeypatch, request, capsys) + assert exit_code == 1, out + assert receipt["outcome"] == "partial" + assert [row["step"] for row in receipt["followups"]] == ["completion"] + assert notice in out + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("never_built", ["dependencies", "refused", "died"]) +@pytest.mark.parametrize("desktop", [True, False]) +def test_desktop_build_that_never_ran_is_named_for_the_handoff(transition, capsys, never_built, desktop): + """Review P4 (F6): the hand-off scripts read ``Desktop app build owed:`` to tell the user the app is + stale. The builder prints it only when the build fails; a run whose build never ran (dependency + sync failed, the tail refused by the update lock, the prepared child died) must print it too.""" + scripts = Path(update_completion.__file__).parents[1] / "scripts/desktop-update" + for script in ("posix.sh", "windows.ps1"): # the exact prefix the hand-off scripts match + assert "Desktop app build owed: " in (scripts / script).read_text(encoding="utf-8-sig") + assert update_completion.DESKTOP_BUILD_OWED == "Desktop app build owed:" + root, git, old, new, request = transition + request["desktop"] = desktop + shutil.copy2(update_completion.__file__, root / "hermes_cli/update_completion.py") + if never_built == "dependencies": + (root / "pm/__init__.py").write_text("def sync_venv(**kw): raise RuntimeError('dependency refused')\n", + encoding="utf-8") + elif never_built == "refused": + (root / "hermes_cli/source_completion.py").write_text( + "def complete_source_checkout(*args, **kwargs):\n" + " raise RuntimeError('an update is still running (pid 1); wait for it to exit')\n", encoding="utf-8") + else: + (root / "hermes_cli/source_completion.py").write_text( + "import os\n" + "def complete_source_checkout(*args, **kwargs): os.kill(os.getpid(), 9) # posix-only test\n", + encoding="utf-8") + result = update_completion.run_completion(request) + out = capsys.readouterr().out + assert result["exit_code"] == 0, out + owed = [line.strip() for line in out.splitlines() if line.strip().startswith("Desktop app build owed: ")] + assert len(owed) == (1 if desktop else 0), out + + +def test_unwritable_gateway_status_never_fails_a_settled_commit(tmp_path, monkeypatch): + """Review P1: the bootstrap's gateway status write is best effort, like + ``_write_gateway_update_exit_code``; the committed run still answers its result with exit 0.""" + from hermes_cli import venv_sync + + monkeypatch.setattr(venv_sync, "arm_completion", lambda root: None) + receipt = {"update_id": "u1", "outcome": "success", "finished_at": "now"} + monkeypatch.setattr(update_completion, "_read_terminal_receipt", lambda request: receipt) + home = tmp_path / "home-is-a-file" + home.write_text("x", encoding="utf-8") + result = tmp_path / "result.json" + request = {"source": str(tmp_path), "receipt": {"update_id": "u1"}, "gateway_mode": True, + "home": str(home), "pm_receipt": None} + assert update_completion._settle_after_commit(request, result, "dependencies", "pip failed") == 0 + answer = json.loads(result.read_text(encoding="utf-8-sig")) + assert answer["exit_code"] == 0 and answer["receipt"] == receipt + + @pytest.mark.parametrize("cleanup_failure", [None, "kill", "wait"]) def test_interrupt_after_child_success_demotes_gateway_marker_at_boundary(transition, monkeypatch, cleanup_failure): import io @@ -334,21 +457,28 @@ def complete(args, gateway_mode): monkeypatch.setattr(subprocess, "Popen", capture_child) update_cmd._complete_source_update(request) - monkeypatch.setattr(sys, "stdout", Interrupt()) + stdout = Interrupt() + monkeypatch.setattr(sys, "stdout", stdout) monkeypatch.setattr(update_cmd, "_cmd_update_impl", complete) - with pytest.raises(KeyboardInterrupt) as error: + # Was KeyboardInterrupt + a ``failed`` receipt: Ctrl-C after the commit point is an interrupt + # (exit 130), and the receipt keeps the truth the child already wrote (C3, item 5). + with pytest.raises(SystemExit) as error: main.cmd_update(SimpleNamespace(gateway=True)) + assert error.value.code == 130 assert interrupted, "child never published success before cancellation" assert children[0].poll() is not None assert children[0].stdout.closed assert waits and all(timeout is not None and timeout > 0 for timeout in waits) + assert isinstance(error.value.__cause__, KeyboardInterrupt) if cleanup_failure: - assert error.value.__cause__ is cleanup_error + assert error.value.__cause__.__cause__ is cleanup_error + assert "Interrupted after the code was updated" in stdout.getvalue() receipt = update_receipt.read_latest_receipt() assert receipt["update_id"] == request["receipt"]["update_id"] - assert receipt["outcome"] == "failed" - assert receipt["exit_code"] == 1 + assert receipt["outcome"] == "interrupted" # the child was stopped before it closed the run + assert receipt["exit_code"] == 130 assert receipt["stop_reason"].startswith("KeyboardInterrupt:") + # The gateway watcher still hears the interrupt (a non-zero exit is fine for Ctrl-C). assert marker.read_text().strip() == "1" lock = update_lock.UpdateLock() assert lock.acquire() @@ -366,11 +496,17 @@ def test_killed_selected_python_returns_signal_exit_status(transition): "def build_update_products(*a, **kw): os.kill(os.getpid(), signal.SIGKILL)\n" ) result = update_completion.run_completion(request) - assert result["exit_code"] == 137 + # Was 137: the tree already moved, so a prepared child that dies without a result is an owed + # ``completion`` follow-up (C3/A6), the tail obligation armed for the next launch. + assert result["exit_code"] == 0 + assert result["receipt"]["outcome"] == "success" + assert [f["step"] for f in result["receipt"]["followups"]] == ["completion"] + assert "exited 137" in result["receipt"]["followups"][0]["reason"] assert result["pm_receipt"]["update_id"] == request["receipt"]["update_id"] -def test_failed_build_preserves_exit_status_without_maintenance(transition): +def test_failed_build_after_commit_is_a_followup_and_later_steps_still_run(transition): + """Contract C3: the code is committed, so a failed product build cannot fail the update.""" from hermes_cli import update_completion root, git, old, new, request = transition @@ -380,18 +516,28 @@ def test_failed_build_preserves_exit_status_without_maintenance(transition): "def build_update_products(*a, **kw): raise subprocess.CalledProcessError(23, ['builder'])\n" ) result = update_completion.run_completion(request) - assert result["exit_code"] == 23 - assert result["receipt"]["outcome"] == "failed" + # Was 23 / "failed": a post-commit build failure exits 0 with a success receipt. + assert result["exit_code"] == 0 + assert result["receipt"]["outcome"] == "success" + # The failure is not silent: it is a named receipt follow-up. + assert [f["step"] for f in result["receipt"]["followups"]] == ["build"] + # The tail obligation stays armed so the next launch / `hermes update` rebuilds. assert (Path(request["home"]) / "completion-pending").read_text() == "owed" events = [json.loads(line)["name"] for line in (root / "events.jsonl").read_text().splitlines()] - assert "maintenance" not in events - assert "restart" not in events - assert "emergency_resume" in events - - -def test_prepare_failure_preserves_correlated_pm_receipt(transition, monkeypatch): - from types import SimpleNamespace - from hermes_cli import main, update_cmd, update_completion, update_receipt + # Was "not in": the failed build no longer skips maintenance (config migration) ... + assert "maintenance" in events + # ... nor the gateway restart and its verification. + assert "restart" in events and "verify" in events + # The tail was not complete, so the install stamp is not written. + assert "stamp" not in events + # The gateway watcher hears success: the code it runs is the committed code. + assert {"name": "exit_marker", "ok": True}.items() <= next( + json.loads(line) for line in (root / "events.jsonl").read_text().splitlines() + if json.loads(line)["name"] == "exit_marker").items() + + +def test_prepare_failure_preserves_correlated_pm_receipt(transition): + from hermes_cli import update_completion root, git, old, new, request = transition shutil.copy2(update_completion.__file__, root / "hermes_cli/update_completion.py") @@ -400,24 +546,17 @@ def test_prepare_failure_preserves_correlated_pm_receipt(transition, monkeypatch ) with (root / "pm/receipt.py").open("a") as stream: stream.write("last_for_update = lambda update_id: {'update_id': update_id, 'outcome': 'refused', 'refusal': {'reason': 'dependency refused'}}\n") - monkeypatch.setenv("HERMES_HOME", request["home"]) - monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) - monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) - monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) - - def complete(args, gateway_mode): - update_receipt.begin_update_receipt() - request["receipt"] = update_receipt._current.get().data - update_cmd._complete_source_update(request) - - monkeypatch.setattr(update_cmd, "_cmd_update_impl", complete) - with pytest.raises(SystemExit) as error: - main.cmd_update(SimpleNamespace(gateway=True)) - assert error.value.code == 1 - receipt = update_receipt.read_latest_receipt() - assert receipt["update_id"] == request["receipt"]["update_id"] - assert receipt["pm_sync_outcome"] == "refused" - assert receipt["pm_refusal"] == {"reason": "dependency refused"} + result = update_completion.run_completion(request) + # Was exit 1 / no receipt: the dependency sync runs after the tree moved, so its failure is a + # ``dependencies`` follow-up (A6) with the tail obligation armed — and the correlated pm + # refusal still travels back for the receipt. + assert result["exit_code"] == 0 + assert result["receipt"]["outcome"] == "success" + assert [f["step"] for f in result["receipt"]["followups"]] == ["dependencies"] + assert "dependency refused" in result["receipt"]["followups"][0]["reason"] + assert (Path(request["home"]) / "completion-pending").read_text() == "owed" + assert result["pm_receipt"] == {"update_id": request["receipt"]["update_id"], "outcome": "refused", + "refusal": {"reason": "dependency refused"}} def test_bootstrap_does_not_initialize_old_site_packages(transition, tmp_path, monkeypatch): diff --git a/tests/hermes_cli/test_update_config_migration_on_current.py b/tests/hermes_cli/test_update_config_migration_on_current.py index e511d9920ce6..d7bf507967ca 100644 --- a/tests/hermes_cli/test_update_config_migration_on_current.py +++ b/tests/hermes_cli/test_update_config_migration_on_current.py @@ -49,7 +49,12 @@ def prompt(text): monkeypatch.setattr('builtins.input', prompt) gateway_prompts = [] monkeypatch.setattr(update_cmd, '_gateway_prompt', lambda text, default: gateway_prompts.append((text, default)) or 'y') - update_cmd._check_and_apply_config_migration(assume_yes=case == 'yes', gateway_mode=case == 'gateway') + if case == 'migration-error': + # A failed write is owed, not silently completed: the owed-step guard needs the raise. + with pytest.raises(RuntimeError, match=r'v2 → v3 was not written: cannot write config'): + update_cmd._check_and_apply_config_migration(assume_yes=False, gateway_mode=False) + else: + update_cmd._check_and_apply_config_migration(assume_yes=case == 'yes', gateway_mode=case == 'gateway') assert calls == expected assert bool(prompts) is (case in {'tty-yes', 'tty-decline', 'eof', 'unicode'}) assert bool(gateway_prompts) is (case == 'gateway') @@ -90,3 +95,43 @@ def test_update_copies_bundled_skill_bytes_to_default_active_and_sibling(tmp_pat witness = next(bundled.rglob('SKILL.md')) for profile in homes: assert (profile / 'skills' / witness.relative_to(bundled)).read_bytes() == witness.read_bytes() + + +def test_active_config_write_failure_still_migrates_siblings_and_restores_snapshots(tmp_path, monkeypatch): + """A failed ACTIVE config write is owed debt, but the independent sibling migration and the + pre-update snapshot safety nets (the only holders of this run's snapshot ids) still run.""" + import json + from pathlib import Path + + from hermes_cli import backup, update_cmd_config + + root = tmp_path / '.hermes' + sibling = root / 'profiles' / 'sibling' + jobs = {'jobs': [{'id': 'nightly', 'name': 'nightly', 'schedule': '0 3 * * *', 'prompt': 'report'}]} + for home in (root, sibling): + (home / 'cron').mkdir(parents=True) + (home / 'config.yaml').write_text('_config_version: 48\n', encoding='utf-8') + (home / 'cron' / 'jobs.json').write_text(json.dumps(jobs), encoding='utf-8') + monkeypatch.setattr(Path, 'home', lambda: tmp_path) + monkeypatch.setenv('HERMES_HOME', str(root)) + active_snapshot = backup.create_quick_snapshot(label='pre-update', hermes_home=root) + monkeypatch.setattr(update_cmd_config, '_LAST_SIBLING_SNAPSHOTS', + backup.create_pre_update_snapshots_all_profiles(root)) + for home in (root, sibling): # the documented mutation-window loss the safety nets repair + (home / 'cron' / 'jobs.json').write_text('{"jobs": []}', encoding='utf-8') + real_replace = config.atomic_config_replace + + def replace(path, *args, **kwargs): + if Path(path) == root / 'config.yaml': + raise OSError('disk full') + return real_replace(path, *args, **kwargs) + + monkeypatch.setattr(config, 'atomic_config_replace', replace) + with pytest.raises(RuntimeError, match='was not written: disk full'): + update_cmd_config._check_and_apply_config_migration(pre_update_snapshot_id=active_snapshot) + for home in (root, sibling): + assert json.loads((home / 'cron' / 'jobs.json').read_text(encoding='utf-8')) == jobs + assert config.check_config_version()[0] == 48 # the active write stays owed + monkeypatch.setenv('HERMES_HOME', str(sibling)) + sibling_version, latest = config.check_config_version() + assert sibling_version == latest > 48 diff --git a/tests/hermes_cli/test_update_external_profile_reconciliation.py b/tests/hermes_cli/test_update_external_profile_reconciliation.py index 4ce7666af1e6..32439847d8e0 100644 --- a/tests/hermes_cli/test_update_external_profile_reconciliation.py +++ b/tests/hermes_cli/test_update_external_profile_reconciliation.py @@ -14,6 +14,7 @@ import pytest from hermes_cli import update_cmd_fleet as fleet +from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli import update_receipt from hermes_cli.update_inventory import ( RuntimeRecord, @@ -61,8 +62,8 @@ def external_profile_host(tmp_path, monkeypatch): monkeypatch.setattr(update_receipt, "_socket_identity", lambda _home: None) live = {root.resolve(): 1111, other_home.resolve(): 4242} monkeypatch.setattr("gateway.status.live_gateway_pid_for_home", lambda home: live.get(Path(home).resolve())) - monkeypatch.setattr(fleet, "_time", SimpleNamespace(sleep=lambda _s: None, monotonic=time.monotonic)) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_time", SimpleNamespace(sleep=lambda _s: None, monotonic=time.monotonic)) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd._finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None) return SimpleNamespace(other_checkout=other_checkout) @@ -79,7 +80,7 @@ def test_symlinked_external_profile_gateway_does_not_fail_update(external_profil plan = UpdatePlan(runtimes=[_gateway("default", 1111), _gateway("work", 4242)]) update_receipt.begin_update_receipt() - fleet._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( _restart_outcome(), _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, ) @@ -94,20 +95,22 @@ def test_symlinked_external_profile_gateway_does_not_fail_update(external_profil def test_external_profile_does_not_mask_a_missed_own_gateway(external_profile_host): - """The root's own gateway missed by the restart phase still fails the update.""" + """The root's own gateway missed by the restart phase is still an owed restart.""" plan = UpdatePlan(runtimes=[_gateway("default", 1111), _gateway("work", 4242)]) restart = _restart_outcome() restart.restarted_services.clear() update_receipt.begin_update_receipt() - with pytest.raises(SystemExit) as exc: - fleet._verify_fleet_after_update( - restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, - ) + # Contract C3: no longer SystemExit(1); the miss is flagged and owed instead. + fleet_verify._verify_fleet_after_update( + restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True, + ) - assert exc.value.code == 1 - outcomes = {o["profile"]: o["outcome"] for o in update_receipt.read_latest_receipt()["runtime_outcomes"]} + assert restart.incomplete + receipt = update_receipt.read_latest_receipt() + outcomes = {o["profile"]: o["outcome"] for o in receipt["runtime_outcomes"]} assert outcomes == {"default": "unaccounted", "work": "external"} + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] def test_external_requires_verified_pid_evidence(): diff --git a/tests/hermes_cli/test_update_finish.py b/tests/hermes_cli/test_update_finish.py index be9047b7447d..fe2874c2e048 100644 --- a/tests/hermes_cli/test_update_finish.py +++ b/tests/hermes_cli/test_update_finish.py @@ -154,6 +154,7 @@ def execute(module): from hermes_cli import update_cmd as update from hermes_cli import update_cmd_maint as maint from hermes_cli import update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as fleet_verify from hermes_cli import gateway_migrate from hermes_cli import macos_tcc_anchor from hermes_cli import source_build @@ -196,12 +197,14 @@ def maintain(**kwargs): maint._print_post_update_notices_and_self_heals = lambda: None maint._sync_profiles_after_update = lambda: None maint._print_bundled_skills_sync_report = lambda: None - fleet._print_legacy_units_warning = lambda: None + fleet_verify._print_legacy_units_warning = lambda: None maint._refresh_dashboard_after_update = lambda **kwargs: None gateway_migrate.maybe_auto_migrate_after_update = lambda: None update._surviving_pre_update_serve_runtimes = lambda plan: [] - fleet._collect_fleet_snapshot = lambda *args: [] + fleet_verify._collect_fleet_snapshot = lambda *args: [] def restart(plan, gateway_mode): + if fault == 'restart': + raise RuntimeError('systemctl restart timed out') assert isinstance(plan, UpdatePlan) assert isinstance(plan.runtimes[0], RuntimeRecord) assert plan.to_dict() == request['plan'] | { @@ -212,12 +215,26 @@ def restart(plan, gateway_mode): restarted_services=[], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set()) update._restart_gateway_fleet_after_update = restart + if fault == 'interrupt': + verify = update._verify_fleet_after_update + def verify_then_interrupt(*args, **kwargs): + verify(*args, **kwargs) # closes the run as a success + raise KeyboardInterrupt() + update._verify_fleet_after_update = verify_then_interrupt merge = update._resume_windows_gateways_and_merge_outcome def resume(outcome, token, gateway_mode): assert token == request['windows_resume'] (root / 'resumed-token.json').write_text(json.dumps(token)) return merge(outcome, token, gateway_mode) update._resume_windows_gateways_and_merge_outcome = resume + if fault == 'scm': + # The Windows service manager refuses the paused gateway's restart. + from hermes_cli import update_cmd_windows + def scm(token): + if token and token.get('resume_needed'): + raise RuntimeError('Could not restart Windows gateway service(s): HermesGatewayProbe') + module._resume_windows_gateways_after_update = scm + update_cmd_windows._resume_windows_gateways_after_update = scm spec.loader.exec_module = execute return spec sys.meta_path.insert(0, CompletionImports()) @@ -269,6 +286,64 @@ def test_failure_preserves_original_receipt_before_build(completion, fault): assert not (source / "build-environment.json").exists() +@pytest.mark.platforms("posix") +def test_refused_gateway_resume_after_commit_is_a_followup_not_exit_1(completion): + """C3 for the historical takeover child: the code is committed, so a Windows gateway resume + that the service manager refuses is an owed ``windows_resume`` follow-up, never exit 1.""" + source, home, request, context, result, run = completion + request["gateway_mode"] = True + # A real start time names the run's archive file, so the running and terminal records share one + # file (the fixture's placeholder falls back to the clock and can split them across a second). + request["receipt"]["started_at"] = "2026-10-04T12:00:00+00:00" + request["windows_resume"] = {"resume_needed": True, "profiles": {}, "unmapped": [], + "services": ["HermesGatewayProbe"], + "service_profiles": {"HermesGatewayProbe": "default"}} + context.write_text(json.dumps(request), encoding="utf-8") + child = run("scm") + assert child.returncode == 0, child.stdout + child.stderr + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) + assert receipt["update_id"] == request["update_id"] + assert receipt["outcome"] == "success" + assert "windows_resume" in [row["step"] for row in receipt["followups"]], child.stdout + child.stderr + assert (home / ".update_exit_code").read_text().strip() == "0" + # The attempt is reported: the historical parent must not replay it at its own exit. + assert json.loads(result.read_text())["resume_handled"] is True + + +@pytest.mark.platforms("posix") +def test_raising_fleet_restart_after_commit_is_a_followup_not_exit_1(completion): + """Review P3 (invariant 3): the takeover child's restart/verify run after the commit point, so + a raising fleet restart is an owed ``gateway_restart`` follow-up and exit 0, never exit 1 with a + ``failed`` receipt and a gateway watcher told 1.""" + source, home, request, context, result, run = completion + request["gateway_mode"] = True + request["receipt"]["started_at"] = "2026-10-04T12:00:00+00:00" + context.write_text(json.dumps(request), encoding="utf-8") + child = run("restart") + assert child.returncode == 0, child.stdout + child.stderr + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + assert receipt["update_id"] == request["update_id"] + assert receipt["outcome"] == "success" + owed = {row["step"]: row["reason"] for row in receipt["followups"]} + assert "systemctl restart timed out" in owed.get("gateway_restart", ""), child.stdout + child.stderr + assert (home / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0" + + +@pytest.mark.platforms("posix") +def test_interrupt_after_verification_closed_the_run_keeps_gateway_status_0(completion): + """Review regression 3: verification already finalized the takeover's run as ``success``; an + interrupt landing after it must not rewrite the gateway /update status to 1.""" + source, home, request, context, result, run = completion + request["gateway_mode"] = True + request["receipt"]["started_at"] = "2026-10-04T12:00:00+00:00" + context.write_text(json.dumps(request), encoding="utf-8") + child = run("interrupt") + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + assert receipt["update_id"] == request["update_id"] + assert receipt["outcome"] == "success", child.stdout + child.stderr + assert (home / ".update_exit_code").read_text(encoding="utf-8-sig").strip() == "0", child.stdout + child.stderr + + @pytest.mark.platforms("posix") @pytest.mark.parametrize("captured, installed, expected", [ (None, None, False), @@ -399,30 +474,42 @@ def test_selected_child_builds_and_finalizes_under_parent_lock(completion): @pytest.mark.platforms("posix") -def test_real_compiler_failure_retains_receipt_and_skips_completion(completion): +def test_real_compiler_failure_is_owed_build_and_completion_continues(completion): + """Contract C3 for the historical takeover child: the code is committed, so a failed product + build is an owed ``build`` follow-up, never a failed update. Maintenance (config migration) + and the fleet restart still run, the exit is 0, and the tail stays pending until it builds.""" + from hermes_cli.config_defaults import DEFAULT_CONFIG + from hermes_cli.venv_sync import completion_pending_path + import hermes_yaml + source, home, request, context, result, run = completion _npm_graph(source) - # A real TypeScript error: dependency installation and TUI succeed, but - # neither the web product nor config/fleet completion may claim success. + # A real TypeScript error: dependency installation and TUI succeed; the web product fails. _put(source, "web/src/main.ts", "const broken: string = 42; document.body.textContent = broken;\n") - config_before = (home / "config.yaml").read_bytes() context_before = context.read_bytes() child = run() - assert child.returncode == 1, child.stdout + child.stderr - # Contained output reports a failure through the step's tail on stdout. + assert child.returncode == 0, child.stdout + child.stderr assert "TypeScript build failed" in child.stdout + child.stderr assert (source / "npm-python.json").is_file() assert (source / "ui-tui/dist/entry.js").is_file() assert not (source / "hermes_cli/web_dist/index.html").exists() - assert not (source / "restarted-plan.json").exists() - assert (home / "config.yaml").read_bytes() == config_before + # Maintenance and the fleet restart were not skipped by the build failure. + config = hermes_yaml.safe_load((home / "config.yaml").read_text()) + assert config["_config_version"] == DEFAULT_CONFIG["_config_version"] + assert config["model"]["default"] == "retained-model" + assert (source / "restarted-plan.json").is_file() assert context.read_bytes() == context_before receipt = json.loads((home / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == "failed" - assert receipt["exit_code"] == child.returncode + assert receipt["outcome"] == "success" + assert [row["step"] for row in receipt["followups"]] == ["build"] + # Only a build follow-up marks the build stage failed (an owed config/dependency step does not). + assert {mark["name"]: mark["outcome"] for mark in receipt["stages"]}.get("build") == "failed" assert receipt["update_id"] == request["update_id"] assert receipt["steps"][0] == request["receipt"]["steps"][0] assert receipt["pm_venv_rebuild"] == request["pm_receipt"]["venv_rebuild"] + # The owed tail is retried by the next launch; nothing claims the tree was built. + assert completion_pending_path(source).is_file() + assert not (source / "install-stamp.json").exists() assert json.loads(result.read_text()) == {"resume_handled": True, "receipt_handled": True} diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index 42dd9aae9659..179c46f2d5a7 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -23,6 +23,7 @@ from __future__ import annotations +import json import types import pytest @@ -39,10 +40,10 @@ class TestCallSiteWiring: @pytest.mark.parametrize("had_gateway", [False, True], ids=["idle", "plan-saw-gateway"]) def test_empty_probe_settles_and_fails_only_when_rows_expected(self, monkeypatch, tmp_path, capsys, had_gateway): import json - from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_receipt + from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify, update_receipt monkeypatch.setattr(main, "PROJECT_ROOT", tmp_path) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) # Reconciliation is a separate guard; it must not supply this test's failure. @@ -59,7 +60,7 @@ def collect(**kwargs): events.append("probe") return [] - monkeypatch.setattr(fleet, "_time", types.SimpleNamespace(monotonic=lambda: now[0], sleep=sleep)) + monkeypatch.setattr(fleet_verify, "_time", types.SimpleNamespace(monotonic=lambda: now[0], sleep=sleep)) monkeypatch.setattr(update_receipt, "collect_fleet_versions", collect) restart = fleet._GatewayRestartOutcome( incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], @@ -69,22 +70,26 @@ def collect(**kwargs): update_receipt.begin_update_receipt() def verify(): - fleet._verify_fleet_after_update(restart, _pre_update_plan=plan, + fleet_verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None, update_complete=True) + cleared = [] + monkeypatch.setattr(fleet, "_clear_fleet_restart_pending_marker", lambda: cleared.append(True)) + # Contract C3: the code is committed, so verification never fails the update (was SystemExit(1)). + verify() if had_gateway: - with pytest.raises(SystemExit) as failure: - verify() - assert failure.value.code == 1 assert events[0] == "settle" assert events.count("probe") > 1 assert "returned no rows" in capsys.readouterr().out else: - verify() assert events == ["probe"] + # Fail-closed survives as an OWED restart: the fleet obligation is kept armed ... assert restart.incomplete is had_gateway + assert cleared == ([] if had_gateway else [True]) receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == ("partial" if had_gateway else "success") + # ... and the receipt is a success that names the owed step (was outcome "partial"). + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt.get("followups", [])] == (["gateway_restart"] if had_gateway else []) @@ -105,3 +110,131 @@ def test_unmapped_stops_are_not_expected_rows(): out.stopped_unmapped_pids.discard(102) pre, killed = out.fleet_probe_signals() assert _fleet_probe_expected_runtimes(_plan([]), pre, None, out.restarted_services, killed) + + +def test_unmapped_stop_keeps_the_restart_owed(monkeypatch, tmp_path): + # Codemap §6 V7/V12: an unmapped gateway stopped with no successor used to be "accounted for" + # (update exit 0, obligation cleared) and stayed down silently. It still predicts no row (the + # test above), but the restart is now OWED: a receipt follow-up and an armed fleet obligation. + import json + from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify, update_receipt + + monkeypatch.setattr(main, "PROJECT_ROOT", tmp_path) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) + monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: []) + cleared = [] + monkeypatch.setattr(fleet, "_clear_fleet_restart_pending_marker", lambda: cleared.append(True)) + out = fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[101], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={101}, stopped_unmapped_pids={101}, + ) + update_receipt.begin_update_receipt() + fleet_verify._verify_fleet_after_update(out, _pre_update_plan=_plan([]), _windows_gateway_resume=None, + update_complete=True) + receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] + assert "101" in receipt["followups"][0]["reason"] + assert cleared == [] + + +def test_unmapped_stop_debt_survives_startup_until_a_current_gateway_runs(monkeypatch, tmp_path): + # An unmapped gateway leaves no gateway_state.json, so on the next start the empty host looked + # gateway-less and the inventory-less obligation was discharged: the stopped gateway's restart + # debt vanished. Only a live gateway serving the checkout may settle it. + from hermes_cli import update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify + from hermes_cli import update_cmd_fleet_gatewayless as gatewayless, update_host_obligation as host, update_receipt + + monkeypatch.setattr(host, "host_obligation_path", lambda: tmp_path / "host-update-restart.json") + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) + monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) + monkeypatch.setattr(gatewayless, "host_owes_no_gateway_restart", lambda: True) + fleet_rows: list = [] + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: list(fleet_rows)) + fleet._write_fleet_restart_pending_marker(expected_sha="head") # armed by the pull, inventory-less + out = fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[101], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={101}, stopped_unmapped_pids={101}, + ) + update_receipt.begin_update_receipt() + fleet_verify._verify_fleet_after_update(out, _pre_update_plan=_plan([]), _windows_gateway_resume=None, + update_complete=True) + + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is True + assert fleet._pending_fleet_restart_needed(receipt={}, pending_manual=[]) is True # already-current update + assert fleet._fleet_restart_obligation_armed() + + fleet_rows.append({"profile": "default", "state": "current", "code_sha": "head"}) # `hermes gateway run` + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is False + assert not fleet._fleet_restart_obligation_armed() + + +def test_unmapped_stop_debt_is_not_settled_by_the_mapped_gateways_restart(monkeypatch, tmp_path): + # The inventory owes `default` AND an unmapped gateway: `default` coming back current is the + # successor of `default`, not of the unmapped one, so the debt stays until a further local + # gateway (the unmapped one's successor, whatever profile it names) runs the checkout. + from hermes_cli import update_cmd_fleet as fleet, update_host_obligation as host, update_receipt + + monkeypatch.setattr(host, "host_obligation_path", lambda: tmp_path / "host-update-restart.json") + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "head") + fleet._write_fleet_restart_pending_marker(expected_sha="head", runtimes=[ + {"kind": "gateway", "profile": "default", "pid": 7}, + {"kind": "gateway", "profile": None, "pid": 101, "stopped_unmapped": True}, + ]) + fleet_rows = [{"profile": "default", "state": "current", "code_sha": "head", "pid": 8}] + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: list(fleet_rows)) + + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is True + assert fleet._fleet_restart_obligation_armed() + + fleet_rows.append({"profile": "work", "state": "current", "code_sha": "head", "pid": 9}) + assert fleet._update_owes_fleet_restart(receipt={}, pending_manual=[]) is False + assert not fleet._fleet_restart_obligation_armed() + + +def test_same_sha_retry_adds_newly_stopped_unmapped_debt_to_an_existing_inventory(monkeypatch, tmp_path): + # A same-SHA catch-up keeps the standing inventory (gateway `default`, still down). If that retry + # also stops an unmapped gateway, its debt must join the inventory: otherwise `default` coming + # back settles the obligation while the unmapped gateway still has no successor. + from hermes_cli import update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as fleet_verify + from hermes_cli import update_host_obligation as host, update_receipt + from hermes_cli.update_inventory import UpdatePlan + + monkeypatch.setattr(host, "host_obligation_path", lambda: tmp_path / "host-update-restart.json") + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: "head") + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None) + monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) + monkeypatch.setattr(fleet_verify, "_FLEET_PROBE_SETTLE_TIMEOUT_SECONDS", 0) # A stays down: one poll + # Signal delivery is the external boundary: the down row has no process to drain. + monkeypatch.setattr("hermes_cli.update_cmd_stale_survivors.signal_stale_fleet_survivors", lambda *a: None) + fleet_rows = [{"profile": "default", "state": "down", "code_sha": "old", "pid": 7}] + monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: list(fleet_rows)) + fleet._write_fleet_restart_pending_marker(expected_sha="head", runtimes=[ + {"kind": "gateway", "profile": "default", "pid": 7}]) + out = fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[7999], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={7999}, stopped_unmapped_pids={7999}, + ) + update_receipt.begin_update_receipt() + fleet_verify._verify_fleet_after_update( + out, _pre_update_plan=UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default", pid=7)]), + _windows_gateway_resume=None, update_complete=True) + + rows = json.loads(fleet._obligation_fields()["inventory"])["runtimes"] + assert {"kind": "gateway", "profile": "default", "pid": 7} in rows # earlier debt kept + assert {"kind": "gateway", "profile": None, "pid": 7999, "stopped_unmapped": True} in rows + fleet_rows[:] = [{"profile": "default", "state": "current", "code_sha": "head", "pid": 8}] + assert fleet._marker_only_restart_obsolete() is False # `default`'s successor is not 7999's + assert fleet._fleet_restart_obligation_armed() + fleet_rows.append({"profile": "work", "state": "current", "code_sha": "head", "pid": 9}) + assert fleet._marker_only_restart_obsolete() is True diff --git a/tests/hermes_cli/test_update_fleet_completion.py b/tests/hermes_cli/test_update_fleet_completion.py index 984bfb0b1ffb..b423756dd225 100644 --- a/tests/hermes_cli/test_update_fleet_completion.py +++ b/tests/hermes_cli/test_update_fleet_completion.py @@ -1,11 +1,10 @@ """SQLite completion and fleet verification remain independent update outcomes.""" -from contextlib import nullcontext import json import pytest -from hermes_cli import update_cmd, update_cmd_fleet, update_cmd_maint, update_receipt +from hermes_cli import update_cmd, update_cmd_fleet, update_cmd_fleet_verify as fleet_verify, update_cmd_maint, update_receipt from hermes_constants import get_hermes_home @@ -23,7 +22,7 @@ def test_fleet_completion_preserves_runtime_verdict_and_restart_obligation( restarted_services=["hermes-gateway"], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(), ) - monkeypatch.setattr(update_cmd_fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) monkeypatch.setattr( update_cmd_maint, "_refresh_dashboard_after_update", @@ -39,23 +38,24 @@ def collect(outcome, rows_expected): assert rows_expected is True return snapshot - monkeypatch.setattr(update_cmd_fleet, "_collect_fleet_snapshot", collect) + monkeypatch.setattr(fleet_verify, "_collect_fleet_snapshot", collect) update_cmd_fleet._write_fleet_restart_pending_marker() assert update_cmd_fleet._fleet_restart_obligation_armed() healthy = update_complete and state == "current" with update_receipt.update_receipt_scope(): update_receipt.begin_update_receipt() - with nullcontext() if healthy else pytest.raises(SystemExit) as exc: - update_cmd_fleet._verify_fleet_after_update( - restart, _pre_update_plan=None, _windows_gateway_resume=None, - update_complete=update_complete, - ) - if not healthy: - assert exc.value.code == 1 + # Contract C3: neither verdict fails the committed update any more (was SystemExit(1)). + fleet_verify._verify_fleet_after_update( + restart, _pre_update_plan=None, _windows_gateway_resume=None, + update_complete=update_complete, + ) receipt = json.loads((get_hermes_home() / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == ("success" if healthy else "partial") + # Was "partial" when not healthy: the run is a success; an owed fleet restart is a follow-up. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt.get("followups", [])] == ([] if state == "current" else ["gateway_restart"]) assert receipt["fleet"] == snapshot + # The two verdicts stay independent: only the fleet keeps the restart obligation armed. assert restart.incomplete is (state != "current") assert update_cmd_fleet._fleet_restart_obligation_armed() is (state != "current") assert migrated == ([True] if healthy else []) diff --git a/tests/hermes_cli/test_update_fleet_restart_pending.py b/tests/hermes_cli/test_update_fleet_restart_pending.py index 357dbd8c79ab..d7417177170a 100644 --- a/tests/hermes_cli/test_update_fleet_restart_pending.py +++ b/tests/hermes_cli/test_update_fleet_restart_pending.py @@ -412,8 +412,9 @@ def test_clean_update_escalates_surviving_serve_as_unaccounted( ``hermes-gateway.service``) and an unmanaged ``serve`` on the same default profile. The serve survives the update as the SAME process, so the update must (1) warn, (2) reconcile it as ``unaccounted`` instead of - borrowing the gateway's restart, and (3) exit 1 with a ``partial`` - receipt — not print a clean success.""" + borrowing the gateway's restart, and (3) under contract C3 (the code is + committed) end ``success`` with an owed ``gateway_restart`` follow-up and + the restart obligation still armed — never a silent clean success.""" from hermes_cli.update_inventory import ( RuntimeRecord, UpdatePlan, _restart_mechanism, ) @@ -460,18 +461,24 @@ def _match(p, **kw): lambda **_k: [{"pid": 5555, "purpose": "serve", "create_time": 1000.0}], ) - with pytest.raises(SystemExit) as excinfo: - hermes_main.cmd_update(args) - assert excinfo.value.code == 1 + # Was SystemExit(1): the committed update no longer fails (contract C3). + hermes_main.cmd_update(args) out = capsys.readouterr().out + # (1) still warned, unchanged. assert "pid 5555" in out and "pre-update code" in out assert "Planned runtimes the restart phase never touched" in out assert "serve [default] pid 5555" in out + assert "follow-up 'gateway_restart'" in out latest = get_hermes_home() / "logs" / "update_receipts" / "latest.json" receipt = json.loads(latest.read_text(encoding="utf-8")) - assert receipt["outcome"] == "partial" + # (3) was "partial": a success that names the owed restart. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["gateway_restart"] + from hermes_cli import update_cmd_fleet + assert update_cmd_fleet._fleet_restart_obligation_armed() + # (2) the reconciliation verdict is unchanged. by_pid = {o["pid"]: o["outcome"] for o in receipt["runtime_outcomes"]} assert by_pid == {4444: "restarted", 5555: "unaccounted"} @@ -552,9 +559,11 @@ def _interrupt(*_a, **_k): monkeypatch.setattr(hermes_main, "_clear_bytecode_cache", _interrupt) - with pytest.raises(KeyboardInterrupt): + # Was KeyboardInterrupt: Ctrl-C after the commit point exits 130 as an interrupt, not a failure. + with pytest.raises(SystemExit) as exc: hermes_main.cmd_update(args) + assert exc.value.code == 130 assert update_cmd_fleet._fleet_restart_obligation_armed() record = json.loads(host_obligation.host_obligation_path().read_text(encoding="utf-8")) assert record["expected_sha"] == "def456" diff --git a/tests/hermes_cli/test_update_fleet_snapshot_settle.py b/tests/hermes_cli/test_update_fleet_snapshot_settle.py index 62b55d3494d3..894b152da53a 100644 --- a/tests/hermes_cli/test_update_fleet_snapshot_settle.py +++ b/tests/hermes_cli/test_update_fleet_snapshot_settle.py @@ -14,6 +14,7 @@ from hermes_cli import update_cmd import hermes_cli.update_cmd_fleet as update_cmd_fleet +import hermes_cli.update_cmd_fleet_verify as fleet_verify from hermes_constants import get_hermes_home @@ -45,11 +46,11 @@ def test_snapshot_waits_for_late_current_gateway_state(monkeypatch) -> None: restart = SimpleNamespace(pre_restart_gateway_pids=[101]) - result = update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) + result = fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) assert result == [expected] assert clock.now > 30.0 - assert clock.now <= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + assert clock.now <= fleet_verify._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> None: @@ -64,7 +65,7 @@ def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> stdout="LoadState=loaded\nActiveState=failed\n", stderr="", returncode=0)) restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"user/hermes-gateway.service"}) - assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [] + assert fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) == [] assert clock.now < 30.0 @@ -85,11 +86,11 @@ def systemctl(cmd, *, timeout): stderr="", returncode=0) monkeypatch.setattr(update_cmd_fleet, "_systemctl", systemctl) - assert update_cmd_fleet._restarted_units_gone(("system/hermes-gateway.service",)) is False + assert fleet_verify._restarted_units_gone(("system/hermes-gateway.service",)) is False restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"system/hermes-gateway.service"}) - assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [] - assert clock.now >= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + assert fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) == [] + assert clock.now >= fleet_verify._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS def test_snapshot_waits_for_the_relaunched_pid_to_publish_its_identity(monkeypatch) -> None: @@ -104,20 +105,20 @@ def test_snapshot_waits_for_the_relaunched_pid_to_publish_its_identity(monkeypat monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: next(snapshots)) restart = SimpleNamespace(pre_restart_gateway_pids=[32512], restarted_scoped_units=set()) - assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [current] + assert fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) == [current] assert clock.now == 12.0 # Control: the same unknown row for a pid that was already running pre-update is settled as-is. clock.now = 0.0 monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: [dict(unknown)]) survivor = SimpleNamespace(pre_restart_gateway_pids=[34516], restarted_scoped_units=set()) - result = update_cmd_fleet._collect_fleet_snapshot(survivor, rows_expected=True) + result = fleet_verify._collect_fleet_snapshot(survivor, rows_expected=True) assert clock.now == 2.0 and "identity_pending" not in result[0] # Still unknown at the deadline: flagged so the matrix prints restart-aware copy, never "predates". clock.now = 0.0 - result = update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) - assert clock.now >= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS + result = fleet_verify._collect_fleet_snapshot(restart, rows_expected=True) + assert clock.now >= fleet_verify._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS assert result[0]["identity_pending"] is True @@ -139,7 +140,7 @@ def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> "hermes_cli.update_receipt.print_fleet_version_matrix", lambda _fleet: False, ) - monkeypatch.setattr(update_cmd_fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(update_cmd, "_finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda _plan: []) monkeypatch.setattr(update_cmd, "_warn_stale_serve_runtimes", lambda _rows: None) @@ -157,7 +158,7 @@ def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> fleet_probe_signals=lambda: ([101], set()), ) - update_cmd_fleet._verify_fleet_after_update( + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, diff --git a/tests/hermes_cli/test_update_hangup_protection.py b/tests/hermes_cli/test_update_hangup_protection.py index a4b6c5a54133..b924353f09cd 100644 --- a/tests/hermes_cli/test_update_hangup_protection.py +++ b/tests/hermes_cli/test_update_hangup_protection.py @@ -118,7 +118,7 @@ def test_wraps_stdout_and_stderr_with_mirror(self, tmp_path, monkeypatch): def test_non_fatal_if_log_setup_fails(self, monkeypatch): - """If get_hermes_home() raises, stdio must be left untouched but SIGHUP still handled.""" + """If the root-home lookup raises, stdio must be left untouched but SIGHUP still handled.""" prev_out, prev_err = sys.stdout, sys.stderr def _boom(): @@ -126,7 +126,7 @@ def _boom(): # Patch the import inside _install_hangup_protection. monkeypatch.setattr( - "hermes_cli.config.get_hermes_home", _boom, raising=True + "hermes_constants.get_default_hermes_root", _boom, raising=True ) original_handler = ( diff --git a/tests/hermes_cli/test_update_post_commit_followups.py b/tests/hermes_cli/test_update_post_commit_followups.py new file mode 100644 index 000000000000..43c7d3164314 --- /dev/null +++ b/tests/hermes_cli/test_update_post_commit_followups.py @@ -0,0 +1,322 @@ +"""Contract C3: after the commit point nothing fails `hermes update`. + +Post-commit step failures are ⚠ + a receipt follow-up + an armed obligation; the receipt +is durable while the run is open; receipts resolve to the root home. +""" + +from __future__ import annotations + +import json +import os +import subprocess + +import pytest + +from hermes_cli import update_receipt + + +def _latest(home): + return json.loads((home / "logs/update_receipts/latest.json").read_text()) + + +def test_open_receipt_is_durably_running_and_finalizes_in_place(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + running = _latest(tmp_path) + assert running["outcome"] == "running" and running["pid"] == os.getpid() + run_files = list((tmp_path / "logs/update_receipts").glob(f"update_*_{running['update_id']}.json")) + assert len(run_files) == 1 + update_receipt.record_stage("apply", "success") + assert [s["name"] for s in _latest(tmp_path)["stages"]] == ["apply"] + update_receipt.record_followup("build", "web UI build: npm exited 1") + update_receipt.finalize_update_receipt("success") + final = _latest(tmp_path) + assert final["outcome"] == "success" + assert [(f["step"], f["reason"]) for f in final["followups"]] == [("build", "web UI build: npm exited 1")] + # The terminal record replaced the running one in the SAME archive file (no duplicate per run). + assert list((tmp_path / "logs/update_receipts").glob(f"update_*_{final['update_id']}.json")) == run_files + + +@pytest.mark.parametrize("finish", [lambda: update_receipt.finalize_update_receipt("success"), + lambda: update_receipt.finalize_pending_update_receipt(1, "local changes parked")]) +def test_parked_local_changes_never_finalize_as_success(tmp_path, monkeypatch, finish): + # A follow-up is retried by the next launch; a stash whose restore conflicted is not, so the + # committed run must stay partial (#122557) on both the verify path and the boundary net. + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + update_receipt.record_user_action("local_changes", "⚠ hermes update stashed 1 local modification(s)\n Stash ref: abc") + finish() + final = _latest(tmp_path) + assert final["outcome"] == "partial" + assert final["user_action"] == {"step": "local_changes", + "reason": "⚠ hermes update stashed 1 local modification(s) Stash ref: abc"} + + +def test_dead_running_record_is_reported_interrupted_by_the_next_run(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + dead = subprocess.Popen(["true"]) + dead.wait() + update_receipt.begin_update_receipt() + update_receipt.record_stage("deps", "success") + killed = update_receipt._current.get().data + killed_id = killed["update_id"] + # Simulate the kill: the record on disk names a process that is gone. + for path in (tmp_path / "logs/update_receipts").glob("*.json"): + data = json.loads(path.read_text()) + data.update(pid=dead.pid, writer_pid=dead.pid, pid_create_time=None) + path.write_text(json.dumps(data)) + update_receipt._current.set(None) + + update_receipt.begin_update_receipt() + out = capsys.readouterr().out + assert "interrupted" in out and "last stage: deps" in out + (archived,) = (tmp_path / "logs/update_receipts").glob(f"update_*_{killed_id}.json") + assert json.loads(archived.read_text())["outcome"] == "interrupted" + assert _latest(tmp_path)["update_id"] != killed_id + update_receipt.finalize_update_receipt("success") + + +def test_live_running_record_is_not_reclaimed(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + own_id = update_receipt._current.get().data["update_id"] + for path in (tmp_path / "logs/update_receipts").glob("*.json"): + data = json.loads(path.read_text()) + from hermes_cli.process_identity import _process_create_time + + parent_time = _process_create_time(os.getppid()) + data.update(pid=os.getppid(), writer_pid=os.getppid(), + pid_create_time=parent_time, writer_create_time=parent_time) + path.write_text(json.dumps(data)) + assert update_receipt.reconcile_interrupted_runs() == [] + (archived,) = (tmp_path / "logs/update_receipts").glob(f"update_*_{own_id}.json") + assert json.loads(archived.read_text())["outcome"] == "running" + update_receipt.finalize_update_receipt("success") + + +def test_profile_process_writes_receipts_to_the_root_home(tmp_path, monkeypatch): + root = tmp_path / ".hermes" + profile = root / "profiles/work" + profile.mkdir(parents=True) + monkeypatch.setattr("hermes_constants._get_platform_default_hermes_home", lambda: root) + monkeypatch.setattr("hermes_constants._default_hermes_root_memo", None) + monkeypatch.setenv("HERMES_HOME", str(profile)) + assert update_receipt._receipt_dir() == root / "logs/update_receipts" + + +def test_failed_web_build_does_not_skip_the_tui(tmp_path, monkeypatch, capsys): + from hermes_cli import source_build + + built = [] + monkeypatch.setattr("hermes_cli.main_install_repair._install_configured_features_missing_deps", lambda root: None) + monkeypatch.setattr("hermes_cli.update_stage.publish_stage", lambda text: None) + monkeypatch.setattr("hermes_cli.memory_provider_migration.migrate_all_homes", lambda: None) + monkeypatch.setattr(source_build, "source_frontends", lambda root: ("web", "ui-tui")) + monkeypatch.setattr(source_build, "source_build_env", lambda **kw: {"PATH": ""}) + monkeypatch.setattr(source_build, "prepare_source_dependencies", lambda *a, **kw: None) + monkeypatch.setattr(source_build, "source_product_current", lambda *a: False) + monkeypatch.setattr(source_build, "build_source_tui", lambda root, env: built.append("tui")) + + def broken_web(root, env): + raise subprocess.CalledProcessError(1, ["npm", "run", "build"]) + + monkeypatch.setattr(source_build, "build_source_web", broken_web) + with pytest.raises(source_build.ProductBuildError) as failure: + source_build.build_update_products(tmp_path, desktop=False) + assert built == ["tui"] + assert [name for name, _ in failure.value.failures] == ["web UI build"] + assert "npm run build exited 1" in str(failure.value) + + +def test_failed_config_migration_is_owed_and_later_maintenance_runs(tmp_path, monkeypatch, capsys): + from hermes_cli import update_cmd, update_cmd_maint as maint + + calls = [] + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + for name in ("_verify_and_restore_state_dbs_post_update", "_invalidate_live_plugin_catalog_caches", + "_print_bundled_skills_sync_report", "_sync_profiles_after_update"): + monkeypatch.setattr(maint, name, lambda name=name: calls.append(name)) + monkeypatch.setattr("hermes_cli.gitlock.fetch_full_commit_graph", lambda *a, **kw: False) + monkeypatch.setattr("hermes_cli.model_catalog.seed_cache_from_checkout", lambda root: False) + + def broken_migration(**kwargs): + raise SystemExit(1) # a migration helper that exits must not fail the committed update + + monkeypatch.setattr(update_cmd, "_check_and_apply_config_migration", broken_migration) + monkeypatch.setattr(maint, "_print_verified_update_completion", lambda message: calls.append("verdict") or True) + monkeypatch.setattr(maint, "_print_post_update_notices_and_self_heals", lambda: calls.append("notices")) + update_receipt.begin_update_receipt() + owed: list = [] + assert maint._run_post_update_maintenance( + assume_yes=True, gateway_mode=False, pre_update_snapshot_id=None, + had_desktop_app_before_update=False, pre_update_version=None, followups=owed) is True + assert [step for step, _ in owed] == ["config_migration"] + assert calls[-2:] == ["verdict", "notices"] + assert "⚠ Update follow-up 'config_migration'" in capsys.readouterr().out + update_receipt.finalize_update_receipt("success") + assert [f["step"] for f in _latest(tmp_path)["followups"]] == ["config_migration"] + + +def test_owed_restart_names_its_gateways_so_a_dead_fleet_cannot_discharge_it(tmp_path, monkeypatch): + # A gateway that died at boot leaves no live row. An inventory-less obligation would be settled + # by the gateway-less discharge, silencing the warning that replaces exit 1 under C3. + from hermes_cli import update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as fleet_verify + from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("HERMES_GATEWAY_LOCK_DIR", str(tmp_path / "locks")) + fleet._write_fleet_restart_pending_marker(expected_sha="a" * 40) + plan = UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default", pid=4242), + RuntimeRecord(kind="dashboard", profile="default", pid=4343)]) + fleet_verify._record_owed_gateway_inventory(plan) + + inventory = json.loads(fleet._obligation_fields()["inventory"]) + assert [(r["kind"], r["profile"]) for r in inventory["runtimes"]] == [("gateway", "default")] + # HEAD still holds the pulled code, so only the fleet evidence can settle it. + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "a" * 40) + monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda: []) + assert fleet._marker_only_restart_obsolete() is False + assert fleet._fleet_restart_obligation_armed() + + +def test_owed_restart_rearms_a_settled_obligation_and_a_later_run_keeps_owing_it(tmp_path, monkeypatch): + # A pre-restart probe can settle an inventory-less record; the owed restart must re-arm it, and + # a later verify that finds nothing to restart must keep owing the named gateway. + from hermes_cli import update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as fleet_verify + from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("HERMES_GATEWAY_LOCK_DIR", str(tmp_path / "locks")) + monkeypatch.setattr(fleet, "_current_checkout_sha", lambda: "b" * 40) + monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda: []) + assert not fleet._fleet_restart_obligation_armed() + fleet_verify._record_owed_gateway_inventory(UpdatePlan(runtimes=[RuntimeRecord(kind="gateway", profile="default")])) + assert fleet._obligation_fields()["expected_sha"] == "b" * 40 + # A later verify with nothing live to restart still owes the named gateway (and keeps it armed). + assert fleet_verify._named_gateways_still_owed() + assert fleet._fleet_restart_obligation_armed() + + +def test_followup_line_is_the_one_both_desktop_handoffs_parse(tmp_path, monkeypatch, capsys): + """The hand-offs read owed follow-ups from this printed line (review regression 1): pin the + producer against the exact parsers in posix.sh (sed) and windows.ps1 (.NET regex).""" + import re + from pathlib import Path + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.record_followup("gateway_restart", "x" * 900) + update_receipt.record_followup("windows_resume", "Windows gateway recovery failed: 'quoted' reason") + out = capsys.readouterr().out + scripts = Path(__file__).resolve().parents[2] / "scripts/desktop-update" + posix = (scripts / "posix.sh").read_text(encoding="utf-8") + function = re.search(r"^owed_followup_steps\(\) \{.*?^\}", posix, re.S | re.M).group(0) + steps = subprocess.run(["bash", "-c", f"{function}\nowed_followup_steps"], env={**os.environ, "OUT": out}, + capture_output=True, text=True, encoding="utf-8", check=True).stdout.strip() + assert steps == "gateway_restart windows_resume" + pattern = re.search(r"\[regex\]::Matches\(\(\$res\.Output -join \"`n\"\), \"([^\"]+)\"\)", + (scripts / "windows.ps1").read_text(encoding="utf-8")).group(1) + assert [m.group(1) for m in re.finditer(pattern, out)] == ["gateway_restart", "windows_resume"] + + +def test_interrupt_after_the_run_closed_as_success_never_tells_the_gateway_1(tmp_path, monkeypatch): + """Review regression 3, the ``cmd_update`` boundary: once this run's receipt says ``success``, + an interrupt escaping afterwards must not write 1 to the gateway /update status.""" + from types import SimpleNamespace + from hermes_cli import main, update_cmd, update_owning_install + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) + monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) + monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) + monkeypatch.setattr(update_owning_install, "retarget_to_owning_install", lambda root: None) # not the target + + def committed_then_interrupted(args, gateway_mode): + update_receipt.begin_update_receipt() + update_receipt.finalize_update_receipt("success") + raise KeyboardInterrupt() + + monkeypatch.setattr(update_cmd, "_cmd_update_impl", committed_then_interrupted) + with pytest.raises(KeyboardInterrupt): + main.cmd_update(SimpleNamespace(gateway=True)) + assert _latest(tmp_path)["outcome"] == "success" + assert (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "0" + + +def test_a_failure_before_the_run_closed_still_tells_the_gateway_1(tmp_path, monkeypatch): + from types import SimpleNamespace + from hermes_cli import main, update_cmd, update_owning_install + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(main, "_update_preflight_handled", lambda args: False) + monkeypatch.setattr(main, "_install_hangup_protection", lambda **kw: None) + monkeypatch.setattr(main, "_finalize_update_output", lambda state: None) + monkeypatch.setattr(update_owning_install, "retarget_to_owning_install", lambda root: None) # not the target + + def interrupted_while_open(args, gateway_mode): + update_receipt.begin_update_receipt() + raise KeyboardInterrupt() + + monkeypatch.setattr(update_cmd, "_cmd_update_impl", interrupted_while_open) + with pytest.raises(KeyboardInterrupt): + main.cmd_update(SimpleNamespace(gateway=True)) + assert (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "1" + + +def test_completion_child_interrupted_after_verification_answers_its_success(tmp_path, monkeypatch): + """Review regression 3, the selected completion child: verification finalized ``success`` and + then an interrupt landed. Exit, result and gateway status follow the receipt (0), not 130/1.""" + from hermes_cli import update_completion + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + update_receipt.begin_update_receipt() + data = dict(update_receipt._current.get().data) + update_receipt._current.set(None) + + def verified_then_interrupted(request): + update_receipt.finalize_update_receipt("success") + raise KeyboardInterrupt() + + monkeypatch.setattr(update_completion, "_complete_selected", verified_then_interrupted) + request = {"receipt": data, "pm_receipt": None, "gateway_mode": True, "windows_resume": None} + result_path = tmp_path / "result.json" + assert update_completion._finish(request, result_path) == 0 + answer = json.loads(result_path.read_text(encoding="utf-8")) + assert answer["exit_code"] == 0 and answer["receipt"]["outcome"] == "success" + assert not (tmp_path / ".update_exit_code").exists() or \ + (tmp_path / ".update_exit_code").read_text(encoding="utf-8").strip() == "0" + + +def test_both_completion_children_run_in_utf8_mode(tmp_path, monkeypatch): + """-I drops PYTHONIOENCODING, and both completion children print the ✓/⚠ follow-up protocol + into a pipe: each argv carries -X utf8 (win-utf8 review; the bootstrap child got it first).""" + import pm + from pm import client, environments, receipt + from hermes_cli import gitlock, update_completion, venv_sync + + calls = [] + + def popen(command, **kwargs): + calls.append(command) + raise OSError("stop after capturing the argv") + + monkeypatch.setattr(subprocess, "call", lambda command, **kw: calls.append(command) or 0) + monkeypatch.setattr(subprocess, "Popen", popen) + # The treeless-checkout conversion's git probe would be captured as a third spawn. + for module, name in ((venv_sync, "refuse_foreign_owned_venv"), (venv_sync, "arm_completion"), + (venv_sync, "collect_superseded_generations"), (client, "ensure_tools_for_sync"), + (pm, "sync_venv"), (gitlock, "convert_treeless_checkout_first")): + monkeypatch.setattr(module, name, lambda *a, **k: None) + monkeypatch.setattr(receipt, "last_for_update", lambda *a, **k: None) + monkeypatch.setattr(environments, "project_python", lambda root: "python") + monkeypatch.setattr(environments, "activation_environment", lambda root: {}) + monkeypatch.setattr(update_completion, "_settle_after_commit", lambda *a: 0) + request = {"source": str(tmp_path), "receipt": {"update_id": "u1"}, "bytecode_cache": str(tmp_path / "bc")} + update_completion._prepare(request, tmp_path / "request.json", tmp_path / "result.json") + monkeypatch.setattr(update_completion.sys, "platform", "win32") # skip the Linux libatomic pre-install + with pytest.raises(OSError): + update_completion.run_completion({**request, "home": str(tmp_path)}) + assert len(calls) == 2 + for command in calls: + assert command[command.index("-I"):].count("utf8") == 1 and "-X" in command, command diff --git a/tests/hermes_cli/test_update_post_commit_safety.py b/tests/hermes_cli/test_update_post_commit_safety.py new file mode 100644 index 000000000000..7b7c4d92193f --- /dev/null +++ b/tests/hermes_cli/test_update_post_commit_safety.py @@ -0,0 +1,403 @@ +"""Post-commit completion keeps exit status, runtime safety and owed debt as separate facts. + +Each test runs the real completion (or receipt rotation) in a separate process against a +disposable home and a real git checkout. The only seams are machine boundaries: the selected +interpreter's linked SQLite (a shim interpreter that answers the SQLite probe and execs the real +Python for everything else), and the gateway fleet restart / dashboard / multiplex migration +service calls, which are replaced by recorders so the test can never touch a host's services. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def _child_env(**values: str) -> dict[str, str]: + """A fresh env for a child of THIS interpreter: same dependency path, no hermes state.""" + env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} + # The test runner may provide dependencies through sys.path rather than site-packages. + env["PYTHONPATH"] = os.pathsep.join(path for path in sys.path if path) + env.update(values) + return env + +_CHILD = textwrap.dedent(''' + import copy, json, os, subprocess, sys + from pathlib import Path + root_checkout, cell, mode = sys.argv[1:4] + sys.path.insert(0, root_checkout) + cell = Path(cell) + source = cell / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\\nname="completion-probe"\\nversion="0.0.0"\\n') + git = ["git", "-c", "user.email=probe@example.invalid", "-c", "user.name=Probe", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run([*git, "-C", str(source), "commit", "-qm", "fixture"], check=True) + if mode.startswith("sqlite"): + # The selected interpreter is the shim: its SQLite probe is the runtime-safety fact. + sys.executable = os.environ["PROBE_SHIM"] + import hermes_yaml as yaml + from hermes_cli.config import DEFAULT_CONFIG + home = Path(os.environ["HERMES_HOME"]) + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text(yaml.safe_dump({ + "_config_version": DEFAULT_CONFIG["_config_version"] - 1, + "telemetry": {"shared_metrics": {"enabled": False}}, + "updates": {"refresh_cua_driver": False}})) + # Service boundaries: record instead of touching the host's gateways/dashboards. + from hermes_cli import gateway_migrate, update_cmd, update_cmd_fleet as fleet + from hermes_cli import update_cmd_fleet_verify as verify, update_cmd_maint as maint + entered = cell / "migration-entered" + gateway_migrate.maybe_auto_migrate_after_update = lambda: entered.write_text("entered") + real_restart = update_cmd._restart_gateway_fleet_after_update + update_cmd._restart_gateway_fleet_after_update = lambda plan, gateway_mode: fleet._GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set()) + update_cmd._fleet_restart_skip_reason = lambda plan: None + update_cmd._surviving_pre_update_serve_runtimes = lambda plan: [] + maint._refresh_dashboard_after_update = lambda **kwargs: None + verify._print_legacy_units_warning = lambda: None + verify._collect_fleet_snapshot = lambda *args: [] + windows_resume = None + marker = home / ".update_exit_code" + if mode.startswith("ipc-"): + # Barrier: verification starts after the restart phase returned. A gateway /update reader + # polling the marker can consume it from here on, before the run ever finalizes. + verify._print_legacy_units_warning = lambda: (cell / "marker-at-verify").write_text( + marker.read_text() if marker.exists() else "absent") + if mode in ("ipc-failed-unit", "ipc-abort"): + # The real restart phase; only the host's service manager and process table are replaced. + from hermes_cli import gateway as gateway_module + update_cmd._restart_gateway_fleet_after_update = real_restart + fleet._restart_manual_gateways = lambda out, budget: None + fleet._force_kill_stuck_gateways = lambda pids: None + if mode == "ipc-failed-unit": + gateway_module.find_gateway_pids = lambda **kwargs: [] + fleet._restart_systemd_gateway_units = lambda restarted, failed, *rest: failed.append( + "hermes-gateway-probe.service") + else: + def unavailable(*args, **kwargs): + raise RuntimeError("systemctl unavailable") + gateway_module.find_gateway_pids = unavailable + fleet._restart_systemd_gateway_units = unavailable + update_cmd._recover_gateway_restart_after_abort = lambda plan, **kwargs: {} + if mode == "ipc-windows": + # The Windows service manager refuses the paused gateway's restart. + from hermes_cli import main as hermes_main + def scm(token): + if token and token.get("resume_needed"): + raise RuntimeError("Could not restart Windows gateway service(s): HermesGatewayProbe") + hermes_main._resume_windows_gateways_after_update = scm + update_cmd._resume_windows_gateways_after_update = scm + windows_resume = {"resume_needed": True, "services": ["HermesGatewayProbe"], + "service_profiles": {"HermesGatewayProbe": "probe"}} + if mode == "profile-sync": + def profile_sync(): + raise SystemExit("profile skill sync exited 3") + maint._sync_profiles_after_update = profile_sync + from hermes_cli import update_receipt + from hermes_cli.venv_sync import arm_completion, completion_pending_path + arm_completion(source) + update_receipt.begin_update_receipt() + receipt = copy.deepcopy(update_receipt._current.get().data) + # The completion process never inherits the parent's open receipt context. + update_receipt._current.reset(update_receipt._current.get().current_token) + request = {"schema": 1, "source": str(source), "home": str(home), "branch": "main", "desktop": False, + "assume_yes": True, "gateway_mode": mode.startswith("ipc-"), + "no_gateway_restart": mode == "config-ro", + "pre_update_version": None, "snapshot_id": None, "sibling_snapshots": {}, "plan": None, + "receipt": receipt, "windows_resume": windows_resume} + if mode == "config-ro": + home.chmod(0o555) # a late filesystem failure: the profile home refuses the config write + if mode.startswith("receipts-blocked"): + # The receipt store fails after the commit point: its directory is now an ordinary file. + import shutil + shutil.rmtree(update_receipt._receipt_dir()) + update_receipt._receipt_dir().write_text("not a directory") + from hermes_cli.update_completion import _finish, _settle_after_commit + try: + if mode == "receipts-blocked-settle": + code = _settle_after_commit(request, cell / "result.json", "dependencies", "dependency sync refused") + else: + code = _finish(request, cell / "result.json") + finally: + home.chmod(0o755) + latest = update_receipt.read_latest_receipt() or {} + result = json.loads((cell / "result.json").read_text()) if (cell / "result.json").exists() else None + (cell / "summary.json").write_text(json.dumps({ + "code": code, "outcome": latest.get("outcome"), "update_id": receipt["update_id"], "result": result, + "followups": [row["step"] for row in latest.get("followups", [])], + "stages": {mark["name"]: mark["outcome"] for mark in latest.get("stages", [])}, + "config_version": yaml.safe_load((home / "config.yaml").read_text())["_config_version"], + "latest_config_version": DEFAULT_CONFIG["_config_version"], + "pending": completion_pending_path(source).is_file(), + "stamp": (source / "install-stamp.json").is_file(), + "marker_at_verify": (cell / "marker-at-verify").read_text() if (cell / "marker-at-verify").exists() else None, + "marker_final": marker.read_text() if marker.exists() else None, + "migration_entered": entered.is_file()})) +''') + + +def _run_completion(tmp_path: Path, mode: str, *, sqlite: str | None = None) -> dict: + cell = tmp_path / "cell" + root_home = cell / "home" / ".hermes" + # A named profile: receipts stay in the (writable) root home while the profile home refuses. + profile = root_home / "profiles" / "probe" + for name in ("logs/update_receipts", "skills", "cache", "profiles/probe"): + (root_home / name).mkdir(parents=True, exist_ok=True) + env = _child_env(HOME=str(cell / "home"), HERMES_HOME=str(profile), HERMES_DISABLE_LAZY_INSTALLS="1", + HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", + XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache"), + XDG_STATE_HOME=str(cell / "home/.local/state"), + HERMES_GATEWAY_LOCK_DIR=str(cell / "home/.local/state/hermes/gateway-locks")) + if sqlite is not None: + payload = json.dumps({"base_prefix": sys.base_prefix, "executable": sys.executable, + "python_version": list(sys.version_info[:3]), + "sqlite_version": [int(p) for p in sqlite.split(".")], + "sqlite_version_string": sqlite, "sqlite_source_id": "fixture"}) + shim = tmp_path / "python-shim" + script = (f"#!/bin/sh\ncase \"$3\" in *sqlite_source_id*) printf '%s\\n' '{payload}'; exit 0;; esac\n" + f"exec '{sys.executable}' \"$@\"\n") + shim.write_text(script, encoding="utf-8") + shim.chmod(0o755) + env["PROBE_SHIM"] = str(shim) + child = tmp_path / "child.py" + child.write_text(_CHILD, encoding="utf-8") + done = subprocess.run([sys.executable, "-B", str(child), str(ROOT), str(cell), mode], + env=env, cwd=tmp_path, capture_output=True, text=True, timeout=240) + assert done.returncode == 0, done.stdout + done.stderr + summary = json.loads((cell / "summary.json").read_text(encoding="utf-8-sig")) + summary["output"] = done.stdout + done.stderr + return summary + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("sqlite, safe", [("3.46.1", False), ("3.53.1", True)]) +def test_unsafe_sqlite_runtime_keeps_exit_zero_but_vetoes_topology_migration(tmp_path, sqlite, safe): + summary = _run_completion(tmp_path, f"sqlite-{sqlite}", sqlite=sqlite) + # Committed code: success and exit 0 either way. + assert summary["code"] == 0, summary["output"] + assert summary["outcome"] == "success" + if safe: + assert summary["followups"] == [] + assert summary["migration_entered"] is True + else: + # The unsafe runtime is a visible, owed step — and it must still veto the migration. + assert f"SQLite ({sqlite})" in summary["output"] + assert summary["followups"] == ["sqlite_runtime"] + assert summary["migration_entered"] is False + + +@pytest.mark.platforms("posix") +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores directory modes") +def test_config_format_write_failure_is_owed_not_reported_complete(tmp_path): + summary = _run_completion(tmp_path, "config-ro") + assert summary["code"] == 0, summary["output"] + assert summary["config_version"] == summary["latest_config_version"] - 1 + # The receipt names the debt, the tail stays pending and nothing stamps the tree as done. + assert summary["followups"] == ["config_migration"] + assert summary["pending"] is True + assert summary["stamp"] is False + # C3: the receipt names what actually failed. The build succeeded; only the config is owed. + assert summary["stages"].get("build") == "success", summary["stages"] + + +@pytest.mark.platforms("posix") +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores directory modes") +def test_profile_sync_followup_keeps_the_tail_owed(tmp_path): + summary = _run_completion(tmp_path, "profile-sync") + assert summary["code"] == 0, summary["output"] + assert summary["outcome"] == "success" + assert summary["followups"] == ["profile_sync"] + # A profile sync that did not finish is tail work: retried by the next launch, never stamped done. + assert summary["pending"] is True + assert summary["stamp"] is False + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("path", ["finish", "settle"]) +def test_receipt_store_failure_after_commit_keeps_exit_zero_and_a_correlated_receipt(tmp_path, path): + """The receipt store is written after the commit point: losing it is loud, never exit 1.""" + summary = _run_completion(tmp_path, f"receipts-blocked-{path}") + assert "Update receipt not written" in summary["output"] + assert summary["code"] == 0, summary["output"] + result = summary["result"] + assert result["exit_code"] == 0 + # run_completion's correlation gate: this run's id, terminal, and its outcome agrees with exit 0. + receipt = result["receipt"] + assert receipt is not None, summary["output"] + assert receipt["update_id"] == summary["update_id"] + assert receipt["finished_at"] + assert receipt["outcome"] == "success" + # Retry obligations are unchanged: owed dependencies stay armed, a finished tail is cleared. + if path == "settle": + assert [row["step"] for row in receipt["followups"]] == ["dependencies"] + assert summary["pending"] is (path == "settle") + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("branch", ["failed-unit", "abort", "windows"]) +def test_restart_failure_is_owed_and_never_reported_as_a_failed_update(tmp_path, branch): + """The gateway /update reader hears the committed success; the restart is a follow-up.""" + summary = _run_completion(tmp_path, f"ipc-{branch}") + assert summary["code"] == 0, summary["output"] + assert summary["outcome"] == "success" + assert "gateway_restart" in summary["followups"], summary["output"] + # What a reader sees between the failed restart and the run's finalize, and at the end. + assert summary["marker_at_verify"] == "0", summary["output"] + assert summary["marker_final"] == "0" + + +# The NEW tree's completion child after a committed update whose Windows resume the service +# manager refused (what _finish answers): the run is a success with a windows_resume follow-up, +# and the token it hands back still owes the restart. +_REFUSED_RESUME_COMPLETION = textwrap.dedent(''' + import json, os, sys + from pathlib import Path + request = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8-sig")) + receipts = Path(os.environ["PROBE_RECEIPTS"]) + receipt = dict(request["receipt"], outcome="success", exit_code=0, finished_at="now", followups=[ + {"step": "windows_resume", "reason": "Windows gateway recovery failed: refused"}]) + payload = json.dumps(receipt) + (receipts / ("update_probe_" + receipt["update_id"] + ".json")).write_text(payload) + (receipts / "latest.json").write_text(payload) + (Path(request["home"]) / ".update_exit_code").write_text("0") + Path(sys.argv[2]).write_text(json.dumps({ + "schema": 1, "update_id": receipt["update_id"], "exit_code": 0, "receipt": receipt, + "windows_resume": dict(request["windows_resume"], resume_needed=True)})) +''') + +_ZIP_PARENT = textwrap.dedent(''' + import json, os, sys + from pathlib import Path + from types import SimpleNamespace + root_checkout, cell = sys.argv[1:3] + sys.path.insert(0, root_checkout) + cell = Path(cell) + calls = cell / "parent-resume-calls.jsonl" + from hermes_cli import main as hermes_main, update_cmd, update_inventory, update_receipt + hermes_main.PROJECT_ROOT = cell / "source" + receipts = update_receipt._receipt_dir() + receipts.mkdir(parents=True, exist_ok=True) + os.environ["PROBE_RECEIPTS"] = str(receipts) + # Machine boundaries only: the host's process inventory, backups, terminal/hangup handling, + # the Windows service manager, and the ZIP download + swap (a no-.git install). + def no_inventory(): + raise RuntimeError("the host's runtime inventory is not part of this probe") + update_inventory.collect_runtime_inventory = no_inventory + hermes_main._update_preflight_handled = lambda args: False + hermes_main._install_hangup_protection = lambda **kwargs: None + hermes_main._finalize_update_output = lambda state: None + hermes_main._run_pre_update_backup = lambda args: None + token = {"resume_needed": True, "profiles": {}, "unmapped": [], "services": ["HermesGatewayProbe"], + "service_profiles": {"HermesGatewayProbe": "probe"}} + hermes_main._pause_windows_gateways_for_update = lambda: token + def scm(resume_token): + with calls.open("a") as handle: + handle.write(json.dumps(bool(resume_token and resume_token.get("resume_needed"))) + "\\n") + if resume_token and resume_token.get("resume_needed"): + raise RuntimeError("Could not restart Windows gateway service(s): HermesGatewayProbe") + hermes_main._resume_windows_gateways_after_update = scm + update_cmd._prepare_git_command = lambda: (True, None, False) + def via_zip(args, *, had_desktop_app_before_update, target_sha=None, completion_request=None, **kwargs): + completion_request["expected_sha"] = target_sha + completion_request["apply_mode"] = "zip" + update_cmd._complete_source_update(completion_request) + update_cmd._update_via_zip = via_zip + class Args(SimpleNamespace): + def __getattr__(self, name): + return None + hermes_main.cmd_update(Args(gateway=True, branch="main", yes=True)) +''') + + +@pytest.mark.platforms("posix") +def test_zip_update_never_replays_a_refused_resume_after_the_commit_point(tmp_path): + """The child already tried the resume and recorded the follow-up: the parent's finally and its + atexit net must not run it again, and a restart error must not become exit 1 / "/update failed".""" + cell = tmp_path / "cell" + home = cell / "home" / ".hermes" + (home / "logs/update_receipts").mkdir(parents=True) + completion = cell / "source" / "hermes_cli" / "update_completion.py" + completion.parent.mkdir(parents=True) + completion.write_text(_REFUSED_RESUME_COMPLETION, encoding="utf-8") + env = _child_env(HOME=str(cell / "home"), HERMES_HOME=str(home), HERMES_DISABLE_LAZY_INSTALLS="1", + HERMES_RUNTIME_DIR=str(tmp_path / "store"), PYTHONDONTWRITEBYTECODE="1", + XDG_CONFIG_HOME=str(cell / "home/.config"), XDG_CACHE_HOME=str(cell / "home/.cache"), + XDG_STATE_HOME=str(cell / "home/.local/state")) + driver = tmp_path / "zip_parent.py" + driver.write_text(_ZIP_PARENT, encoding="utf-8") + done = subprocess.run([sys.executable, "-B", str(driver), str(ROOT), str(cell)], + env=env, cwd=tmp_path, capture_output=True, text=True, timeout=240) + output = done.stdout + done.stderr + assert done.returncode == 0, output + assert (home / ".update_exit_code").read_text().strip() == "0", output + calls = cell / "parent-resume-calls.jsonl" + assert not calls.exists(), calls.read_text() + output + latest = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + assert latest["outcome"] == "success" + assert [row["step"] for row in latest["followups"]] == ["windows_resume"] + + +_RECEIPT_CHILD = textwrap.dedent(''' + import json, sys + root_checkout, action = sys.argv[1:3] + sys.path.insert(0, root_checkout) + from hermes_cli import update_receipt as ur + if action == "seed": + ur.begin_update_receipt() + ur.record_fact("plan", {"runtimes": [json.loads(sys.argv[3])]}) + else: + # An older interpreter's frozen handoff payload: no ``carried_manual_serves`` field. + old = ur.UpdateReceipt().data + old.pop("carried_manual_serves", None) + ur.begin_update_receipt(previous=old, correlation_id=old["update_id"]) + ur.finalize_pending_update_receipt(0) +''') + + +@pytest.mark.platforms("posix") +def test_old_schema_handoff_receipt_keeps_the_previous_runs_manual_serve_debt(tmp_path): + from hermes_cli.process_identity import _process_create_time + + home = tmp_path / "home" / ".hermes" + home.mkdir(parents=True) + # The durable reminder store is obstructed, so the receipt is the only record of the debt. + (home / "serve_restart_pending").write_text("not a directory\n", encoding="utf-8") + env = _child_env(HOME=str(tmp_path / "home"), HERMES_HOME=str(home), PYTHONDONTWRITEBYTECODE="1") + child = tmp_path / "receipt_child.py" + child.write_text(_RECEIPT_CHILD, encoding="utf-8") + serve = subprocess.Popen([sys.executable, "-I", "-c", "import time; print('ready', flush=True); time.sleep(120)"], + stdout=subprocess.PIPE, text=True) + assert serve.stdout is not None + try: + assert serve.stdout.readline().strip() == "ready" + row = {"kind": "serve", "profile": "fixture", "supervisor": "manual-serve", "restart_via": "respawn-argv", + "pid": serve.pid, "detail": {"create_time": _process_create_time(serve.pid)}} + + def run(*args): + done = subprocess.run([sys.executable, "-B", str(child), str(ROOT), *args], + env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stdout + done.stderr + + def owed(): + receipt = json.loads((home / "logs/update_receipts/latest.json").read_text(encoding="utf-8-sig")) + rows = (receipt.get("plan") or {}).get("runtimes", []) + receipt.get("pending_manual_serves", []) + return [r["pid"] for r in rows if r.get("supervisor") == "manual-serve"] + + run("seed", json.dumps(row)) + assert owed() == [serve.pid] + run("handoff") + assert owed() == [serve.pid] + finally: + serve.terminate() + serve.wait(timeout=10) + serve.stdout.close() diff --git a/tests/hermes_cli/test_update_receipt_durability.py b/tests/hermes_cli/test_update_receipt_durability.py new file mode 100644 index 000000000000..11a8ead1b559 --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_durability.py @@ -0,0 +1,112 @@ +"""Receipt recovery must preserve terminal and user-action facts across processes.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +ROOT = Path(__file__).resolve().parents[2] + + +def _run(tmp_path: Path, body: str) -> dict: + home = tmp_path / "account" + state = home / ".hermes" + state.mkdir(parents=True) + script = tmp_path / "receipt_probe.py" + script.write_text(textwrap.dedent(""" + import json, os, subprocess, sys + from pathlib import Path + sys.path.insert(0, sys.argv[1]) + from hermes_cli import update_receipt as receipts + from hermes_cli.process_identity import _process_create_time + """) + textwrap.dedent(body), encoding="utf-8") + env = {key: value for key, value in os.environ.items() + if not key.startswith(("HERMES_", "PYTEST_"))} + env.update(HOME=str(home), USERPROFILE=str(home), HERMES_HOME=str(state), + HERMES_RUNTIME_DIR=str(tmp_path / "tools"), + XDG_CONFIG_HOME=str(home / ".config"), XDG_CACHE_HOME=str(home / ".cache"), + PYTHONPATH=os.pathsep.join([str(ROOT), *filter(None, sys.path)]), + PYTHONDONTWRITEBYTECODE="1") + done = subprocess.run([sys.executable, str(script), str(ROOT)], env=env, cwd=tmp_path, + text=True, capture_output=True, timeout=60) + assert done.returncode == 0, done.stdout + done.stderr + return json.loads((tmp_path / "result.json").read_text(encoding="utf-8")) + + +def test_nested_begin_does_not_reconcile_live_outer_receipt(tmp_path): + result = _run(tmp_path, """ + receipts.begin_update_receipt() + outer = dict(receipts._current.get().data) + receipts.begin_update_receipt() + archive = receipts._run_file(receipts._receipt_dir(), outer) + before = json.loads(archive.read_text()) + receipts.finalize_pending_update_receipt(0) + restored = receipts.current_correlation_id() + receipts.finalize_pending_update_receipt(0) + Path('result.json').write_text(json.dumps({ + 'outcome': before['outcome'], 'restored': restored, 'outer': outer['update_id']})) + """) + assert result["outcome"] == "running" + assert result["restored"] == result["outer"] + + +def test_writer_identity_cannot_resurrect_a_mismatched_owner(tmp_path): + result = _run(tmp_path, """ + child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']) + try: + ct = _process_create_time(child.pid) + assert ct is not None + same = {'pid': child.pid, 'writer_pid': child.pid, 'pid_create_time': ct - 60} + writer = {'pid': 0, 'writer_pid': child.pid, 'writer_create_time': ct - 60} + control = {'pid': child.pid, 'writer_pid': child.pid, 'pid_create_time': ct} + Path('result.json').write_text(json.dumps({ + 'same': receipts._owner_alive(same), 'writer': receipts._owner_alive(writer), + 'control': receipts._owner_alive(control)})) + finally: + child.terminate() + child.wait(timeout=10) + """) + assert result == {"same": False, "writer": False, "control": True} + + +def test_user_action_survives_interrupt_before_next_stage(tmp_path): + result = _run(tmp_path, """ + receipts.begin_update_receipt() + receipts.record_user_action('local_changes', 'Reapply stash@{0} by hand') + running = receipts.read_latest_receipt() + receipts.finalize_interrupted_update_receipt('operator interrupted') + terminal = receipts.read_latest_receipt() + Path('result.json').write_text(json.dumps({'running': running, 'terminal': terminal})) + """) + expected = {"step": "local_changes", "reason": "Reapply stash@{0} by hand"} + assert result["running"].get("user_action") == expected + assert result["terminal"].get("user_action") == expected + assert result["terminal"]["outcome"] == "interrupted" + + +def test_parent_followup_does_not_replace_child_terminal_with_running(tmp_path): + result = _run(tmp_path, """ + receipts.begin_update_receipt() + frozen = dict(receipts._current.get().data) + Path('request.json').write_text(json.dumps(frozen)) + child = subprocess.run([sys.executable, '-c', + 'import json,sys; from pathlib import Path; ' + 'from hermes_cli.update_completion import _resume_receipt; ' + 'from hermes_cli.update_receipt import finalize_pending_update_receipt; ' + '_resume_receipt(json.loads(Path("request.json").read_text())); ' + 'finalize_pending_update_receipt(0)'], check=True) + before = receipts.read_latest_receipt() + receipts.record_followup('windows_resume', 'recovery refused') + receipts.amend_terminal_followup(frozen['update_id'], 'windows_resume', 'recovery refused') + after = receipts.read_latest_receipt() + archive = json.loads(receipts._run_file(receipts._receipt_dir(), frozen).read_text()) + Path('result.json').write_text(json.dumps({'before': before, 'after': after, 'archive': archive})) + """) + assert result["before"]["outcome"] == "success" + for terminal in (result["after"], result["archive"]): + assert terminal["outcome"] == "success" + assert terminal["finished_at"] == result["before"]["finished_at"] + assert [row["step"] for row in terminal["followups"]] == ["windows_resume"] diff --git a/tests/hermes_cli/test_update_receipt_endpoint.py b/tests/hermes_cli/test_update_receipt_endpoint.py index bcdc34fd1449..e9d4d5daed4e 100644 --- a/tests/hermes_cli/test_update_receipt_endpoint.py +++ b/tests/hermes_cli/test_update_receipt_endpoint.py @@ -32,6 +32,9 @@ def client(): return c +_ACTION_ID = "d" * 32 + + def _write_receipt(tmp_path: Path, monkeypatch, *, outcome="success") -> dict: receipt = { "schema": 1, @@ -40,6 +43,7 @@ def _write_receipt(tmp_path: Path, monkeypatch, *, outcome="success") -> dict: "argv": ["hermes", "update"], "pid": 12345, "outcome": outcome, + "action_id": _ACTION_ID, "pre_update": {"sha": "a" * 40, "version": "0.20.4"}, "post_update": {"sha": "b" * 40, "version": "0.20.5"}, "steps": [{"name": "pre_update_backup", "ok": True, "detail": "", "at": ""}], @@ -91,6 +95,9 @@ class TestUpdateStatusReadsReceipt: def _clear_registries(self, monkeypatch, tmp_path): monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path / "actions") (tmp_path / "actions").mkdir(exist_ok=True) + # The dashboard's own log names the action it spawned; the receipt must name the same one. + (tmp_path / "actions" / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-23 07:00:00 {_ACTION_ID} ===\n", encoding="utf-8") monkeypatch.setattr(_web_server_gateway, "_ACTION_PROCS", {}) monkeypatch.setattr(_web_server_gateway, "_ACTION_RESULTS", {}) monkeypatch.setattr(_web_server_gateway, "_ACTION_COMMANDS", {}) diff --git a/tests/hermes_cli/test_update_receipt_owe_followup.py b/tests/hermes_cli/test_update_receipt_owe_followup.py new file mode 100644 index 000000000000..381e4bb8f660 --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_owe_followup.py @@ -0,0 +1,36 @@ +"""A post-commit follow-up lands on the run exactly once, whether its receipt is still open or closed.""" +from __future__ import annotations + +import json + +from hermes_cli import update_receipt + + +def _steps(path): + return [f["step"] for f in json.loads(path.read_text(encoding="utf-8")).get("followups") or []] + + +def test_follow_up_on_an_open_run_is_recorded_once_not_amended_again(_isolate_hermes_home): + update_receipt.begin_update_receipt() + run = update_receipt._current.get().data + archive = update_receipt._run_file(update_receipt._receipt_dir(), run) + + update_receipt.owe_followup(run["update_id"], "channel_adoption", "remote refused", + retry="the next `hermes update` adopts it again") + + # The durable running record (what an interrupted run leaves behind) owes it once. + assert _steps(archive) == ["channel_adoption"] + update_receipt.finalize_update_receipt("success") + assert _steps(archive) == ["channel_adoption"] + + +def test_follow_up_after_the_run_closed_amends_its_terminal_receipt(_isolate_hermes_home): + update_receipt.begin_update_receipt() + run = dict(update_receipt._current.get().data) + update_receipt.finalize_update_receipt("success") + + update_receipt.owe_followup(run["update_id"], "windows_resume", "Windows gateway recovery failed: boom") + + assert _steps(update_receipt._run_file(update_receipt._receipt_dir(), run)) == ["windows_resume"] + assert [f["step"] for f in update_receipt.read_latest_receipt()["followups"]] == ["windows_resume"] + assert update_receipt.read_latest_receipt()["outcome"] == "success" diff --git a/tests/hermes_cli/test_update_receipt_profile_store.py b/tests/hermes_cli/test_update_receipt_profile_store.py new file mode 100644 index 000000000000..b0cab31aa271 --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_profile_store.py @@ -0,0 +1,59 @@ +"""Under a named profile the updater (root store) and pm (profile store) still see each other's receipts.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +ROOT = Path(__file__).resolve().parents[2] + +_PROBE = textwrap.dedent(""" + import json, sys, time + from pathlib import Path + sys.path.insert(0, sys.argv[1]) + from hermes_cli import update_receipt as receipts + from pm import receipt as pm_receipt + + old = Path(sys.argv[2]) / 'logs' / 'update_receipts' + old.mkdir(parents=True) + (old / 'update_20260101_000000_1_old.json').write_text( + json.dumps({'update_id': 'old', 'action_id': 'a' * 32, 'outcome': 'success'}), encoding='utf-8') + receipts.begin_update_receipt() + receipts.finalize_update_receipt('success') + update_id = receipts.read_latest_receipt()['update_id'] + pm_sees_update = (pm_receipt.latest() or {}).get('update_id') + time.sleep(0.05) + pm_receipt.begin('sync') + pm_receipt.finalize('ok') + Path('result.json').write_text(json.dumps({ + 'update_id': update_id, 'pm_sees_update': pm_sees_update, + 'updater_sees_pm': (receipts.read_latest_receipt() or {}).get('kind'), + 'old_archive': (receipts.read_receipt_for_action('a' * 32) or {}).get('update_id'), + 'root_archives': [p.name for p in receipts._receipt_dir().glob('update_*.json')], + })) +""") + + +def test_named_profile_sees_update_and_pm_receipts_in_both_directions(tmp_path): + home = tmp_path / "account" + profile = home / ".hermes" / "profiles" / "foo" + profile.mkdir(parents=True) + script = tmp_path / "probe.py" + script.write_text(_PROBE, encoding="utf-8") + env = {key: value for key, value in os.environ.items() if not key.startswith(("HERMES_", "PYTEST_"))} + env.update(HOME=str(home), USERPROFILE=str(home), HERMES_HOME=str(profile), + HERMES_RUNTIME_DIR=str(tmp_path / "tools"), PYTHONDONTWRITEBYTECODE="1", + PYTHONPATH=os.pathsep.join([str(ROOT), *filter(None, sys.path)])) + done = subprocess.run([sys.executable, str(script), str(ROOT), str(profile)], env=env, cwd=tmp_path, + text=True, capture_output=True, timeout=60) + assert done.returncode == 0, done.stdout + done.stderr + result = json.loads((tmp_path / "result.json").read_text(encoding="utf-8")) + + # The updater still writes the root store the Desktop and hand-off scripts read. + assert any(result["update_id"] in name for name in result["root_archives"]) + assert result["pm_sees_update"] == result["update_id"] # `hermes -p foo pm status` + assert result["updater_sees_pm"] == "sync" # read_latest_receipt() consumers + assert result["old_archive"] == "old" # receipts written to the profile store before the move diff --git a/tests/hermes_cli/test_update_receipt_run_record.py b/tests/hermes_cli/test_update_receipt_run_record.py new file mode 100644 index 000000000000..c60121d1274f --- /dev/null +++ b/tests/hermes_cli/test_update_receipt_run_record.py @@ -0,0 +1,42 @@ +"""A run's own archive is read through one lookup: an unreadable record is a missing record.""" +from __future__ import annotations + +import pytest + +from hermes_cli import update_completion, update_receipt + + +def _lost_run(tmp_path): + update_receipt.begin_update_receipt() + run = dict(update_receipt._current.get().data) + request = {"source": str(tmp_path / "checkout"), "receipt": run, "desktop": False, "gateway_mode": False} + return run, update_receipt._run_file(update_receipt._receipt_dir(), run), request + + +@pytest.mark.parametrize("content", ["null", "[]", "{torn"]) +def test_lost_completion_over_a_malformed_run_archive_still_closes_the_run(_isolate_hermes_home, tmp_path, content): + # The code is committed: an archive that is valid JSON but not a record (hand edit, foreign + # writer) must settle exactly like a missing one, never raise out of the post-commit path. + run, archive, request = _lost_run(tmp_path) + archive.write_text(content, encoding="utf-8") + + result = update_completion.settle_lost_completion(request, "the completion process exited -9") + + assert result["exit_code"] == 0 + assert result["receipt"]["update_id"] == run["update_id"] + assert result["receipt"]["outcome"] == "success" + assert [row["step"] for row in result["receipt"]["followups"]] == ["completion"] + + +def test_lost_completion_keeps_the_stages_the_completion_child_persisted(_isolate_hermes_home, tmp_path): + # The child resumed the parent's snapshot, persisted its own progress, then died (OOM, SIGKILL). + # The parent's pre-child snapshot is stale: closing the run from it would erase what ran. + run, archive, request = _lost_run(tmp_path) + progress = [{"name": "deps", "outcome": "success", "at": "t1"}, {"name": "build", "outcome": "failed", "at": "t2"}] + update_receipt._persist_running({**run, "stages": progress}) + + result = update_completion.settle_lost_completion(request, "the completion process exited -9") + + assert result["receipt"]["stages"][:2] == progress + assert [row["step"] for row in result["receipt"]["followups"]] == ["completion"] + assert update_receipt._current.get() is None # the parent's own receipt closed, nothing left open diff --git a/tests/hermes_cli/test_update_stale_gateway_yield.py b/tests/hermes_cli/test_update_stale_gateway_yield.py index 9a2fcba478c6..f967d21b1579 100644 --- a/tests/hermes_cli/test_update_stale_gateway_yield.py +++ b/tests/hermes_cli/test_update_stale_gateway_yield.py @@ -103,27 +103,29 @@ def test_update_signals_proven_stale_gateway_survivor(cron_home, tmp_path): def test_verify_fleet_hands_stale_rows_to_survivor_signalling(monkeypatch): - """The wiring: a stale fleet matrix reaches signal_stale_fleet_survivors before exit 1.""" + """The wiring: a stale fleet matrix reaches signal_stale_fleet_survivors; the restart stays owed.""" import hermes_cli.update_cmd_fleet as fleet_mod + import hermes_cli.update_cmd_fleet_verify as fleet_verify import hermes_cli.update_cmd_stale_survivors as surv import hermes_cli.update_cmd as update_cmd from hermes_cli.update_cmd_fleet import _GatewayRestartOutcome stale_fleet = [{"profile": "default", "pid": 4242, "state": "stale", "code_sha": "a" * 40}] seen = {} - monkeypatch.setattr(fleet_mod, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(update_cmd, "_finish_dashboard_update_cleanup", lambda *a, **k: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) - monkeypatch.setattr(fleet_mod, "_collect_fleet_snapshot", lambda restart, expected: stale_fleet) + monkeypatch.setattr(fleet_verify, "_collect_fleet_snapshot", lambda restart, expected: stale_fleet) monkeypatch.setattr(surv, "signal_stale_fleet_survivors", lambda fleet, restart, budget: seen.setdefault("fleet", fleet) and []) restart = _GatewayRestartOutcome( incomplete=False, phase_errors=[], pre_restart_gateway_pids=[4242], restarted_services=[], failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(), ) - with contextlib.redirect_stdout(io.StringIO()), pytest.raises(SystemExit) as exc: - fleet_mod._verify_fleet_after_update( + with contextlib.redirect_stdout(io.StringIO()): + fleet_verify._verify_fleet_after_update( restart, _pre_update_plan=None, _windows_gateway_resume=None, update_complete=True, ) - assert exc.value.code == 1 + # Contract C3: was SystemExit(1); now flagged incomplete (owed restart, marker kept). + assert restart.incomplete assert seen["fleet"] == stale_fleet diff --git a/tests/hermes_cli/test_update_zip_completion.py b/tests/hermes_cli/test_update_zip_completion.py index a60d76b59603..514179e00add 100644 --- a/tests/hermes_cli/test_update_zip_completion.py +++ b/tests/hermes_cli/test_update_zip_completion.py @@ -11,7 +11,7 @@ import pytest from hermes_cli import main, update_cmd, update_cmd_fleet as fleet, update_cmd_maint as maint -from hermes_cli import update_cmd_zip, update_receipt +from hermes_cli import update_cmd_fleet_verify as fleet_verify, update_cmd_zip, update_receipt from hermes_cli.config_defaults import DEFAULT_CONFIG from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan import hermes_yaml @@ -84,10 +84,10 @@ def prepare(selected, *, desktop): monkeypatch.setattr("hermes_cli.profiles.seed_profile_skills", lambda *a, **kw: {}) monkeypatch.setattr(update_cmd, "_reload_config_modules", lambda: None) monkeypatch.setattr(update_cmd, "_post_update_sqlite_runtime_status", lambda: (True, None)) - monkeypatch.setattr(fleet, "_print_legacy_units_warning", lambda: None) + monkeypatch.setattr(fleet_verify, "_print_legacy_units_warning", lambda: None) monkeypatch.setattr(maint, "_refresh_dashboard_after_update", lambda **kwargs: None) monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: []) - monkeypatch.setattr(fleet, "_collect_fleet_snapshot", lambda *args: []) + monkeypatch.setattr(fleet_verify, "_collect_fleet_snapshot", lambda *args: []) monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None) def resume(received): @@ -168,26 +168,24 @@ def test_zip_helper_propagates_completion_status_after_real_verification(zip_upd {"pid": plan.runtimes[0].pid, "profile": "default"}]) request = update_cmd._source_completion_request( update_cmd._resolve_update_options(args, True), plan, snapshot, state.token, False, True) - if verdict == "healthy": - assert update_cmd_zip._update_via_zip(args, completion_request=request) is True - else: - with pytest.raises(SystemExit) as error: - update_cmd_zip._update_via_zip(args, completion_request=request) - assert error.value.code == 1 - assert state.events == ["prepare", ("marker", verdict != "unsafe-sqlite"), - "restart", "resume", "finalize"] + # Contract C3: the swapped code is committed, so neither verdict fails the update (was SystemExit(1)). + assert update_cmd_zip._update_via_zip(args, completion_request=request) is True + # The gateway watcher hears success for every verdict (was False for unsafe SQLite). + assert state.events == ["prepare", ("marker", True), "restart", "resume", "finalize"] receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) - assert receipt["outcome"] == ("success" if verdict == "healthy" else "partial") + # Was "partial": a success that names what is still owed. + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt.get("followups", [])] == { + "healthy": [], "unsafe-sqlite": ["sqlite_runtime"], "stale-fleet": ["gateway_restart"]}[verdict] assert receipt["runtime_outcomes"][0]["outcome"] == ( "unaccounted" if verdict == "stale-fleet" else "restarted") assert json.loads(state.jobs.read_text()) == state.original_jobs @pytest.mark.parametrize("route", ["direct", "git-failure"]) -@pytest.mark.parametrize("failure", ["swap", "late-swap", "stage", "preparation"]) +@pytest.mark.parametrize("failure", ["swap", "late-swap", "stage"]) def test_zip_failure_recovers_pause_without_completion_mutations(zip_update, monkeypatch, route, failure): import os - import pm state = zip_update before = {profile: (profile / "config.yaml").read_bytes() @@ -225,12 +223,6 @@ def fail_copy(src, dst, *args, **kwargs): return copytree(src, dst, *args, **kwargs) monkeypatch.setattr(shutil, "copytree", fail_copy) expected = SystemExit - else: - def fail_preparation(*args, **kwargs): - assert (state.root / "payload.txt").read_text() == "new" - raise pm.InstallError("venv", "preparation stopped") - monkeypatch.setattr("hermes_cli.source_build.build_update_products", fail_preparation) - expected = pm.InstallError with pytest.raises(expected) as raised: update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=True) if failure in {"swap", "late-swap"}: @@ -246,10 +238,110 @@ def fail_preparation(*args, **kwargs): assert {profile: (profile / "config.yaml").read_bytes() for profile in before} == before assert not (state.sibling / ".env").exists() assert json.loads(state.jobs.read_text()) == state.original_jobs - assert not (state.active / "logs/update_receipts/latest.json").exists() + # The receipt is durable from the start (was: no latest.json at all). This direct + # ``_cmd_update_impl`` call has no command-boundary finalizer, so it is still the open + # record of THIS run, never a success. + latest = state.active / "logs/update_receipts/latest.json" + assert not latest.exists() or json.loads(latest.read_text())["outcome"] != "success" assert not list(state.root.glob("*.hermes-update-*")) +@pytest.mark.parametrize("route", ["direct", "git-failure"]) +def test_zip_build_failure_after_swap_is_a_followup(zip_update, monkeypatch, route): + # Was the "preparation" case above (pm.InstallError propagated): the build runs after the + # swap committed the new tree, so under contract C3 it is an owed follow-up, not a failure. + import pm + + state = zip_update + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) + monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) + + def fail_fetch(*args, **kwargs): + raise subprocess.CalledProcessError(1, ["git", "fetch"]) + monkeypatch.setattr(update_cmd, "_git_run", fail_fetch) + + def fail_build(*args, **kwargs): + assert (state.root / "payload.txt").read_text() == "new" + raise pm.InstallError("venv", "preparation stopped") + monkeypatch.setattr("hermes_cli.source_build.build_update_products", fail_build) + update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=True) + assert (state.root / "payload.txt").read_text() == "new" + receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "success" + assert "build" in [f["step"] for f in receipt["followups"]] + assert state.token["resume_needed"] is False + + +@pytest.mark.parametrize("route", ["direct", "git-failure"]) +def test_zip_dependency_sync_failure_after_swap_is_a_followup(zip_update, monkeypatch, tmp_path, route): + # The dropped "preparation" case, restored at the real seam: PM's dependency sync in the + # completion bootstrap fails AFTER the swap. Was pm.InstallError / exit 1 / no receipt, which + # the Desktop reads as "still on the previous version" while the tree is new. Contract C3/A6: + # exit 0, a ``dependencies`` follow-up, the tail obligation armed, nothing of the tail run. + import pm + import pm.client + from hermes_cli import update_completion + from hermes_cli.venv_sync import completion_pending_path + + state = zip_update + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (route == "direct", ["git"], False)) + monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) + monkeypatch.setattr(update_cmd, "_should_zip_fallback_on_update_error", lambda exc: True) + + def fail_fetch(*args, **kwargs): + raise subprocess.CalledProcessError(1, ["git", "fetch"]) + monkeypatch.setattr(update_cmd, "_git_run", fail_fetch) + monkeypatch.setattr(pm.client, "ensure_tools_for_sync", lambda: None) + + def fail_sync(*args, **kwargs): + assert (state.root / "payload.txt").read_text() == "new" + raise pm.InstallError("venv", "dependency sync stopped") + monkeypatch.setattr(pm, "sync_venv", fail_sync) + + def bootstrap(request): # the completion child's first interpreter, in-process + request = {**request, "bytecode_cache": str(tmp_path / "bytecode")} + request_path, result_path = tmp_path / "request.json", tmp_path / "result.json" + update_completion._write_json(request_path, request) + update_completion._bootstrap(request, request_path, result_path) + return json.loads(result_path.read_text()) + monkeypatch.setattr(update_cmd, "run_completion", bootstrap) + + update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=True) + + assert (state.root / "payload.txt").read_text() == "new" + receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "success" + assert [f["step"] for f in receipt["followups"]] == ["dependencies"] + assert completion_pending_path(state.root).is_file() # the next launch syncs and finishes the tail + assert "prepare" not in state.events # no build/maintenance ran against stale dependencies + assert (state.active / ".update_exit_code").read_text() == "0" + assert "resume" in state.events and state.token["resume_needed"] is False + + +def test_ctrl_c_after_the_swap_reports_the_new_code_not_a_failure(zip_update, monkeypatch, capsys): + # Ctrl-C while the completion child runs: run_completion kills the child group and re-raises. + # The tree is new, so the run is ``interrupted`` (never "failed" / "still on the previous + # version") and the message says what is owed; a non-zero exit for an interrupt is fine. + state = zip_update + monkeypatch.setattr(update_cmd, "_prepare_git_command", lambda: (True, ["git"], False)) + monkeypatch.setattr(main, "_warn_orphaned_update_autostashes", lambda *args: None) + + def interrupted(request): + assert (state.root / "payload.txt").read_text() == "new" + raise KeyboardInterrupt + monkeypatch.setattr(update_cmd, "run_completion", interrupted) + + with pytest.raises(SystemExit) as exc: + update_cmd._cmd_update_impl(SimpleNamespace(branch="main", yes=True), gateway_mode=False) + + assert exc.value.code == 130 + assert "Interrupted after the code was updated: the new code is in place" in capsys.readouterr().out + receipt = json.loads((state.active / "logs/update_receipts/latest.json").read_text()) + assert receipt["outcome"] == "interrupted" and receipt["exit_code"] == 130 + assert (state.root / "payload.txt").read_text() == "new" + + @pytest.mark.parametrize("entry", ["payload.txt", "tools"]) def test_zip_recovers_crashed_backup_before_failed_copy_and_retry(zip_update, monkeypatch, entry): import shutil diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 9157381ec55e..ade89ea0ea9a 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -1454,35 +1454,6 @@ def fail_spawn(*_args, **_kwargs): assert check_data["can_apply"] is False assert check_data["update_command"] == data["update_command"] - def test_update_status_recovers_completed_result_after_dashboard_restart(self, monkeypatch, tmp_path): - - action_id = "c" * 32 - (tmp_path / "hermes-update.log").write_text( - "=== hermes-update started 2026-08-17 11:19:34 ===\n" - "pulling updates...\n", - encoding="utf-8", - ) - (tmp_path / "update.log").write_text( - "=== hermes update started 2026-08-17T11:19:35 ===\n" - "✓ Update complete!\n" - f"=== hermes-update completed {action_id} ===\n", - encoding="utf-8", - ) - monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) - monkeypatch.setattr(_web_server_gateway, "_ACTION_PROCS", {}) - monkeypatch.setattr(_web_server_gateway, "_ACTION_RESULTS", {}) - monkeypatch.setattr(_web_server_gateway, "_ACTION_COMMANDS", {}) - monkeypatch.setattr(_web_server_gateway, "_ACTION_IDS", {}) - - status = self.client.get("/api/actions/hermes-update/status?lines=2000") - - assert status.status_code == 200 - data = status.json() - assert data["running"] is False - assert data["exit_code"] == 0 - assert data["action_id"] == action_id - assert f"=== hermes-update completed {action_id} ===" in data["lines"] - def test_update_hermes_spawns_with_action_id(self, monkeypatch): import hermes_cli.web_server as web_server diff --git a/tests/hermes_cli/test_web_server_update_status.py b/tests/hermes_cli/test_web_server_update_status.py new file mode 100644 index 000000000000..fdab5db065cf --- /dev/null +++ b/tests/hermes_cli/test_web_server_update_status.py @@ -0,0 +1,197 @@ +"""Dashboard update status reads the ROOT home's shared update.log, correlated by action id.""" + +import types + +import pytest + +import hermes_cli.web_server_gateway as _web_server_gateway + + +class TestUpdateStatusRootLog: + @pytest.fixture(autouse=True) + def _setup_test_client(self, monkeypatch, _isolate_hermes_home): + """Create a TestClient and isolate the state DB under the test HERMES_HOME.""" + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_constants import get_hermes_home + from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + + self.client = TestClient(app) + self.client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + + def test_update_status_recovers_completed_result_after_dashboard_restart(self, monkeypatch, tmp_path): + # The dashboard runs under a profile home; ``hermes update`` mirrors to the ROOT home's + # update.log (main_dashboard), so the durable completion marker is read there. + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) + action_id = "c" * 32 + (tmp_path / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-17 11:19:34 {action_id} ===\n" + "pulling updates...\n", + encoding="utf-8", + ) + (root / "logs" / "update.log").write_text( + "=== hermes update started 2026-08-17T11:19:35 ===\n" + "✓ Update complete!\n" + f"=== hermes-update completed {action_id} ===\n", + encoding="utf-8", + ) + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) + monkeypatch.setattr(_web_server_gateway, "_ACTION_PROCS", {}) + monkeypatch.setattr(_web_server_gateway, "_ACTION_RESULTS", {}) + monkeypatch.setattr(_web_server_gateway, "_ACTION_COMMANDS", {}) + monkeypatch.setattr(_web_server_gateway, "_ACTION_IDS", {}) + + status = self.client.get("/api/actions/hermes-update/status?lines=2000") + + assert status.status_code == 200 + data = status.json() + assert data["running"] is False + assert data["exit_code"] == 0 + assert data["action_id"] == action_id + assert f"=== hermes-update completed {action_id} ===" in data["lines"] + + + def test_update_status_ignores_another_profiles_completion_in_the_shared_root_log(self, monkeypatch, tmp_path): + # Every profile's ``hermes update`` mirrors into the ROOT update.log; profile A's success + # there must not certify the action profile B's dashboard started (and lost on restart). + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + b_logs = tmp_path / "b-logs" + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", b_logs) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + monkeypatch.setattr(_web_server_gateway.subprocess, "Popen", lambda *a, **kw: types.SimpleNamespace(pid=7)) + b_id = "b" * 32 + _web_server_gateway._spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": b_id}) + _web_server_gateway._ACTION_PROCS.clear() # B's dashboard restarted: in-memory result lost + + def status_after_root_completion(action_id): + (root / "logs" / "update.log").write_text( + "=== hermes update started 2026-08-17T11:19:35 ===\n" + f"=== hermes-update completed {action_id} ===\n", encoding="utf-8") + return self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + foreign = status_after_root_completion("a" * 32) + assert foreign["exit_code"] is None + assert "action_id" not in foreign + own = status_after_root_completion(b_id) + assert own["exit_code"] == 0 + assert own["action_id"] == b_id + + @pytest.mark.parametrize("log_shape,b_outcome", [ + ("short", "success"), ("long", "success"), ("rotated", "success"), ("long", "failed"), ("long", None)]) + def test_restarted_dashboard_keeps_its_action_identity_past_the_log_tail( + self, monkeypatch, tmp_path, log_shape, b_outcome): + # The status route reads at most 2,000 log lines: build output (or rotation) pushing the + # start header out must not turn B's own success into exit null ("Action failed (exit ?)"), + # nor let profile A's newer latest receipt answer for B. + from hermes_cli import update_receipt + + root, b_id, a_id = tmp_path / "root", "b" * 32, "a" * 32 + (root / "logs").mkdir(parents=True) + b_logs = tmp_path / "b-logs" + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", b_logs) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + monkeypatch.setattr(_web_server_gateway.subprocess, "Popen", lambda *a, **kw: types.SimpleNamespace(pid=7)) + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + _web_server_gateway._spawn_hermes_action(["update"], "hermes-update", env_overrides={"HERMES_ACTION_ID": b_id}) + for profile, action_id, outcome in (("b", b_id, b_outcome), ("a", a_id, "success")): + if outcome: + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / profile)) + monkeypatch.setenv("HERMES_ACTION_ID", action_id) + update_receipt.begin_update_receipt() + update_receipt.finalize_update_receipt(outcome) + monkeypatch.delenv("HERMES_ACTION_ID") + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + log = b_logs / "hermes-update.log" + if log_shape == "long": + with log.open("a", encoding="utf-8") as handle: + handle.write("build output\n" * 2001) + elif log_shape == "rotated": + log.unlink() + if b_outcome == "success": + (root / "logs" / "update.log").write_text(f"=== hermes-update completed {b_id} ===\n", encoding="utf-8") + _web_server_gateway._ACTION_PROCS.clear() # dashboard restarted: in-memory registries lost + _web_server_gateway._ACTION_IDS.clear() + + data = self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + assert data["running"] is False + assert data["exit_code"] == (0 if b_outcome == "success" else None) + assert data.get("action_id") == (b_id if b_outcome == "success" else None) + assert data["receipt"]["action_id"] == (b_id if b_outcome else a_id) + + @pytest.mark.parametrize("b_outcome", [None, "failed", "success"]) + def test_root_latest_receipt_of_another_profile_never_certifies_this_action(self, monkeypatch, tmp_path, b_outcome): + # The receipt store is root-wide too: A's later success in latest.json must not become B's + # exit 0. Only a receipt naming B's action (found in the archive when A is latest) counts. + from hermes_cli import update_receipt + + root = tmp_path / "root" + (root / "logs").mkdir(parents=True) + b_id, a_id = "b" * 32, "a" * 32 + for profile, action_id, outcome in (("b", b_id, b_outcome), ("a", a_id, "success")): + if outcome: + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / profile)) + monkeypatch.setenv("HERMES_ACTION_ID", action_id) + update_receipt.begin_update_receipt() + update_receipt.finalize_update_receipt(outcome) + monkeypatch.delenv("HERMES_ACTION_ID") + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "b")) + assert update_receipt.read_latest_receipt()["action_id"] == a_id + (tmp_path / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-17 11:19:34 {b_id} ===\n", encoding="utf-8") + (root / "logs" / "update.log").write_text(f"=== hermes-update completed {a_id} ===\n", encoding="utf-8") + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + + data = self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + assert data["exit_code"] == (0 if b_outcome == "success" else None) + assert "action_id" not in data # A's root completion marker is not B's + # The attached receipt names its writer: B's own when it exists, else A's (never B's). + assert data["receipt"]["action_id"] == (b_id if b_outcome else a_id) + assert data["receipt"]["outcome"] == (b_outcome or "success") + + @pytest.mark.parametrize("debt", ["none", "followups", "user_action"]) + def test_status_receipt_carries_owed_steps_of_a_committed_update(self, monkeypatch, tmp_path, debt): + # C3: owed post-commit steps keep the run a success (exit 0); the summary must still name + # them so the dashboard and Desktop report the debt instead of plain success. + from hermes_cli import update_receipt + + root, action_id = tmp_path / "root", "c" * 32 + monkeypatch.setenv("HERMES_HOME", str(root / "profiles" / "coder")) + monkeypatch.setenv("HERMES_ACTION_ID", action_id) + update_receipt.begin_update_receipt() + if debt == "followups": + update_receipt.record_followup("dependencies", "synthetic selected-interpreter sync failed") + update_receipt.record_followup("config_migration", "disk full") + elif debt == "user_action": + update_receipt.record_user_action("autostash", "re-apply the parked stash") + update_receipt.finalize_update_receipt("success") + (tmp_path / "hermes-update.log").write_text( + f"=== hermes-update started 2026-08-17 11:19:34 {action_id} ===\n", encoding="utf-8") + monkeypatch.setattr(_web_server_gateway, "_ACTION_LOG_DIR", tmp_path) + for registry in ("_ACTION_PROCS", "_ACTION_RESULTS", "_ACTION_COMMANDS", "_ACTION_IDS"): + monkeypatch.setattr(_web_server_gateway, registry, {}) + + data = self.client.get("/api/actions/hermes-update/status?lines=2000").json() + + receipt = data["receipt"] + assert [f["step"] for f in receipt["followups"]] == ( + ["dependencies", "config_migration"] if debt == "followups" else []) + assert receipt["user_action"] == ( + {"step": "autostash", "reason": "re-apply the parked stash"} if debt == "user_action" else None) + # Debt never changes the C3 exit mapping: owed follow-ups stay exit 0, a user action is partial. + assert data["exit_code"] == (1 if debt == "user_action" else 0) diff --git a/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py b/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py index 36133d6ce78a..3905d819b6bc 100644 --- a/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py +++ b/tests/scripts/desktop_update/test_desktop_update_gateway_flag.py @@ -13,6 +13,7 @@ from __future__ import annotations +import json import os import subprocess import time @@ -34,10 +35,11 @@ FAKE_HERMES = """#!/usr/bin/env bash case "$*" in *--help*) echo "--keep-stash"; exit 0 ;; esac printf '%s\\n' "$*" >> "$HERMES_TEST_ARGV" +case " $* " in *" update "*) printf '%s\\n' "${HERMES_TEST_UPDATE_OUTPUT:-}" ;; esac exit 0 """ -def _run_handoff(tmp_path: Path, extra_args: list[str]) -> list[str]: +def _run_handoff(tmp_path: Path, extra_args: list[str], update_output: str = "") -> list[str]: """Run the real hand-off end to end; return the argv of each hermes call.""" install_root = tmp_path / "hermes-agent" (install_root / "venv" / "bin").mkdir(parents=True) @@ -47,7 +49,8 @@ def _run_handoff(tmp_path: Path, extra_args: list[str]) -> list[str]: argv_log = tmp_path / "argv.jsonl" # posix.sh honours an ambient HERMES_HOME; the result file must land where this test looks. - env = {**os.environ, "TMPDIR": str(tmp_path), "HERMES_TEST_ARGV": str(argv_log), "HERMES_HOME": str(tmp_path)} + env = {**os.environ, "TMPDIR": str(tmp_path), "HERMES_TEST_ARGV": str(argv_log), "HERMES_HOME": str(tmp_path), + "HERMES_TEST_UPDATE_OUTPUT": update_output} subprocess.run( ["/bin/bash", str(SHIM_DIR / "posix.sh"), "--install-root", str(install_root), "--no-ui", *extra_args], env=env, @@ -89,3 +92,44 @@ def test_no_gateway_flag_omits_gateway_from_update(tmp_path): assert "--gateway" not in argv, f"--gateway reappeared in update argv: {call}" assert "--keep-stash" in argv, "--no-gateway must not disturb --keep-stash" assert "--yes" in argv + + +@requires_posix_handoff +def test_owed_desktop_build_is_a_manual_outcome_even_when_the_followup_is_truncated(tmp_path): + """Contract C3: the update committed (exit 0) but this app was not rebuilt. + + The receipt follow-up is capped at 500 chars and leads with whichever product failed first, + so it can lose the desktop part; the CLI's own whole-line marker is what the hand-off reads. + """ + followup = " ⚠ Update follow-up 'build' did not finish: feature dependencies: " + "x" * 440 + _run_handoff(tmp_path, [], f"{followup}\n Desktop app build owed: desktop app build failed\n✓ Update complete!") + + result = json.loads((tmp_path / ".hermes-update-result.json").read_text(encoding="utf-8-sig")) + assert result["exit_code"] == 0 + assert result["manual"] is True + assert "could not be rebuilt" in result["message"] + + +@requires_posix_handoff +@pytest.mark.parametrize("step", ["gateway_restart", "windows_resume", "channel_adoption", "completion"]) +def test_every_owed_followup_reaches_the_desktop_result(tmp_path, step): + """Contract C3 (review regression 1): a committed update exits 0, but a follow-up it still owes + (a gateway proven to run old code, a refused resume, a lost completion) must never end as a + plain "Update complete." -- the result names the owed step.""" + line = f" ⚠ Update follow-up '{step}' did not finish: old code still serving (the next launch retries it)" + _run_handoff(tmp_path, [], f"{line}\n✓ Update complete!") + + result = json.loads((tmp_path / ".hermes-update-result.json").read_text(encoding="utf-8-sig")) + assert result["exit_code"] == 0 and result["ok"] is True + assert result["manual"] is True + assert step in result["message"] + assert ("hermes gateway restart" in result["message"]) is (step == "gateway_restart") + + +@requires_posix_handoff +def test_a_clean_update_stays_plain_success(tmp_path): + _run_handoff(tmp_path, [], "✓ Update complete!") + + result = json.loads((tmp_path / ".hermes-update-result.json").read_text(encoding="utf-8-sig")) + assert result["exit_code"] == 0 and result["manual"] is False + assert result["message"] == "Update complete." diff --git a/web/src/contexts/SystemActions.test.tsx b/web/src/contexts/SystemActions.test.tsx new file mode 100644 index 000000000000..dc6bf2ac2a00 --- /dev/null +++ b/web/src/contexts/SystemActions.test.tsx @@ -0,0 +1,100 @@ +// @vitest-environment jsdom +// The update toast names what a committed update still owes for every committed outcome: a +// partial run (exit 1 after record_user_action) shows the producer's own instruction, and only +// this action's receipt may name debt (the status route attaches the latest receipt otherwise). + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; + +const apiMocks = vi.hoisted(() => ({ + getActionStatus: vi.fn(), + getStatus: vi.fn(), + restartGateway: vi.fn(), + updateHermes: vi.fn(), +})); + +vi.mock("@/lib/api", () => ({ api: apiMocks })); + +import { I18nProvider } from "@/i18n"; +import { SystemActionsProvider } from "./SystemActions"; +import { useSystemActions } from "./useSystemActions"; + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; + +const INSTRUCTION = + "Your local changes are parked in stash@{0}; run `git stash pop` in ~/.hermes/hermes-agent."; + +let container: HTMLDivElement; +let root: Root; + +function Probe() { + const { runAction } = useSystemActions(); + return