From 45e038227aa7c4d5e0994ddbc0651f1a92c1883d Mon Sep 17 00:00:00 2001 From: doudouOUC Date: Tue, 28 Jul 2026 21:23:00 +0800 Subject: [PATCH 1/6] fix(core): add certified session writer handoff Seal managed writer ownership after a durable recorder drain and allow trusted replacements to take over only when the transcript proof still matches. Co-authored-by: Qwen-Coder --- .../certified-session-writer-handoff.md | 182 ++++ .../cli/src/acp-integration/acpAgent.test.ts | 7 + packages/cli/src/acp-integration/acpAgent.ts | 3 +- packages/core/src/config/config.test.ts | 23 + packages/core/src/config/config.ts | 23 +- .../src/services/chatRecordingService.test.ts | 25 + .../core/src/services/chatRecordingService.ts | 23 +- .../src/services/session-writer-lease.test.ts | 513 ++++++++++++ .../core/src/services/session-writer-lease.ts | 778 +++++++++++++++++- 9 files changed, 1535 insertions(+), 42 deletions(-) create mode 100644 docs/design/certified-session-writer-handoff.md diff --git a/docs/design/certified-session-writer-handoff.md b/docs/design/certified-session-writer-handoff.md new file mode 100644 index 00000000000..bfc455eb5ca --- /dev/null +++ b/docs/design/certified-session-writer-handoff.md @@ -0,0 +1,182 @@ +# Certified session writer handoff + +## Problem + +Cooperative managed shutdown currently releases each session writer lock before +the ACP child exits. That fixes the ordinary replacement path, but it cannot +distinguish a writer that deliberately stopped recording from a foreign-host +writer that disappeared without releasing its active lock. Treating hostname, +PID visibility, lock age, or transcript inactivity as proof of death would +allow two live Pods to append to the same transcript. + +## Scope + +This change adds an integrity-protected handoff state for managed ACP writers. +After closing admission and durably draining accepted recorder work, a trusted +managed child can replace its active lock with a sealed record. A trusted +managed replacement can take ownership only after validating that record +against the exact transcript requested by the new Config. + +The protocol remains gated by `experimental.sessionWriterLease`, which is +disabled by default and snapshotted at ACP process startup. Standalone ACP, +interactive, and headless recorders do not gain certified takeover. Normal +per-session close still releases its lock instead of leaving a sealed record. + +This change does not reclaim an active lock left by SIGKILL, an event-loop +stall, an unhandled crash, or a storage failure before sealing completes. It +does not add TTLs, heartbeats, hostname stealing, Kubernetes API lookups, +operator force-steal endpoints, maintenance leases, or mixed-version support. +Those active-lock cases still require an authoritative external writer fence +and explicit recovery. + +## Lock records + +New owners write schema v2 records. An active record retains the existing +immutable owner diagnostics and adds: + +```json +{ + "schema_version": 2, + "state": "active" +} +``` + +A sealed record retains the previous owner's diagnostics and adds: + +```json +{ + "schema_version": 2, + "state": "sealed", + "sealed_at": "2026-07-28T00:00:00.000Z", + "transcript": { + "relative_path": "", + "exists": true, + "byte_length": 1234, + "sha256": "" + } +} +``` + +The relative key must resolve to the transcript path already supplied by the +new Config. It is never used to select an arbitrary filesystem path. Schema v1 +records remain valid active records for compatibility during rollback, but +they can never be interpreted as sealed. + +## Fixed claim + +The fixed claim path is: + +```text +.claim +``` + +It serializes the two transitions that temporarily remove the primary path: +active-to-sealed and sealed-to-active. The claim is created with the existing +write-sync-and-hard-link primitive. A claim is never reclaimed by PID, +hostname, or age. Any pre-existing claim returns +`session_writer_unavailable`; manual cleanup is allowed only after an +authoritative external writer fence. + +Ordinary acquisition checks the claim before every attempt to install a +missing primary lock. It checks again after installation and removes only its +own exact candidate if a concurrent transition acquired the claim. This keeps +the current active-lock fast path and local stale-owner recovery while making +both paths respect a handoff transition. Mixed-version writers remain +unsupported because an older writer does not know about the fixed claim. + +## Sealing + +Managed shutdown synchronously stops session and recorder admission, then +starts all writer terminals in parallel. A writer terminal: + +1. drains every recorder operation accepted before the cutoff; +2. opens the expected transcript without following a symlink, verifies the + existing owner and transcript snapshot, and hashes the bytes through that + kept-open file descriptor; +3. writes and syncs an owner-unique sealed candidate; +4. acquires the fixed claim and revalidates the active owner plus the kept-open + transcript identity and metadata; +5. renames the exact active primary to an owner-unique retired path; +6. hard-links the sealed candidate into the now-missing primary path without + replacement; and +7. removes only its exact retired, candidate, and claim records. + +The primary-path transition is logically atomic for cooperating writers +because every installation respects the fixed claim, and the final hard link +cannot overwrite a lock created by another process. An error-after-effect is +reconciled from exact record bytes. The old owner never deletes or overwrites +an unknown primary. + +A managed flush or proof failure retains the active lock. Normal per-session +close preserves the existing release behavior, including exact-owner cleanup. +If a normal release races managed shutdown and commits first, the missing +primary is already a safe handoff and the replacement performs ordinary +acquisition. + +## Certified takeover + +Only a Config created under a trusted managed parent enables certified +takeover. When acquisition observes a sealed record, it: + +1. verifies that the record's relative key matches the Config's expected + transcript; +2. opens and hashes that transcript outside the fixed claim, retaining the + file descriptor and its identity; +3. acquires the fixed claim; +4. re-reads the exact sealed primary and revalidates the kept-open descriptor, + path identity, metadata, byte length, and digest; +5. renames the sealed primary to a candidate-unique retired path; +6. hard-links the synced active candidate from the claim into the primary + path without replacement; and +7. removes only the exact retired record and its own claim. + +The lease then performs the existing authoritative session reload and +transcript fence before recorder activation. Two replacements racing the same +sealed record can produce at most one active owner. A loser receives a +conflict or unavailable result depending on whether it observes the winner's +active primary or an in-progress/residual claim. + +## Failure contract + +| Condition | Result | +| ---------------------------------------------------------------------------------- | ---------------------------------- | +| Valid active owner, including a foreign-host or managed dead-PID record | `session_writer_conflict` / 409 | +| Sealed proof does not match the expected transcript | `session_transcript_changed` / 409 | +| Malformed record, non-regular path, residual claim, or uncertain filesystem result | `session_writer_unavailable` / 503 | +| Current writer no longer owns its exact active record | `session_writer_lost` / 409 | + +Public errors remain sanitized. Successful seal and takeover logs include the +session ID, previous hostname/PID, and sealing time, but never the owner token +or transcript path. + +## Compatibility and rollout + +The feature gate must remain disabled during a mixed-version rollout. Enabling +or disabling it requires draining old ACP processes. A schema v2 reader still +accepts schema v1 active records, but an older reader does not understand +schema v2. Rollback therefore requires draining all new writers and confirming +that no active, sealed, or claim record from this protocol remains. + +Hashing is intentionally performed at seal and takeover instead of adding an +incremental digest to every append. This keeps the first implementation small +and makes the proof independent of process memory. A very large transcript may +cause managed sealing to exceed the parent deadline; that fails closed with +the active lock retained and is observable as an unclean shutdown. + +## Verification + +Unit and multiprocess coverage must prove: + +- schema v1 and v2 active locks keep their existing live/stale behavior; +- managed acquisition never reclaims an active dead-PID or foreign-host lock; +- successful sealing records the exact relative key, existence, byte length, + and SHA-256 digest; +- a certified replacement reloads the sealed transcript and becomes active; +- two replacements racing one sealed record elect exactly one owner; +- transcript modification, replacement, truncation, proof corruption, + malformed sealed records, and residual claims fail closed; +- failures before and after each primary-path transition never overwrite or + delete an unknown successor; +- managed flush failure retains the active primary; +- normal recorder close releases instead of sealing; and +- the default-off and standalone ACP paths remain unchanged. diff --git a/packages/cli/src/acp-integration/acpAgent.test.ts b/packages/cli/src/acp-integration/acpAgent.test.ts index fbe3603a332..85c1ed15d4c 100644 --- a/packages/cli/src/acp-integration/acpAgent.test.ts +++ b/packages/cli/src/acp-integration/acpAgent.test.ts @@ -2074,6 +2074,12 @@ describe('QwenAgent MCP SSE/HTTP support', () => { expect(innerConfig.setSessionWriterReclaimPolicy).toHaveBeenCalledWith( 'never', ); + expect(innerConfig.setSessionWriterTakeoverPolicy).toHaveBeenCalledWith( + 'certified', + ); + expect(innerConfig.closeSessionWriter).toHaveBeenCalledWith({ + handoff: true, + }); expect(order).toEqual(['writer', 'resources']); expect(mockRunExitCleanup).toHaveBeenCalledOnce(); }); @@ -2968,6 +2974,7 @@ describe('QwenAgent MCP SSE/HTTP support', () => { shutdown: vi.fn().mockResolvedValue(undefined), closeSessionWriter: vi.fn().mockResolvedValue(undefined), setSessionWriterReclaimPolicy: vi.fn(), + setSessionWriterTakeoverPolicy: vi.fn(), waitForMcpReady: vi.fn().mockResolvedValue(undefined), getModelsConfig: vi.fn().mockReturnValue({ getCurrentAuthType: vi.fn().mockReturnValue('api-key'), diff --git a/packages/cli/src/acp-integration/acpAgent.ts b/packages/cli/src/acp-integration/acpAgent.ts index 6e6b98fb966..99257c63de9 100644 --- a/packages/cli/src/acp-integration/acpAgent.ts +++ b/packages/cli/src/acp-integration/acpAgent.ts @@ -3451,7 +3451,7 @@ class QwenAgent implements Agent { const writerTerminals: Array> = []; for (const config of configList) { try { - writerTerminals.push(config.closeSessionWriter()); + writerTerminals.push(config.closeSessionWriter({ handoff: true })); } catch (error) { writerTerminals.push(Promise.reject(error)); } @@ -10776,6 +10776,7 @@ class QwenAgent implements Agent { this.assertManagedSessionAdmission(); if (this.isTrustedManagedParent()) { config.setSessionWriterReclaimPolicy('never'); + config.setSessionWriterTakeoverPolicy('certified'); } } catch (error) { return this.cleanupAfterRequestFailure(error, () => diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index 20351214950..5c049d26a7b 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -2772,6 +2772,29 @@ describe('Server Config (config.ts)', () => { }); describe('initialize', () => { + it('accepts managed handoff only after certified takeover is configured', async () => { + const standalone = new Config(baseParams); + await standalone.closeSessionWriter({ handoff: true }); + expect( + ( + standalone as unknown as { + sessionWriterHandoffRequested: boolean; + } + ).sessionWriterHandoffRequested, + ).toBe(false); + + const managed = new Config(baseParams); + managed.setSessionWriterTakeoverPolicy('certified'); + await managed.closeSessionWriter({ handoff: true }); + expect( + ( + managed as unknown as { + sessionWriterHandoffRequested: boolean; + } + ).sessionWriterHandoffRequested, + ).toBe(true); + }); + it.each([ [ 'an ACP session without an opt-in', diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 5e62f849288..dce2cafe3d0 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -1649,7 +1649,9 @@ export class Config { private sessionProjectDirRegistered = false; private pendingSessionWriterLease?: SessionWriterLease; private sessionWriterReclaimPolicy: 'local' | 'never' = 'local'; + private sessionWriterTakeoverPolicy: 'never' | 'certified' = 'never'; private sessionWriterShutdownRequested = false; + private sessionWriterHandoffRequested = false; private sessionWriterActivationPromise: Promise | undefined; private sessionWriterClosePromise: Promise | undefined; /** @@ -2989,6 +2991,7 @@ export class Config { processKind: 'acp', qwenVersion: this.cliVersion ?? null, reclaimPolicy: this.sessionWriterReclaimPolicy, + takeoverPolicy: this.sessionWriterTakeoverPolicy, onOwnershipAcquired: (acquiredLease) => { lease = acquiredLease; this.pendingSessionWriterLease = acquiredLease; @@ -7031,9 +7034,21 @@ export class Config { this.sessionWriterReclaimPolicy = policy; } - closeSessionWriter(): Promise { + setSessionWriterTakeoverPolicy(policy: 'never' | 'certified'): void { + if (this.initialized) { + throw new SessionWriterUnavailableError(); + } + this.sessionWriterTakeoverPolicy = policy; + } + + closeSessionWriter(options?: { handoff?: boolean }): Promise { + if (options?.handoff && this.sessionWriterTakeoverPolicy === 'certified') { + this.sessionWriterHandoffRequested = true; + } this.sessionWriterShutdownRequested = true; - this.chatRecordingService?.beginClose(); + this.chatRecordingService?.beginClose({ + handoff: this.sessionWriterHandoffRequested, + }); this.sessionWriterClosePromise ??= this.closeSessionWriterOnce(); return this.sessionWriterClosePromise; } @@ -7048,7 +7063,9 @@ export class Config { } } try { - await this.chatRecordingService?.close(); + await this.chatRecordingService?.close({ + handoff: this.sessionWriterHandoffRequested, + }); } catch (error) { failures.push(error); } diff --git a/packages/core/src/services/chatRecordingService.test.ts b/packages/core/src/services/chatRecordingService.test.ts index b9d57209330..4f93ba4dda1 100644 --- a/packages/core/src/services/chatRecordingService.test.ts +++ b/packages/core/src/services/chatRecordingService.test.ts @@ -111,6 +111,7 @@ describe('ChatRecordingService', () => { ), assertOwnedAndUnchanged: vi.fn().mockResolvedValue(undefined), release: vi.fn().mockResolvedValue(undefined), + sealForHandoff: vi.fn().mockResolvedValue(undefined), } as unknown as SessionWriterLease; chatRecordingService = activateRecording( new ChatRecordingService(mockConfig), @@ -1880,6 +1881,30 @@ describe('ChatRecordingService', () => { }); describe('close', () => { + it('seals instead of releasing after a successful handoff drain', async () => { + chatRecordingService.recordUserMessage([{ text: 'durable' }]); + + await expect( + chatRecordingService.close({ handoff: true }), + ).resolves.toBeUndefined(); + expect(mockLease.sealForHandoff).toHaveBeenCalledOnce(); + expect(mockLease.release).not.toHaveBeenCalled(); + expect(chatRecordingService.hasWriteOwnership()).toBe(false); + }); + + it('retains ownership when a handoff drain fails', async () => { + const failure = new SessionTranscriptChangedError(); + vi.mocked(mockLease.appendJsonLine).mockRejectedValueOnce(failure); + chatRecordingService.recordUserMessage([{ text: 'not durable' }]); + + await expect(chatRecordingService.close({ handoff: true })).rejects.toBe( + failure, + ); + expect(mockLease.sealForHandoff).not.toHaveBeenCalled(); + expect(mockLease.release).not.toHaveBeenCalled(); + expect(chatRecordingService.hasWriteOwnership()).toBe(true); + }); + it('releases the writer lease before reporting a flush failure', async () => { const failure = new SessionTranscriptChangedError(); vi.mocked(mockLease.appendJsonLine).mockRejectedValueOnce(failure); diff --git a/packages/core/src/services/chatRecordingService.ts b/packages/core/src/services/chatRecordingService.ts index f4136f10c8b..9c87dc091e8 100644 --- a/packages/core/src/services/chatRecordingService.ts +++ b/packages/core/src/services/chatRecordingService.ts @@ -611,6 +611,7 @@ export class ChatRecordingService { private operationTail: Promise = Promise.resolve(); private acceptingWrites = false; private closePromise: Promise | undefined; + private handoffRequested = false; /** First async JSONL write failure; permanently degrades this recorder. */ private writeFailure: Error | undefined; private integrityFailure: Error | undefined; @@ -1152,15 +1153,21 @@ export class ChatRecordingService { } } - close(): Promise { + close(options?: { handoff?: boolean }): Promise { + if (options?.handoff) { + this.handoffRequested = true; + } if (this.closePromise) return this.closePromise; if (this.state === 'closed') return Promise.resolve(); - this.beginClose(); + this.beginClose(options); this.closePromise = this.closeOnce(); return this.closePromise; } - beginClose(): void { + beginClose(options?: { handoff?: boolean }): void { + if (options?.handoff) { + this.handoffRequested = true; + } this.autoTitleController?.abort(); this.acceptingWrites = false; if (this.state === 'active') { @@ -1175,9 +1182,17 @@ export class ChatRecordingService { } catch (error) { flushFailure = error; } + if (this.handoffRequested && flushFailure !== undefined) { + this.state = 'integrity_failed'; + throw flushFailure; + } const lease = this.binding?.lease; try { - await lease?.release(); + if (this.handoffRequested) { + await lease?.sealForHandoff(); + } else { + await lease?.release(); + } this.binding = undefined; this.state = 'closed'; } catch (error) { diff --git a/packages/core/src/services/session-writer-lease.test.ts b/packages/core/src/services/session-writer-lease.test.ts index 49fc5c2d1f3..49b8eab3b24 100644 --- a/packages/core/src/services/session-writer-lease.test.ts +++ b/packages/core/src/services/session-writer-lease.test.ts @@ -5,6 +5,7 @@ */ import { fork, type ChildProcess } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { chmodSync, unlinkSync } from 'node:fs'; import * as fs from 'node:fs/promises'; import * as os from 'node:os'; @@ -42,6 +43,21 @@ const lstatFault = vi.hoisted(() => ({ calls: 0, })); +const transitionFault = vi.hoisted(() => ({ + renameFrom: undefined as string | undefined, + renameTo: undefined as string | undefined, + afterRename: undefined as (() => Promise) | undefined, + linkFrom: undefined as string | undefined, + linkTo: undefined as string | undefined, + throwAfterLink: false, +})); + +const writeFault = vi.hoisted(() => ({ + contains: undefined as string | undefined, + onEntered: undefined as (() => void) | undefined, + wait: undefined as Promise | undefined, +})); + vi.mock('node:fs/promises', async (importOriginal) => { const actual = await importOriginal(); return { @@ -58,6 +74,50 @@ vi.mock('node:fs/promises', async (importOriginal) => { } return actual.lstat(filePath); }, + rename: async ( + oldPath: Parameters[0], + newPath: Parameters[1], + ) => { + await actual.rename(oldPath, newPath); + if ( + oldPath === transitionFault.renameFrom && + (transitionFault.renameTo === undefined || + newPath === transitionFault.renameTo) + ) { + await transitionFault.afterRename?.(); + } + }, + link: async ( + existingPath: Parameters[0], + newPath: Parameters[1], + ) => { + await actual.link(existingPath, newPath); + if ( + transitionFault.throwAfterLink && + existingPath === transitionFault.linkFrom && + newPath === transitionFault.linkTo + ) { + throw Object.assign(new Error('injected error after link'), { + code: 'EIO', + }); + } + }, + open: async (...args: Parameters) => { + const handle = await actual.open(...args); + const writeFile = handle.writeFile.bind(handle); + handle.writeFile = async (data, options) => { + if ( + writeFault.contains && + Buffer.isBuffer(data) && + data.toString('utf8').includes(writeFault.contains) + ) { + writeFault.onEntered?.(); + await writeFault.wait; + } + return writeFile(data, options); + }; + return handle; + }, }; }); @@ -162,6 +222,15 @@ afterEach(async () => { lstatFault.path = undefined; lstatFault.remainingFailures = 0; lstatFault.calls = 0; + transitionFault.renameFrom = undefined; + transitionFault.renameTo = undefined; + transitionFault.afterRename = undefined; + transitionFault.linkFrom = undefined; + transitionFault.linkTo = undefined; + transitionFault.throwAfterLink = false; + writeFault.contains = undefined; + writeFault.onEntered = undefined; + writeFault.wait = undefined; setDebugLogSession(null); resetDebugLoggingState(); Storage.setRuntimeBaseDir(null); @@ -280,6 +349,71 @@ describe('SessionWriterLease', () => { ).rejects.toMatchObject({ code: 'ENOENT' }); }); + it('hands a real managed ACP Config to a certified replacement', async () => { + const fixture = await createFixture('managed-config-handoff-session'); + const createConfig = () => + Storage.runWithRuntimeBaseDir( + fixture.runtimeBaseDir, + fixture.projectRoot, + () => + new Config({ + sessionId: fixture.options.sessionId, + cwd: fixture.projectRoot, + targetDir: fixture.projectRoot, + debugMode: false, + model: 'test-model', + chatRecording: true, + experimentalZedIntegration: true, + sessionWriterLeaseEnabled: true, + bareMode: true, + telemetry: { enabled: false }, + usageStatisticsEnabled: false, + }), + ); + const initialize = (config: Config) => + config.initialize({ + skipGeminiInitialization: true, + skipHooks: true, + skipMcpDiscovery: true, + skipSkillManager: true, + skipFileCheckpointing: true, + lenientToolWarmup: true, + }); + + const first = createConfig(); + first.setSessionWriterReclaimPolicy('never'); + first.setSessionWriterTakeoverPolicy('certified'); + await initialize(first); + first.getChatRecordingService()?.recordUserMessage('handoff tail'); + await first.closeSessionWriter({ handoff: true }); + expect( + JSON.parse( + await fs.readFile( + getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ), + 'utf8', + ), + ), + ).toMatchObject({ schema_version: 2, state: 'sealed' }); + + const replacement = createConfig(); + replacement.setSessionWriterReclaimPolicy('never'); + replacement.setSessionWriterTakeoverPolicy('certified'); + await initialize(replacement); + expect( + replacement.getResumedSessionData()?.conversation.messages.at(-1)?.message + ?.parts, + ).toEqual([{ text: 'handoff tail' }]); + + await first.shutdown({ + shutdownTelemetry: false, + skipSessionWriter: true, + }); + await replacement.shutdown({ shutdownTelemetry: false }); + }); + it('restores and re-anchors a persisted title outside the active UUID chain', async () => { const fixture = await createFixture('11111111-1111-4111-8111-111111111111'); const firstUser = record( @@ -814,6 +948,385 @@ describe('SessionWriterLease', () => { ).rejects.toBeInstanceOf(SessionWriterConflictError); }); + it('never treats a foreign-host active record as a certified handoff', async () => { + const fixture = await createFixture('foreign-active-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const active = JSON.parse(await fs.readFile(lockPath, 'utf8')) as Record< + string, + unknown + >; + await first.release(); + await fs.writeFile( + lockPath, + JSON.stringify({ + ...active, + hostname: 'retired-foreign-host', + pid: 2_147_483_647, + }), + ); + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterConflictError); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'active', + hostname: 'retired-foreign-host', + }); + }); + + it('keeps schema v1 records on the active-owner path', async () => { + const fixture = await createFixture('legacy-active-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const active = JSON.parse(await fs.readFile(lockPath, 'utf8')) as Record< + string, + unknown + >; + await first.release(); + delete active['state']; + active['schema_version'] = 1; + await fs.writeFile(lockPath, JSON.stringify(active)); + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterConflictError); + + await fs.writeFile( + lockPath, + JSON.stringify({ + ...active, + pid: 2_147_483_647, + }), + ); + const replacement = await SessionWriterLease.acquire(fixture.options); + await replacement.release(); + }); + + it('seals a transcript proof and permits only certified takeover', async () => { + const fixture = await createFixture('sealed-session'); + const initial = `${JSON.stringify({ record: 'initial' })}\n`; + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, initial); + const first = await SessionWriterLease.acquire(fixture.options); + await first.appendJsonLine({ record: 'final' }); + const expectedTranscript = await fs.readFile(fixture.transcriptPath); + + await first.sealForHandoff(); + + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealed = JSON.parse(await fs.readFile(lockPath, 'utf8')) as { + schema_version: number; + state: string; + transcript: { + relative_path: string; + exists: boolean; + byte_length: number; + sha256: string; + }; + }; + expect(sealed).toMatchObject({ + schema_version: 2, + state: 'sealed', + transcript: { + relative_path: path + .relative(fixture.runtimeBaseDir, fixture.transcriptPath) + .split(path.sep) + .join('/'), + exists: true, + byte_length: expectedTranscript.byteLength, + sha256: createHash('sha256').update(expectedTranscript).digest('hex'), + }, + }); + await expect( + first.appendJsonLine({ record: 'too-late' }), + ).rejects.toBeInstanceOf(SessionWriterLostError); + await expect(fs.readFile(fixture.transcriptPath)).resolves.toEqual( + expectedTranscript, + ); + await expect( + SessionWriterLease.acquire(fixture.options), + ).rejects.toBeInstanceOf(SessionWriterConflictError); + + const replacement = await SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }); + expect(replacement.ownerId).not.toBe(first.ownerId); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + schema_version: 2, + state: 'active', + owner_id: replacement.ownerId, + }); + await replacement.assertOwnedAndUnchanged(); + await replacement.release(); + }); + + it('waits for an accepted append before sealing the transcript', async () => { + const fixture = await createFixture('sealed-append-race-session'); + const lease = await SessionWriterLease.acquire(fixture.options); + let resumeWrite: (() => void) | undefined; + writeFault.contains = '"late":true'; + writeFault.wait = new Promise((resolve) => { + resumeWrite = resolve; + }); + const writeEntered = new Promise((resolve) => { + writeFault.onEntered = resolve; + }); + + const append = lease.appendJsonLine({ late: true }); + await writeEntered; + const seal = lease.sealForHandoff(); + await expect( + Promise.race([ + seal.then( + () => 'settled', + () => 'settled', + ), + new Promise<'pending'>((resolve) => + setTimeout(() => resolve('pending'), 50), + ), + ]), + ).resolves.toBe('pending'); + + resumeWrite?.(); + await append; + await seal; + const transcript = await fs.readFile(fixture.transcriptPath); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealed = JSON.parse(await fs.readFile(lockPath, 'utf8')) as { + transcript: { byte_length: number; sha256: string }; + }; + expect(transcript.toString('utf8')).toBe('{"late":true}\n'); + expect(sealed.transcript).toMatchObject({ + byte_length: transcript.byteLength, + sha256: createHash('sha256').update(transcript).digest('hex'), + }); + }); + + it('reconciles a sealing error reported after the sealed primary is installed', async () => { + const fixture = await createFixture('sealed-after-effect-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + transitionFault.linkFrom = `${lockPath}.sealed-candidate.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkTo = lockPath; + transitionFault.throwAfterLink = true; + + await expect(first.sealForHandoff()).resolves.toBeUndefined(); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + schema_version: 2, + state: 'sealed', + }); + }); + + it('never overwrites a primary installed during the sealing transition', async () => { + const fixture = await createFixture('sealed-successor-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const successorRaw = '{"successor":true}'; + transitionFault.renameFrom = lockPath; + transitionFault.renameTo = `${lockPath}.handoff.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.afterRename = () => + fs.writeFile(lockPath, successorRaw, 'utf8'); + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterLostError, + ); + expect(first.isReleased).toBe(true); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(successorRaw); + }); + + it('elects exactly one certified replacement for a sealed session', async () => { + const fixture = await createFixture('sealed-race-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + + const contenders = [startLeaseProcess(), startLeaseProcess()]; + const options = { + ...fixture.options, + reclaimPolicy: 'never' as const, + takeoverPolicy: 'certified' as const, + }; + const results = await Promise.all( + contenders.map((child) => + requestChild(child, { type: 'acquire', options }), + ), + ); + expect(results.filter((result) => result.ok)).toHaveLength(1); + const winner = contenders[results.findIndex((result) => result.ok)]!; + expect(await requestChild(winner, { type: 'release' })).toMatchObject({ + ok: true, + }); + }); + + it('reconciles a takeover error reported after the active primary is installed', async () => { + const fixture = await createFixture('takeover-after-effect-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + transitionFault.linkFrom = `${lockPath}.claim`; + transitionFault.linkTo = lockPath; + transitionFault.throwAfterLink = true; + + const replacement = await SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + schema_version: 2, + state: 'active', + owner_id: replacement.ownerId, + }); + await replacement.release(); + }); + + it('never overwrites a primary installed during the takeover transition', async () => { + const fixture = await createFixture('takeover-successor-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const successorRaw = '{"successor":true}'; + transitionFault.renameFrom = lockPath; + transitionFault.afterRename = () => + fs.writeFile(lockPath, successorRaw, 'utf8'); + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterLostError); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(successorRaw); + }); + + it.each(['append', 'truncate', 'replace'] as const)( + 'retains a sealed lock when the transcript proof changes by %s', + async (mutation) => { + const fixture = await createFixture(`sealed-${mutation}-session`); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealedRaw = await fs.readFile(lockPath, 'utf8'); + if (mutation === 'append') { + await fs.appendFile(fixture.transcriptPath, '{"external":true}\n'); + } else if (mutation === 'truncate') { + await fs.truncate(fixture.transcriptPath, 0); + } else { + const replacementPath = `${fixture.transcriptPath}.replacement`; + await fs.writeFile(replacementPath, '{"record":"evil"}\n'); + await fs.rename(replacementPath, fixture.transcriptPath); + } + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionTranscriptChangedError); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(sealedRaw); + }, + ); + + it.each([ + ['valid but mismatched', '0'.repeat(64), SessionTranscriptChangedError], + ['malformed', 'invalid', SessionWriterUnavailableError], + ])( + 'retains a sealed primary with a %s transcript digest', + async (_description, sha256, ErrorType) => { + const fixture = await createFixture('sealed-proof-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealed = JSON.parse(await fs.readFile(lockPath, 'utf8')) as { + transcript: { sha256: string }; + }; + sealed.transcript.sha256 = sha256; + const sealedRaw = JSON.stringify(sealed); + await fs.writeFile(lockPath, sealedRaw); + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(ErrorType); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(sealedRaw); + }, + ); + + it('fails closed without changing a sealed primary when a claim remains', async () => { + const fixture = await createFixture('sealed-claim-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealedRaw = await fs.readFile(lockPath, 'utf8'); + await fs.writeFile(`${lockPath}.claim`, '{"residual":true}'); + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(sealedRaw); + }); + it('cannot remove a successor lock after release commits', async () => { const fixture = await createFixture(); const first = await SessionWriterLease.acquire(fixture.options); diff --git a/packages/core/src/services/session-writer-lease.ts b/packages/core/src/services/session-writer-lease.ts index 96bd736d04c..42b9082ddd9 100644 --- a/packages/core/src/services/session-writer-lease.ts +++ b/packages/core/src/services/session-writer-lease.ts @@ -5,14 +5,15 @@ */ import { execFile } from 'node:child_process'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import type { Stats } from 'node:fs'; import * as fs from 'node:fs/promises'; import * as os from 'node:os'; import * as path from 'node:path'; import { createDebugLogger } from '../utils/debugLogger.js'; -const LOCK_SCHEMA_VERSION = 1; +const LEGACY_LOCK_SCHEMA_VERSION = 1; +const LOCK_SCHEMA_VERSION = 2; const MALFORMED_RETRY_COUNT = 3; const MALFORMED_RETRY_DELAY_MS = 50; const RELEASE_PRECHECK_ATTEMPTS = 3; @@ -102,8 +103,7 @@ export class SessionWriterUnavailableError extends SessionWriterError { } } -interface SessionWriterLockRecord { - schema_version: number; +interface SessionWriterOwnerRecord { session_id: string; owner_id: string; pid: number; @@ -114,6 +114,38 @@ interface SessionWriterLockRecord { qwen_version: string | null; } +interface LegacySessionWriterLockRecord extends SessionWriterOwnerRecord { + schema_version: typeof LEGACY_LOCK_SCHEMA_VERSION; +} + +interface ActiveSessionWriterLockRecord extends SessionWriterOwnerRecord { + schema_version: typeof LOCK_SCHEMA_VERSION; + state: 'active'; +} + +interface SealedTranscriptProof { + relative_path: string; + exists: boolean; + byte_length: number; + sha256: string; +} + +interface SealedSessionWriterLockRecord extends SessionWriterOwnerRecord { + schema_version: typeof LOCK_SCHEMA_VERSION; + state: 'sealed'; + sealed_at: string; + transcript: SealedTranscriptProof; +} + +type SessionWriterLockRecord = + | LegacySessionWriterLockRecord + | ActiveSessionWriterLockRecord + | SealedSessionWriterLockRecord; + +type ActiveLockRecord = + | LegacySessionWriterLockRecord + | ActiveSessionWriterLockRecord; + export interface AcquireSessionWriterLeaseOptions { runtimeBaseDir: string; sessionId: string; @@ -121,13 +153,15 @@ export interface AcquireSessionWriterLeaseOptions { processKind?: SessionWriterProcessKind; qwenVersion?: string | null; reclaimPolicy?: 'local' | 'never'; + takeoverPolicy?: 'never' | 'certified'; onOwnershipAcquired?: (lease: SessionWriterLease) => void; } type ExistingLockState = | { kind: 'missing' } - | { kind: 'live' } - | { kind: 'stale'; record: SessionWriterLockRecord } + | { kind: 'live'; record: ActiveLockRecord; raw: string } + | { kind: 'stale'; record: ActiveLockRecord; raw: string } + | { kind: 'sealed'; record: SealedSessionWriterLockRecord; raw: string } | { kind: 'malformed' }; interface TranscriptFingerprint { @@ -146,6 +180,12 @@ type TranscriptState = fingerprint: TranscriptFingerprint; }; +interface OpenTranscriptProof { + readonly state: TranscriptState; + readonly sha256: string; + readonly handle?: fs.FileHandle; +} + function isProcessAlive(pid: number): boolean { try { process.kill(pid, 0); @@ -226,12 +266,11 @@ async function readProcessStartIdentity(pid: number): Promise { return null; } -function isLockRecord(value: unknown): value is SessionWriterLockRecord { - if (!value || typeof value !== 'object' || Array.isArray(value)) return false; - const record = value as Record; +function hasValidOwnerFields( + record: Record, +): record is Record & SessionWriterOwnerRecord { const processKind = record['process_kind']; return ( - record['schema_version'] === LOCK_SCHEMA_VERSION && typeof record['session_id'] === 'string' && record['session_id'].length > 0 && typeof record['owner_id'] === 'string' && @@ -252,6 +291,40 @@ function isLockRecord(value: unknown): value is SessionWriterLockRecord { ); } +function isSealedTranscriptProof( + value: unknown, +): value is SealedTranscriptProof { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const proof = value as Record; + return ( + typeof proof['relative_path'] === 'string' && + proof['relative_path'].length > 0 && + typeof proof['exists'] === 'boolean' && + Number.isSafeInteger(proof['byte_length']) && + (proof['byte_length'] as number) >= 0 && + typeof proof['sha256'] === 'string' && + /^[0-9a-f]{64}$/.test(proof['sha256']) && + (proof['exists'] || proof['byte_length'] === 0) + ); +} + +function isLockRecord(value: unknown): value is SessionWriterLockRecord { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const record = value as Record; + if (!hasValidOwnerFields(record)) return false; + if (record['schema_version'] === LEGACY_LOCK_SCHEMA_VERSION) { + return record['state'] === undefined; + } + if (record['schema_version'] !== LOCK_SCHEMA_VERSION) return false; + if (record['state'] === 'active') return true; + return ( + record['state'] === 'sealed' && + typeof record['sealed_at'] === 'string' && + Number.isFinite(Date.parse(record['sealed_at'])) && + isSealedTranscriptProof(record['transcript']) + ); +} + function parseLockRecord(raw: string): SessionWriterLockRecord | null { try { const parsed: unknown = JSON.parse(raw); @@ -261,17 +334,27 @@ function parseLockRecord(raw: string): SessionWriterLockRecord | null { } } -async function lockStateForRecord( +function isActiveLockRecord( record: SessionWriterLockRecord, +): record is ActiveLockRecord { + return ( + record.schema_version === LEGACY_LOCK_SCHEMA_VERSION || + record.state === 'active' + ); +} + +async function lockStateForRecord( + record: ActiveLockRecord, + raw: string, ): Promise { - if (record.hostname !== os.hostname()) return { kind: 'live' }; - if (!isProcessAlive(record.pid)) return { kind: 'stale', record }; - if (!record.process_start_identity) return { kind: 'live' }; + if (record.hostname !== os.hostname()) return { kind: 'live', record, raw }; + if (!isProcessAlive(record.pid)) return { kind: 'stale', record, raw }; + if (!record.process_start_identity) return { kind: 'live', record, raw }; const currentStartIdentity = await readProcessStartIdentity(record.pid); return currentStartIdentity !== null && currentStartIdentity !== record.process_start_identity - ? { kind: 'stale', record } - : { kind: 'live' }; + ? { kind: 'stale', record, raw } + : { kind: 'live', record, raw }; } async function delay(ms: number): Promise { @@ -367,6 +450,198 @@ async function getTranscriptState(filePath: string): Promise { } } +function getTranscriptRelativePath( + runtimeBaseDir: string, + transcriptPath: string, +): string { + const relativePath = path.relative(runtimeBaseDir, transcriptPath); + if ( + relativePath.length === 0 || + path.isAbsolute(relativePath) || + relativePath === '..' || + relativePath.startsWith(`..${path.sep}`) + ) { + throw new SessionWriterUnavailableError({ + cause: new Error('Session transcript is outside the runtime base'), + }); + } + return relativePath.split(path.sep).join('/'); +} + +async function openTranscriptProof( + filePath: string, +): Promise { + let handle: fs.FileHandle | undefined; + try { + try { + handle = await fs.open(filePath, 'r'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return { + state: { exists: false, byteLength: 0 }, + sha256: createHash('sha256').digest('hex'), + }; + } + throw error; + } + + const [beforeStat, pathStat] = await Promise.all([ + handle.stat(), + fs.lstat(filePath), + ]); + if ( + !beforeStat.isFile() || + !pathStat.isFile() || + pathStat.isSymbolicLink() + ) { + throw new SessionWriterUnavailableError(); + } + const beforeState: TranscriptState = { + exists: true, + byteLength: beforeStat.size, + fingerprint: transcriptFingerprint(beforeStat), + }; + if ( + !sameFileIdentity( + beforeState.fingerprint, + transcriptFingerprint(pathStat), + ) + ) { + throw new SessionTranscriptChangedError(); + } + if (beforeStat.size > 0) { + const lastByte = Buffer.allocUnsafe(1); + const { bytesRead } = await handle.read( + lastByte, + 0, + 1, + beforeStat.size - 1, + ); + if (bytesRead !== 1 || lastByte[0] !== 0x0a) { + throw new SessionTranscriptChangedError(); + } + } + + const hash = createHash('sha256'); + const buffer = Buffer.allocUnsafe(64 * 1024); + let position = 0; + while (position < beforeStat.size) { + const { bytesRead } = await handle.read( + buffer, + 0, + Math.min(buffer.byteLength, beforeStat.size - position), + position, + ); + if (bytesRead === 0) throw new SessionTranscriptChangedError(); + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + + const [afterStat, afterPathStat] = await Promise.all([ + handle.stat(), + fs.lstat(filePath), + ]); + const afterState: TranscriptState = { + exists: true, + byteLength: afterStat.size, + fingerprint: transcriptFingerprint(afterStat), + }; + if ( + !sameTranscriptState(beforeState, afterState) || + !afterPathStat.isFile() || + afterPathStat.isSymbolicLink() || + !sameFileIdentity( + afterState.fingerprint, + transcriptFingerprint(afterPathStat), + ) + ) { + throw new SessionTranscriptChangedError(); + } + + const proof = { + state: beforeState, + sha256: hash.digest('hex'), + handle, + }; + handle = undefined; + return proof; + } catch (error) { + if (error instanceof SessionWriterError) throw error; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } finally { + await handle?.close().catch(() => {}); + } +} + +async function validateOpenTranscriptProof( + filePath: string, + proof: OpenTranscriptProof, +): Promise { + if (!proof.state.exists) { + const current = await getTranscriptState(filePath); + if (!sameTranscriptState(current, proof.state)) { + throw new SessionTranscriptChangedError(); + } + return; + } + const handle = proof.handle; + if (!handle) throw new SessionWriterUnavailableError(); + try { + const [handleStat, pathStat] = await Promise.all([ + handle.stat(), + fs.lstat(filePath), + ]); + if ( + !handleStat.isFile() || + !pathStat.isFile() || + pathStat.isSymbolicLink() + ) { + throw new SessionWriterUnavailableError(); + } + const current: TranscriptState = { + exists: true, + byteLength: handleStat.size, + fingerprint: transcriptFingerprint(handleStat), + }; + if ( + !sameTranscriptState(current, proof.state) || + !sameFileIdentity(current.fingerprint, transcriptFingerprint(pathStat)) + ) { + throw new SessionTranscriptChangedError(); + } + } catch (error) { + if (error instanceof SessionWriterError) throw error; + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + throw new SessionTranscriptChangedError(); + } + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } +} + +async function closeTranscriptProof(proof: OpenTranscriptProof): Promise { + await proof.handle?.close().catch(() => {}); +} + +function assertSealedProofMatches( + sealed: SealedSessionWriterLockRecord, + relativePath: string, + proof: OpenTranscriptProof, +): void { + const transcript = sealed.transcript; + if ( + transcript.relative_path !== relativePath || + transcript.exists !== proof.state.exists || + transcript.byte_length !== proof.state.byteLength || + transcript.sha256 !== proof.sha256 + ) { + throw new SessionTranscriptChangedError(); + } +} + async function restoreMovedLock( movedPath: string, lockPath: string, @@ -418,7 +693,7 @@ async function installLockRecord( async function acquireReclaimGuard( lockPath: string, staleOwnerId: string, - record: SessionWriterLockRecord, + record: ActiveLockRecord, inspect: ( lockPath: string, expectedSessionId: string, @@ -430,8 +705,11 @@ async function acquireReclaimGuard( if (await installLockRecord(guardPath, record)) return guardPath; const state = await inspect(guardPath, record.session_id); if (state.kind === 'missing') continue; - if (state.kind === 'live') throw new SessionWriterUnavailableError(); - if (state.kind === 'malformed') { + if ( + state.kind === 'live' || + state.kind === 'sealed' || + state.kind === 'malformed' + ) { throw new SessionWriterUnavailableError(); } guardPath = `${basePath}.${encodeURIComponent(state.record.owner_id)}`; @@ -444,12 +722,148 @@ async function removeOwnedLock( ownerId: string, ): Promise { const record = parseLockRecord(await fs.readFile(lockPath, 'utf8')); - if (!record || record.owner_id !== ownerId) { + if (!record || !isActiveLockRecord(record) || record.owner_id !== ownerId) { throw new SessionWriterLostError(); } await fs.unlink(lockPath); } +async function assertPathMissing(candidatePath: string): Promise { + try { + await fs.lstat(candidatePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + throw new SessionWriterUnavailableError({ + cause: new Error('Session writer transition claim already exists'), + }); +} + +async function removeExactRecord( + candidatePath: string, + expectedRaw: string, +): Promise { + let raw: string; + try { + raw = await fs.readFile(candidatePath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if (raw !== expectedRaw) throw new SessionWriterLostError(); + try { + await fs.unlink(candidatePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } +} + +async function inspectExactRecord( + candidatePath: string, + expectedRaw: string, +): Promise<'exact' | 'missing' | 'other' | 'unknown'> { + try { + const stat = await fs.lstat(candidatePath); + if (!stat.isFile() || stat.isSymbolicLink()) return 'other'; + const raw = await fs.readFile(candidatePath, 'utf8'); + return raw === expectedRaw ? 'exact' : 'other'; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT' + ? 'missing' + : 'unknown'; + } +} + +async function transitionExactPrimary( + lockPath: string, + expectedRaw: string, + replacementPath: string, + replacementRaw: string, + retiredPath: string, +): Promise { + try { + await fs.rename(lockPath, retiredPath); + await fs.link(replacementPath, lockPath); + return; + } catch (error) { + const [primaryState, primaryExpectedState, retiredState] = + await Promise.all([ + inspectExactRecord(lockPath, replacementRaw), + inspectExactRecord(lockPath, expectedRaw), + inspectExactRecord(retiredPath, expectedRaw), + ]); + if (primaryState === 'exact' && retiredState === 'exact') return; + if (primaryExpectedState === 'exact' && retiredState === 'missing') { + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if (primaryState === 'missing' && retiredState === 'exact') { + try { + await fs.link(retiredPath, lockPath); + await fs.unlink(retiredPath).catch(() => {}); + } catch (restoreError) { + throw new SessionWriterUnavailableError({ + cause: new AggregateError( + [error, restoreError], + 'Session writer primary transition could not be restored', + ), + }); + } + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if (retiredState === 'exact') { + await fs.unlink(retiredPath).catch(() => {}); + } + if (primaryState === 'other' && primaryExpectedState === 'other') { + throw new SessionWriterLostError(); + } + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } +} + +async function rollbackExactTransition( + lockPath: string, + replacementRaw: string, + retiredPath: string, + expectedRaw: string, +): Promise { + const primaryState = await inspectExactRecord(lockPath, replacementRaw); + if (primaryState === 'exact') { + await removeExactRecord(lockPath, replacementRaw); + } else if (primaryState !== 'missing') { + throw primaryState === 'other' + ? new SessionWriterLostError() + : new SessionWriterUnavailableError(); + } + const retiredState = await inspectExactRecord(retiredPath, expectedRaw); + if (retiredState !== 'exact') { + throw retiredState === 'other' + ? new SessionWriterLostError() + : new SessionWriterUnavailableError(); + } + try { + await fs.link(retiredPath, lockPath); + await fs.unlink(retiredPath).catch(() => {}); + } catch (error) { + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } +} + export function getSessionWriterLockPath( runtimeBaseDir: string, sessionId: string, @@ -469,13 +883,15 @@ export class SessionWriterLease { readonly transcriptPath: string; private expectedTranscriptState: TranscriptState | undefined; private released = false; - private releasePromise: Promise | undefined; + private terminalPromise: Promise | undefined; + private operationTail: Promise = Promise.resolve(); private readonly lockRecordRaw: string; private readonly retiredPath: string; + private readonly claimPath: string; private constructor( private readonly lockPath: string, - lockRecord: SessionWriterLockRecord, + lockRecord: ActiveLockRecord, options: AcquireSessionWriterLeaseOptions, ) { this.ownerId = lockRecord.owner_id; @@ -484,6 +900,7 @@ export class SessionWriterLease { this.transcriptPath = options.transcriptPath; this.lockRecordRaw = JSON.stringify(lockRecord); this.retiredPath = `${lockPath}.released.${encodeURIComponent(this.ownerId)}`; + this.claimPath = `${lockPath}.claim`; } get transcriptExistedAtAcquire(): boolean { @@ -546,8 +963,9 @@ export class SessionWriterLease { } const processStartIdentity = await readProcessStartIdentity(process.pid); - const lockRecord: SessionWriterLockRecord = { + const lockRecord: ActiveSessionWriterLockRecord = { schema_version: LOCK_SCHEMA_VERSION, + state: 'active', session_id: normalizedOptions.sessionId, owner_id: randomUUID(), pid: process.pid, @@ -559,9 +977,17 @@ export class SessionWriterLease { acquired_at: new Date().toISOString(), qwen_version: normalizedOptions.qwenVersion ?? null, }; + const claimPath = `${lockPath}.claim`; for (let attempt = 0; attempt < ACQUIRE_ATTEMPTS; attempt++) { + await assertPathMissing(claimPath); if (await installLockRecord(lockPath, lockRecord)) { + try { + await assertPathMissing(claimPath); + } catch (error) { + await removeOwnedLock(lockPath, lockRecord.owner_id).catch(() => {}); + throw error; + } return SessionWriterLease.finishAcquisition( lockPath, lockRecord, @@ -580,6 +1006,17 @@ export class SessionWriterLease { cause: new Error('Existing session writer lock is malformed'), }); } + if (state.kind === 'sealed') { + if (normalizedOptions.takeoverPolicy !== 'certified') { + throw new SessionWriterConflictError(); + } + return SessionWriterLease.takeOverSealed( + lockPath, + state, + lockRecord, + normalizedOptions, + ); + } if (normalizedOptions.reclaimPolicy === 'never') { throw new SessionWriterConflictError(); } @@ -596,6 +1033,7 @@ export class SessionWriterLease { let staleMoved = false; const stalePath = `${lockPath}.stale.${process.pid}.${randomUUID()}`; try { + await assertPathMissing(claimPath); const currentState = await SessionWriterLease.inspectExistingLock( lockPath, normalizedOptions.sessionId, @@ -630,6 +1068,7 @@ export class SessionWriterLease { throw new SessionWriterUnavailableError(); } primaryInstalled = true; + await assertPathMissing(claimPath); const lease = await SessionWriterLease.finishAcquisition( lockPath, lockRecord, @@ -655,10 +1094,116 @@ export class SessionWriterLease { throw new SessionWriterUnavailableError(); } + private static async takeOverSealed( + lockPath: string, + observed: Extract, + lockRecord: ActiveSessionWriterLockRecord, + options: AcquireSessionWriterLeaseOptions, + ): Promise { + const relativePath = getTranscriptRelativePath( + options.runtimeBaseDir, + options.transcriptPath, + ); + const proof = await openTranscriptProof(options.transcriptPath); + const claimPath = `${lockPath}.claim`; + const claimRaw = JSON.stringify(lockRecord); + const retiredPath = `${lockPath}.sealed.${encodeURIComponent( + observed.record.owner_id, + )}.${encodeURIComponent(lockRecord.owner_id)}`; + let claimAcquired = false; + let transitionCommitted = false; + try { + assertSealedProofMatches(observed.record, relativePath, proof); + await assertPathMissing(retiredPath); + if (!(await installLockRecord(claimPath, lockRecord))) { + throw new SessionWriterUnavailableError({ + cause: new Error('Session writer transition claim already exists'), + }); + } + claimAcquired = true; + const current = await SessionWriterLease.inspectExistingLock( + lockPath, + options.sessionId, + ); + if (current.kind !== 'sealed' || current.raw !== observed.raw) { + throw current.kind === 'live' + ? new SessionWriterConflictError() + : new SessionWriterUnavailableError(); + } + await validateOpenTranscriptProof(options.transcriptPath, proof); + assertSealedProofMatches(current.record, relativePath, proof); + await transitionExactPrimary( + lockPath, + observed.raw, + claimPath, + claimRaw, + retiredPath, + ); + transitionCommitted = true; + await validateOpenTranscriptProof(options.transcriptPath, proof); + const lease = await SessionWriterLease.finishAcquisition( + lockPath, + lockRecord, + options, + proof.state, + ); + await removeExactRecord(claimPath, claimRaw); + claimAcquired = false; + await removeExactRecord(retiredPath, observed.raw).catch((error) => { + debugLogger.debug( + `Session writer sealed lock cleanup failed path=${JSON.stringify(retiredPath)} ` + + `error=${describeDiagnosticError(error)}`, + ); + }); + debugLogger.info( + `Certified session writer handoff accepted sessionId=${JSON.stringify(options.sessionId)} ` + + `previousHostname=${JSON.stringify(observed.record.hostname)} ` + + `previousPid=${observed.record.pid} sealedAt=${observed.record.sealed_at}`, + ); + return lease; + } catch (error) { + const cleanupFailures: unknown[] = []; + if (transitionCommitted) { + try { + await rollbackExactTransition( + lockPath, + claimRaw, + retiredPath, + observed.raw, + ); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + } + if (claimAcquired) { + try { + await removeExactRecord(claimPath, claimRaw); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + } + if (cleanupFailures.length > 0) { + throw new SessionWriterUnavailableError({ + cause: new AggregateError( + [error, ...cleanupFailures], + 'Certified session writer takeover cleanup failed', + ), + }); + } + if (error instanceof SessionWriterError) throw error; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } finally { + await closeTranscriptProof(proof); + } + } + private static async finishAcquisition( lockPath: string, - lockRecord: SessionWriterLockRecord, + lockRecord: ActiveLockRecord, options: AcquireSessionWriterLeaseOptions, + requiredTranscriptState?: TranscriptState, ): Promise { const lease = new SessionWriterLease(lockPath, lockRecord, options); try { @@ -666,6 +1211,15 @@ export class SessionWriterLease { lease.expectedTranscriptState = await getTranscriptState( options.transcriptPath, ); + if ( + requiredTranscriptState && + !sameTranscriptState( + lease.expectedTranscriptState, + requiredTranscriptState, + ) + ) { + throw new SessionTranscriptChangedError(); + } return lease; } catch (error) { try { @@ -717,7 +1271,10 @@ export class SessionWriterLease { cause: new Error('Session writer lock belongs to another session'), }); } - return lockStateForRecord(record); + if (!isActiveLockRecord(record)) { + return { kind: 'sealed', record, raw }; + } + return lockStateForRecord(record, raw); } if (attempt + 1 < MALFORMED_RETRY_COUNT) { await delay(MALFORMED_RETRY_DELAY_MS); @@ -726,7 +1283,7 @@ export class SessionWriterLease { return { kind: 'malformed' }; } - private async readOwnedLock(): Promise { + private async readOwnedLock(): Promise { if (this.released) throw new SessionWriterLostError(); let stat: Awaited>; try { @@ -752,6 +1309,7 @@ export class SessionWriterLease { const record = parseLockRecord(raw); if ( !record || + !isActiveLockRecord(record) || record.owner_id !== this.ownerId || raw !== this.lockRecordRaw ) { @@ -760,7 +1318,11 @@ export class SessionWriterLease { return record; } - async assertOwnedAndUnchanged(): Promise { + assertOwnedAndUnchanged(): Promise { + return this.runExclusive(() => this.assertOwnedAndUnchangedOnce()); + } + + private async assertOwnedAndUnchangedOnce(): Promise { await this.readOwnedLock(); if (!this.expectedTranscriptState) { throw new SessionWriterUnavailableError(); @@ -771,7 +1333,11 @@ export class SessionWriterLease { } } - async appendJsonLine(value: unknown): Promise { + appendJsonLine(value: unknown): Promise { + return this.runExclusive(() => this.appendJsonLineOnce(value)); + } + + private async appendJsonLineOnce(value: unknown): Promise { let serialized: string | undefined; try { serialized = JSON.stringify(value); @@ -782,7 +1348,7 @@ export class SessionWriterLease { } if (serialized === undefined) throw new SessionWriterUnavailableError(); const bytes = Buffer.from(`${serialized}\n`, 'utf8'); - await this.assertOwnedAndUnchanged(); + await this.assertOwnedAndUnchangedOnce(); const expectedBefore = this.expectedTranscriptState; if (!expectedBefore) throw new SessionWriterUnavailableError(); const nextByteLength = expectedBefore.byteLength + bytes.byteLength; @@ -845,14 +1411,28 @@ export class SessionWriterLease { } release(): Promise { - this.releasePromise ??= this.releaseOnce(); - return this.releasePromise; + this.terminalPromise ??= this.runExclusive(() => this.releaseOnce()); + return this.terminalPromise; + } + + sealForHandoff(): Promise { + this.terminalPromise ??= this.runExclusive(() => this.sealForHandoffOnce()); + return this.terminalPromise; } get isReleased(): boolean { return this.released; } + private runExclusive(operation: () => Promise): Promise { + const pending = this.operationTail.then(operation, operation); + this.operationTail = pending.then( + () => undefined, + () => undefined, + ); + return pending; + } + private async releaseOnce(): Promise { if (this.released) return; await this.readOwnedLockForRelease(); @@ -887,11 +1467,140 @@ export class SessionWriterLease { } } - private async readOwnedLockForRelease(): Promise { + private async sealForHandoffOnce(): Promise { + if (this.released) return; + const ownedRecord = await this.readOwnedLockForRelease(); + if (!this.expectedTranscriptState) { + throw new SessionWriterUnavailableError(); + } + const relativePath = getTranscriptRelativePath( + this.runtimeBaseDir, + this.transcriptPath, + ); + const proof = await openTranscriptProof(this.transcriptPath); + const sealedRecord: SealedSessionWriterLockRecord = { + ...ownedRecord, + schema_version: LOCK_SCHEMA_VERSION, + state: 'sealed', + sealed_at: new Date().toISOString(), + transcript: { + relative_path: relativePath, + exists: proof.state.exists, + byte_length: proof.state.byteLength, + sha256: proof.sha256, + }, + }; + const sealedRaw = JSON.stringify(sealedRecord); + const sealedCandidatePath = `${this.lockPath}.sealed-candidate.${encodeURIComponent( + this.ownerId, + )}`; + const handoffRetiredPath = `${this.lockPath}.handoff.${encodeURIComponent( + this.ownerId, + )}`; + let claimAcquired = false; + let transitionCommitted = false; + try { + if (!sameTranscriptState(proof.state, this.expectedTranscriptState)) { + throw new SessionTranscriptChangedError(); + } + if (!(await installLockRecord(sealedCandidatePath, sealedRecord))) { + throw new SessionWriterUnavailableError({ + cause: new Error('Session writer sealed candidate already exists'), + }); + } + await assertPathMissing(handoffRetiredPath); + if (!(await installLockRecord(this.claimPath, ownedRecord))) { + throw new SessionWriterUnavailableError({ + cause: new Error('Session writer transition claim already exists'), + }); + } + claimAcquired = true; + await this.readOwnedLock(); + await validateOpenTranscriptProof(this.transcriptPath, proof); + await transitionExactPrimary( + this.lockPath, + this.lockRecordRaw, + sealedCandidatePath, + sealedRaw, + handoffRetiredPath, + ); + transitionCommitted = true; + await validateOpenTranscriptProof(this.transcriptPath, proof); + await removeExactRecord(this.claimPath, this.lockRecordRaw); + claimAcquired = false; + this.released = true; + await removeExactRecord(handoffRetiredPath, this.lockRecordRaw).catch( + (error) => { + debugLogger.debug( + `Session writer handoff retired cleanup failed path=${JSON.stringify(handoffRetiredPath)} ` + + `error=${describeDiagnosticError(error)}`, + ); + }, + ); + await removeExactRecord(sealedCandidatePath, sealedRaw).catch((error) => { + debugLogger.debug( + `Session writer handoff candidate cleanup failed path=${JSON.stringify(sealedCandidatePath)} ` + + `error=${describeDiagnosticError(error)}`, + ); + }); + debugLogger.info( + `Session writer handoff sealed sessionId=${JSON.stringify(this.sessionId)} ` + + `hostname=${JSON.stringify(ownedRecord.hostname)} pid=${ownedRecord.pid} ` + + `sealedAt=${sealedRecord.sealed_at}`, + ); + } catch (error) { + const cleanupFailures: unknown[] = []; + if (transitionCommitted) { + try { + await rollbackExactTransition( + this.lockPath, + sealedRaw, + handoffRetiredPath, + this.lockRecordRaw, + ); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + } + if (claimAcquired) { + try { + await removeExactRecord(this.claimPath, this.lockRecordRaw); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + } + try { + await removeExactRecord(sealedCandidatePath, sealedRaw); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + if ( + (await inspectExactRecord(this.lockPath, this.lockRecordRaw)) !== + 'exact' + ) { + this.released = true; + } + if (cleanupFailures.length > 0) { + throw new SessionWriterUnavailableError({ + cause: new AggregateError( + [error, ...cleanupFailures], + 'Session writer handoff cleanup failed', + ), + }); + } + if (error instanceof SessionWriterError) throw error; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } finally { + await closeTranscriptProof(proof); + } + } + + private async readOwnedLockForRelease(): Promise { for (let attempt = 0; attempt < RELEASE_PRECHECK_ATTEMPTS; attempt++) { try { - await this.readOwnedLock(); - return; + return await this.readOwnedLock(); } catch (error) { if ( !(error instanceof SessionWriterUnavailableError) || @@ -902,6 +1611,7 @@ export class SessionWriterLease { } await delay(RELEASE_PRECHECK_RETRY_DELAY_MS); } + throw new SessionWriterUnavailableError(); } private async inspectReleasePath( From 257f124aa39fe5b34289eaf582443566fcdb0422 Mon Sep 17 00:00:00 2001 From: doudouOUC Date: Tue, 28 Jul 2026 23:44:27 +0800 Subject: [PATCH 2/6] fix(core): retain writer claim after failed rollback Keep the fixed transition claim unless the exact predecessor primary is restored, and avoid rollback after claim ownership changes or unlink completes with a reported error. Co-authored-by: Qwen-Coder --- .../certified-session-writer-handoff.md | 9 + .../src/services/session-writer-lease.test.ts | 310 +++++++++++++++++- .../core/src/services/session-writer-lease.ts | 93 +++++- 3 files changed, 402 insertions(+), 10 deletions(-) diff --git a/docs/design/certified-session-writer-handoff.md b/docs/design/certified-session-writer-handoff.md index bfc455eb5ca..eb5c4048594 100644 --- a/docs/design/certified-session-writer-handoff.md +++ b/docs/design/certified-session-writer-handoff.md @@ -113,6 +113,15 @@ If a normal release races managed shutdown and commits first, the missing primary is already a safe handoff and the replacement performs ordinary acquisition. +Failure cleanup removes the fixed claim only after proving that the exact +pre-transition primary was restored. If rollback cannot restore or verify that +record, the claim remains even when another primary appears, because that path +may be a losing ordinary-acquisition candidate that will remove itself after +observing the claim. Rollback itself is attempted only while the fixed claim +still contains this transition's exact record; a missing or replaced claim +means the current primary must not be changed. Recovery then requires the same +authoritative external writer fence as any other residual claim. + ## Certified takeover Only a Config created under a trusted managed parent enables certified diff --git a/packages/core/src/services/session-writer-lease.test.ts b/packages/core/src/services/session-writer-lease.test.ts index 49b8eab3b24..cb75601eb88 100644 --- a/packages/core/src/services/session-writer-lease.test.ts +++ b/packages/core/src/services/session-writer-lease.test.ts @@ -49,9 +49,21 @@ const transitionFault = vi.hoisted(() => ({ afterRename: undefined as (() => Promise) | undefined, linkFrom: undefined as string | undefined, linkTo: undefined as string | undefined, + afterLink: undefined as (() => Promise | void) | undefined, throwAfterLink: false, })); +const restoreLinkFault = vi.hoisted(() => ({ + linkTo: undefined as string | undefined, + remainingFailures: 0, +})); + +const unlinkFault = vi.hoisted(() => ({ + path: undefined as string | undefined, + afterUnlink: undefined as (() => Promise | void) | undefined, + throwAfterUnlink: false, +})); + const writeFault = vi.hoisted(() => ({ contains: undefined as string | undefined, onEntered: undefined as (() => void) | undefined, @@ -91,7 +103,22 @@ vi.mock('node:fs/promises', async (importOriginal) => { existingPath: Parameters[0], newPath: Parameters[1], ) => { + if ( + newPath === restoreLinkFault.linkTo && + restoreLinkFault.remainingFailures > 0 + ) { + restoreLinkFault.remainingFailures--; + throw Object.assign(new Error('injected restore link failure'), { + code: 'EIO', + }); + } await actual.link(existingPath, newPath); + if ( + existingPath === transitionFault.linkFrom && + newPath === transitionFault.linkTo + ) { + await transitionFault.afterLink?.(); + } if ( transitionFault.throwAfterLink && existingPath === transitionFault.linkFrom && @@ -102,6 +129,17 @@ vi.mock('node:fs/promises', async (importOriginal) => { }); } }, + unlink: async (filePath: Parameters[0]) => { + await actual.unlink(filePath); + if (filePath === unlinkFault.path) { + await unlinkFault.afterUnlink?.(); + } + if (unlinkFault.throwAfterUnlink && filePath === unlinkFault.path) { + throw Object.assign(new Error('injected error after unlink'), { + code: 'EIO', + }); + } + }, open: async (...args: Parameters) => { const handle = await actual.open(...args); const writeFile = handle.writeFile.bind(handle); @@ -227,7 +265,13 @@ afterEach(async () => { transitionFault.afterRename = undefined; transitionFault.linkFrom = undefined; transitionFault.linkTo = undefined; + transitionFault.afterLink = undefined; transitionFault.throwAfterLink = false; + restoreLinkFault.linkTo = undefined; + restoreLinkFault.remainingFailures = 0; + unlinkFault.path = undefined; + unlinkFault.afterUnlink = undefined; + unlinkFault.throwAfterUnlink = false; writeFault.contains = undefined; writeFault.onEntered = undefined; writeFault.wait = undefined; @@ -1146,6 +1190,153 @@ describe('SessionWriterLease', () => { }); }); + it('reconciles a sealing claim unlink error after effect', async () => { + const fixture = await createFixture('sealed-claim-unlink-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + unlinkFault.path = `${lockPath}.claim`; + unlinkFault.throwAfterUnlink = true; + + await expect(first.sealForHandoff()).resolves.toBeUndefined(); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'sealed', + }); + }); + + it('does not roll back after the released claim is replaced', async () => { + const fixture = await createFixture('sealed-replaced-claim-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const successorClaim = '{"successorClaim":true}'; + unlinkFault.path = `${lockPath}.claim`; + unlinkFault.afterUnlink = () => + fs.writeFile(`${lockPath}.claim`, successorClaim, 'utf8'); + unlinkFault.throwAfterUnlink = true; + + await expect(first.sealForHandoff()).resolves.toBeUndefined(); + await expect(fs.readFile(`${lockPath}.claim`, 'utf8')).resolves.toBe( + successorClaim, + ); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'sealed', + }); + }); + + it('does not roll back sealing after claim ownership changes', async () => { + const fixture = await createFixture('sealed-changed-claim-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const successorClaim = '{"successorClaim":true}'; + transitionFault.linkFrom = `${lockPath}.sealed-candidate.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = async () => { + await fs.unlink(`${lockPath}.claim`); + await fs.writeFile(`${lockPath}.claim`, successorClaim, 'utf8'); + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + }; + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.readFile(`${lockPath}.claim`, 'utf8')).resolves.toBe( + successorClaim, + ); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'sealed', + }); + }); + + it('retains the claim when sealing rollback cannot restore the primary', async () => { + const fixture = await createFixture('sealed-rollback-failure-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const activeRaw = await fs.readFile(lockPath, 'utf8'); + transitionFault.linkFrom = `${lockPath}.sealed-candidate.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = () => { + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + restoreLinkFault.linkTo = lockPath; + restoreLinkFault.remainingFailures = 1; + }; + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.lstat(lockPath)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(fs.readFile(`${lockPath}.claim`, 'utf8')).resolves.toBe( + activeRaw, + ); + await expect( + fs.readFile( + `${lockPath}.handoff.${encodeURIComponent(first.ownerId)}`, + 'utf8', + ), + ).resolves.toBe(activeRaw); + await fs.appendFile(fixture.transcriptPath, '{"external":true}\n'); + await expect( + SessionWriterLease.acquire(fixture.options), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + }); + + it('removes the claim after sealing rollback restores the primary', async () => { + const fixture = await createFixture('sealed-rollback-success-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const activeRaw = await fs.readFile(lockPath, 'utf8'); + transitionFault.linkFrom = `${lockPath}.sealed-candidate.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = () => { + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + }; + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(activeRaw); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + it('never overwrites a primary installed during the sealing transition', async () => { const fixture = await createFixture('sealed-successor-session'); const first = await SessionWriterLease.acquire(fixture.options); @@ -1162,10 +1353,11 @@ describe('SessionWriterLease', () => { fs.writeFile(lockPath, successorRaw, 'utf8'); await expect(first.sealForHandoff()).rejects.toBeInstanceOf( - SessionWriterLostError, + SessionWriterUnavailableError, ); expect(first.isReleased).toBe(true); await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(successorRaw); + await expect(fs.lstat(`${lockPath}.claim`)).resolves.toBeDefined(); }); it('elects exactly one certified replacement for a sealed session', async () => { @@ -1218,6 +1410,119 @@ describe('SessionWriterLease', () => { await replacement.release(); }); + it('reconciles a takeover claim unlink error after effect', async () => { + const fixture = await createFixture('takeover-claim-unlink-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + unlinkFault.path = `${lockPath}.claim`; + unlinkFault.throwAfterUnlink = true; + + const replacement = await SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'active', + owner_id: replacement.ownerId, + }); + await replacement.release(); + }); + + it('does not roll back takeover after claim ownership changes', async () => { + const fixture = await createFixture('takeover-changed-claim-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const successorClaim = '{"successorClaim":true}'; + transitionFault.linkFrom = `${lockPath}.claim`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = async () => { + await fs.unlink(`${lockPath}.claim`); + await fs.writeFile(`${lockPath}.claim`, successorClaim, 'utf8'); + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + }; + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + await expect(fs.readFile(`${lockPath}.claim`, 'utf8')).resolves.toBe( + successorClaim, + ); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'active', + }); + }); + + it('retains the claim when takeover rollback cannot restore the primary', async () => { + const fixture = await createFixture('takeover-rollback-failure-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealedRaw = await fs.readFile(lockPath, 'utf8'); + transitionFault.linkFrom = `${lockPath}.claim`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = () => { + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + restoreLinkFault.linkTo = lockPath; + restoreLinkFault.remainingFailures = 1; + }; + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + await expect(fs.lstat(lockPath)).rejects.toMatchObject({ code: 'ENOENT' }); + const claimRaw = await fs.readFile(`${lockPath}.claim`, 'utf8'); + const claim = JSON.parse(claimRaw) as { owner_id: string; state: string }; + expect(claim.state).toBe('active'); + await expect( + fs.readFile( + `${lockPath}.sealed.${encodeURIComponent( + first.ownerId, + )}.${encodeURIComponent(claim.owner_id)}`, + 'utf8', + ), + ).resolves.toBe(sealedRaw); + await fs.appendFile(fixture.transcriptPath, '{"external":true}\n'); + await expect( + SessionWriterLease.acquire(fixture.options), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + }); + it('never overwrites a primary installed during the takeover transition', async () => { const fixture = await createFixture('takeover-successor-session'); const first = await SessionWriterLease.acquire(fixture.options); @@ -1237,8 +1542,9 @@ describe('SessionWriterLease', () => { reclaimPolicy: 'never', takeoverPolicy: 'certified', }), - ).rejects.toBeInstanceOf(SessionWriterLostError); + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(successorRaw); + await expect(fs.lstat(`${lockPath}.claim`)).resolves.toBeDefined(); }); it.each(['append', 'truncate', 'replace'] as const)( diff --git a/packages/core/src/services/session-writer-lease.ts b/packages/core/src/services/session-writer-lease.ts index 42b9082ddd9..35710dc2152 100644 --- a/packages/core/src/services/session-writer-lease.ts +++ b/packages/core/src/services/session-writer-lease.ts @@ -782,6 +782,49 @@ async function inspectExactRecord( } } +async function removeTransitionClaimAfterFailure( + lockPath: string, + expectedPrimaryRaw: string, + claimPath: string, + claimRaw: string, +): Promise { + const primaryState = await inspectExactRecord(lockPath, expectedPrimaryRaw); + if (primaryState !== 'exact') { + throw new SessionWriterUnavailableError({ + cause: new Error( + 'Session writer primary was not restored; transition claim retained', + ), + }); + } + await removeExactRecord(claimPath, claimRaw); +} + +async function releaseTransitionClaim( + claimPath: string, + claimRaw: string, +): Promise { + let raw: string; + try { + raw = await fs.readFile(claimPath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if (raw !== claimRaw) return; + try { + await fs.unlink(claimPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + const state = await inspectExactRecord(claimPath, claimRaw); + if (state === 'missing' || state === 'other') return; + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } +} + async function transitionExactPrimary( lockPath: string, expectedRaw: string, @@ -1147,7 +1190,7 @@ export class SessionWriterLease { options, proof.state, ); - await removeExactRecord(claimPath, claimRaw); + await releaseTransitionClaim(claimPath, claimRaw); claimAcquired = false; await removeExactRecord(retiredPath, observed.raw).catch((error) => { debugLogger.debug( @@ -1163,7 +1206,19 @@ export class SessionWriterLease { return lease; } catch (error) { const cleanupFailures: unknown[] = []; - if (transitionCommitted) { + const claimState = claimAcquired + ? await inspectExactRecord(claimPath, claimRaw) + : 'missing'; + if (claimState === 'unknown') { + cleanupFailures.push( + new SessionWriterUnavailableError({ + cause: new Error( + 'Session writer transition claim ownership is unreadable', + ), + }), + ); + } + if (transitionCommitted && claimState === 'exact') { try { await rollbackExactTransition( lockPath, @@ -1175,9 +1230,14 @@ export class SessionWriterLease { cleanupFailures.push(cleanupError); } } - if (claimAcquired) { + if (claimState === 'exact') { try { - await removeExactRecord(claimPath, claimRaw); + await removeTransitionClaimAfterFailure( + lockPath, + observed.raw, + claimPath, + claimRaw, + ); } catch (cleanupError) { cleanupFailures.push(cleanupError); } @@ -1526,7 +1586,7 @@ export class SessionWriterLease { ); transitionCommitted = true; await validateOpenTranscriptProof(this.transcriptPath, proof); - await removeExactRecord(this.claimPath, this.lockRecordRaw); + await releaseTransitionClaim(this.claimPath, this.lockRecordRaw); claimAcquired = false; this.released = true; await removeExactRecord(handoffRetiredPath, this.lockRecordRaw).catch( @@ -1550,7 +1610,19 @@ export class SessionWriterLease { ); } catch (error) { const cleanupFailures: unknown[] = []; - if (transitionCommitted) { + const claimState = claimAcquired + ? await inspectExactRecord(this.claimPath, this.lockRecordRaw) + : 'missing'; + if (claimState === 'unknown') { + cleanupFailures.push( + new SessionWriterUnavailableError({ + cause: new Error( + 'Session writer transition claim ownership is unreadable', + ), + }), + ); + } + if (transitionCommitted && claimState === 'exact') { try { await rollbackExactTransition( this.lockPath, @@ -1562,9 +1634,14 @@ export class SessionWriterLease { cleanupFailures.push(cleanupError); } } - if (claimAcquired) { + if (claimState === 'exact') { try { - await removeExactRecord(this.claimPath, this.lockRecordRaw); + await removeTransitionClaimAfterFailure( + this.lockPath, + this.lockRecordRaw, + this.claimPath, + this.lockRecordRaw, + ); } catch (cleanupError) { cleanupFailures.push(cleanupError); } From dec0c5687bd482a0c743e81d2e93f147a886b797 Mon Sep 17 00:00:00 2001 From: doudouOUC Date: Wed, 29 Jul 2026 01:05:02 +0800 Subject: [PATCH 3/6] fix(core): close session handoff claim races Wait for claim-aware primary candidates to back out of transition gaps, and release losing claims when no primary transition started. Co-authored-by: Qwen-Coder --- .../certified-session-writer-handoff.md | 14 + .../src/services/session-writer-lease.test.ts | 149 +++++++++++ .../core/src/services/session-writer-lease.ts | 251 ++++++++++++++---- 3 files changed, 366 insertions(+), 48 deletions(-) diff --git a/docs/design/certified-session-writer-handoff.md b/docs/design/certified-session-writer-handoff.md index eb5c4048594..6201c930ba8 100644 --- a/docs/design/certified-session-writer-handoff.md +++ b/docs/design/certified-session-writer-handoff.md @@ -84,6 +84,14 @@ the current active-lock fast path and local stale-owner recovery while making both paths respect a handoff transition. Mixed-version writers remain unsupported because an older writer does not know about the fixed claim. +An acquirer can pass its first claim check immediately before a transition +creates the claim, then install its active candidate during the transition's +primary-path gap. The transition recognizes that same-session schema-v2 active +record as a claim-aware candidate, preserves the retired predecessor, and +waits for the candidate's mandatory second check to remove it before retrying +the hard link. Unknown, malformed, or cross-session successors are never +removed or overwritten. + ## Sealing Managed shutdown synchronously stops session and recorder admission, then @@ -122,6 +130,12 @@ still contains this transition's exact record; a missing or replaced claim means the current primary must not be changed. Recovery then requires the same authoritative external writer fence as any other residual claim. +Failures before the primary transition starts are different: the claimant has +not created a primary-path gap or moved the predecessor, so it releases only +its own exact claim even if another certified contender already replaced the +observed sealed primary. This prevents a delayed losing contender from +stranding a claim after the winner becomes active. + ## Certified takeover Only a Config created under a trusted managed parent enables certified diff --git a/packages/core/src/services/session-writer-lease.test.ts b/packages/core/src/services/session-writer-lease.test.ts index cb75601eb88..426b29679eb 100644 --- a/packages/core/src/services/session-writer-lease.test.ts +++ b/packages/core/src/services/session-writer-lease.test.ts @@ -70,6 +70,18 @@ const writeFault = vi.hoisted(() => ({ wait: undefined as Promise | undefined, })); +const claimInstallFault = vi.hoisted(() => ({ + path: undefined as string | undefined, + afterInstall: undefined as (() => Promise | void) | undefined, +})); + +const readFileFault = vi.hoisted(() => ({ + path: undefined as string | undefined, + triggerCall: 0, + calls: 0, + afterRead: undefined as (() => Promise | void) | undefined, +})); + vi.mock('node:fs/promises', async (importOriginal) => { const actual = await importOriginal(); return { @@ -113,6 +125,9 @@ vi.mock('node:fs/promises', async (importOriginal) => { }); } await actual.link(existingPath, newPath); + if (newPath === claimInstallFault.path) { + await claimInstallFault.afterInstall?.(); + } if ( existingPath === transitionFault.linkFrom && newPath === transitionFault.linkTo @@ -140,6 +155,16 @@ vi.mock('node:fs/promises', async (importOriginal) => { }); } }, + readFile: async (...args: Parameters) => { + const result = await actual.readFile(...args); + if (args[0] === readFileFault.path) { + readFileFault.calls++; + if (readFileFault.calls === readFileFault.triggerCall) { + await readFileFault.afterRead?.(); + } + } + return result; + }, open: async (...args: Parameters) => { const handle = await actual.open(...args); const writeFile = handle.writeFile.bind(handle); @@ -275,6 +300,12 @@ afterEach(async () => { writeFault.contains = undefined; writeFault.onEntered = undefined; writeFault.wait = undefined; + claimInstallFault.path = undefined; + claimInstallFault.afterInstall = undefined; + readFileFault.path = undefined; + readFileFault.triggerCall = 0; + readFileFault.calls = 0; + readFileFault.afterRead = undefined; setDebugLogSession(null); resetDebugLoggingState(); Storage.setRuntimeBaseDir(null); @@ -1337,6 +1368,87 @@ describe('SessionWriterLease', () => { }); }); + it('waits for a claim-aware primary candidate before completing sealing', async () => { + const fixture = await createFixture('sealed-primary-candidate-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const active = JSON.parse(await fs.readFile(lockPath, 'utf8')) as Record< + string, + unknown + >; + const candidateRaw = JSON.stringify({ + ...active, + owner_id: 'claim-aware-candidate', + }); + transitionFault.renameFrom = lockPath; + transitionFault.renameTo = `${lockPath}.handoff.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.afterRename = async () => { + await fs.writeFile(lockPath, candidateRaw, 'utf8'); + readFileFault.path = lockPath; + readFileFault.triggerCall = 2; + readFileFault.afterRead = () => fs.unlink(lockPath); + }; + + await expect(first.sealForHandoff()).resolves.toBeUndefined(); + expect(readFileFault.calls).toBeGreaterThanOrEqual(2); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'sealed', + owner_id: first.ownerId, + }); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('waits for a claim-aware primary candidate while rolling back sealing', async () => { + const fixture = await createFixture('sealed-rollback-candidate-session'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const activeRaw = await fs.readFile(lockPath, 'utf8'); + const active = JSON.parse(activeRaw) as Record; + const candidateRaw = JSON.stringify({ + ...active, + owner_id: 'rollback-candidate', + }); + transitionFault.linkFrom = `${lockPath}.sealed-candidate.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = () => { + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + unlinkFault.path = lockPath; + unlinkFault.afterUnlink = async () => { + unlinkFault.path = undefined; + await fs.writeFile(lockPath, candidateRaw, 'utf8'); + readFileFault.path = lockPath; + readFileFault.triggerCall = 2; + readFileFault.afterRead = () => fs.unlink(lockPath); + }; + }; + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(activeRaw); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + await expect( + fs.lstat(`${lockPath}.handoff.${encodeURIComponent(first.ownerId)}`), + ).rejects.toMatchObject({ code: 'ENOENT' }); + }); + it('never overwrites a primary installed during the sealing transition', async () => { const fixture = await createFixture('sealed-successor-session'); const first = await SessionWriterLease.acquire(fixture.options); @@ -1385,6 +1497,43 @@ describe('SessionWriterLease', () => { }); }); + it('releases a losing takeover claim before its transition starts', async () => { + const fixture = await createFixture( + 'takeover-pre-transition-loser-session', + ); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + let winnerRaw = ''; + claimInstallFault.path = `${lockPath}.claim`; + claimInstallFault.afterInstall = async () => { + const contender = JSON.parse( + await fs.readFile(`${lockPath}.claim`, 'utf8'), + ) as Record; + winnerRaw = JSON.stringify({ + ...contender, + owner_id: 'certified-winner', + }); + await fs.unlink(lockPath); + await fs.writeFile(lockPath, winnerRaw, 'utf8'); + }; + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterConflictError); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(winnerRaw); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + it('reconciles a takeover error reported after the active primary is installed', async () => { const fixture = await createFixture('takeover-after-effect-session'); const first = await SessionWriterLease.acquire(fixture.options); diff --git a/packages/core/src/services/session-writer-lease.ts b/packages/core/src/services/session-writer-lease.ts index 35710dc2152..19fc7ffcce1 100644 --- a/packages/core/src/services/session-writer-lease.ts +++ b/packages/core/src/services/session-writer-lease.ts @@ -825,51 +825,173 @@ async function releaseTransitionClaim( } } +type ClaimedPrimaryState = + | 'source' + | 'candidate' + | 'missing' + | 'other' + | 'unknown'; + +async function inspectClaimedPrimary( + lockPath: string, + sourceRaw: string, + sessionId: string, +): Promise { + try { + const stat = await fs.lstat(lockPath); + if (!stat.isFile() || stat.isSymbolicLink()) return 'other'; + const raw = await fs.readFile(lockPath, 'utf8'); + if (raw === sourceRaw) return 'source'; + const record = parseLockRecord(raw); + return record?.schema_version === LOCK_SCHEMA_VERSION && + record.state === 'active' && + record.session_id === sessionId + ? 'candidate' + : 'other'; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT' + ? 'missing' + : 'unknown'; + } +} + +async function assertExactTransitionClaim( + claimPath: string, + claimRaw: string, +): Promise { + const claimState = await inspectExactRecord(claimPath, claimRaw); + if (claimState !== 'exact') { + throw new SessionWriterUnavailableError({ + cause: new Error('Session writer transition claim ownership was lost'), + }); + } +} + +async function linkClaimedPrimary( + lockPath: string, + sourcePath: string, + sourceRaw: string, + sessionId: string, + claimPath: string, + claimRaw: string, +): Promise { + for (;;) { + await assertExactTransitionClaim(claimPath, claimRaw); + try { + await fs.link(sourcePath, lockPath); + return; + } catch (error) { + const state = await inspectClaimedPrimary(lockPath, sourceRaw, sessionId); + if (state === 'source') return; + if (state === 'candidate') { + // A schema-v2 acquirer can pass its first claim check immediately + // before this transition creates the claim. It must remove its primary + // candidate after the second check, so keep the predecessor and wait. + await delay(MALFORMED_RETRY_DELAY_MS); + continue; + } + if (state === 'other') throw new SessionWriterLostError(); + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + } +} + +async function removeClaimedPrimary( + lockPath: string, + sourceRaw: string, + sessionId: string, + claimPath: string, + claimRaw: string, +): Promise { + for (;;) { + await assertExactTransitionClaim(claimPath, claimRaw); + const state = await inspectClaimedPrimary(lockPath, sourceRaw, sessionId); + if (state === 'missing') return; + if (state === 'candidate') { + await delay(MALFORMED_RETRY_DELAY_MS); + continue; + } + if (state === 'other') throw new SessionWriterLostError(); + if (state === 'unknown') throw new SessionWriterUnavailableError(); + try { + await fs.unlink(lockPath); + return; + } catch (error) { + const afterState = await inspectClaimedPrimary( + lockPath, + sourceRaw, + sessionId, + ); + if (afterState === 'missing') return; + if (afterState === 'candidate') { + await delay(MALFORMED_RETRY_DELAY_MS); + continue; + } + if (afterState === 'other') throw new SessionWriterLostError(); + throw new SessionWriterUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + } +} + async function transitionExactPrimary( lockPath: string, expectedRaw: string, replacementPath: string, replacementRaw: string, retiredPath: string, + sessionId: string, + claimPath: string, + claimRaw: string, ): Promise { try { await fs.rename(lockPath, retiredPath); - await fs.link(replacementPath, lockPath); - return; } catch (error) { - const [primaryState, primaryExpectedState, retiredState] = - await Promise.all([ - inspectExactRecord(lockPath, replacementRaw), - inspectExactRecord(lockPath, expectedRaw), - inspectExactRecord(retiredPath, expectedRaw), - ]); - if (primaryState === 'exact' && retiredState === 'exact') return; + const [primaryExpectedState, retiredState] = await Promise.all([ + inspectExactRecord(lockPath, expectedRaw), + inspectExactRecord(retiredPath, expectedRaw), + ]); if (primaryExpectedState === 'exact' && retiredState === 'missing') { throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, }); } - if (primaryState === 'missing' && retiredState === 'exact') { - try { - await fs.link(retiredPath, lockPath); - await fs.unlink(retiredPath).catch(() => {}); - } catch (restoreError) { - throw new SessionWriterUnavailableError({ - cause: new AggregateError( - [error, restoreError], - 'Session writer primary transition could not be restored', - ), - }); - } + if (retiredState !== 'exact') { throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, }); } - if (retiredState === 'exact') { - await fs.unlink(retiredPath).catch(() => {}); - } - if (primaryState === 'other' && primaryExpectedState === 'other') { - throw new SessionWriterLostError(); + } + try { + await linkClaimedPrimary( + lockPath, + replacementPath, + replacementRaw, + sessionId, + claimPath, + claimRaw, + ); + } catch (error) { + try { + await linkClaimedPrimary( + lockPath, + retiredPath, + expectedRaw, + sessionId, + claimPath, + claimRaw, + ); + await removeExactRecord(retiredPath, expectedRaw); + } catch (restoreError) { + throw new SessionWriterUnavailableError({ + cause: new AggregateError( + [error, restoreError], + 'Session writer primary transition could not be restored', + ), + }); } throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, @@ -882,15 +1004,17 @@ async function rollbackExactTransition( replacementRaw: string, retiredPath: string, expectedRaw: string, + sessionId: string, + claimPath: string, + claimRaw: string, ): Promise { - const primaryState = await inspectExactRecord(lockPath, replacementRaw); - if (primaryState === 'exact') { - await removeExactRecord(lockPath, replacementRaw); - } else if (primaryState !== 'missing') { - throw primaryState === 'other' - ? new SessionWriterLostError() - : new SessionWriterUnavailableError(); - } + await removeClaimedPrimary( + lockPath, + replacementRaw, + sessionId, + claimPath, + claimRaw, + ); const retiredState = await inspectExactRecord(retiredPath, expectedRaw); if (retiredState !== 'exact') { throw retiredState === 'other' @@ -898,8 +1022,15 @@ async function rollbackExactTransition( : new SessionWriterUnavailableError(); } try { - await fs.link(retiredPath, lockPath); - await fs.unlink(retiredPath).catch(() => {}); + await linkClaimedPrimary( + lockPath, + retiredPath, + expectedRaw, + sessionId, + claimPath, + claimRaw, + ); + await removeExactRecord(retiredPath, expectedRaw); } catch (error) { throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, @@ -1154,6 +1285,7 @@ export class SessionWriterLease { observed.record.owner_id, )}.${encodeURIComponent(lockRecord.owner_id)}`; let claimAcquired = false; + let transitionStarted = false; let transitionCommitted = false; try { assertSealedProofMatches(observed.record, relativePath, proof); @@ -1175,12 +1307,16 @@ export class SessionWriterLease { } await validateOpenTranscriptProof(options.transcriptPath, proof); assertSealedProofMatches(current.record, relativePath, proof); + transitionStarted = true; await transitionExactPrimary( lockPath, observed.raw, claimPath, claimRaw, retiredPath, + options.sessionId, + claimPath, + claimRaw, ); transitionCommitted = true; await validateOpenTranscriptProof(options.transcriptPath, proof); @@ -1225,6 +1361,9 @@ export class SessionWriterLease { claimRaw, retiredPath, observed.raw, + options.sessionId, + claimPath, + claimRaw, ); } catch (cleanupError) { cleanupFailures.push(cleanupError); @@ -1232,12 +1371,16 @@ export class SessionWriterLease { } if (claimState === 'exact') { try { - await removeTransitionClaimAfterFailure( - lockPath, - observed.raw, - claimPath, - claimRaw, - ); + if (transitionStarted) { + await removeTransitionClaimAfterFailure( + lockPath, + observed.raw, + claimPath, + claimRaw, + ); + } else { + await releaseTransitionClaim(claimPath, claimRaw); + } } catch (cleanupError) { cleanupFailures.push(cleanupError); } @@ -1558,6 +1701,7 @@ export class SessionWriterLease { this.ownerId, )}`; let claimAcquired = false; + let transitionStarted = false; let transitionCommitted = false; try { if (!sameTranscriptState(proof.state, this.expectedTranscriptState)) { @@ -1577,12 +1721,16 @@ export class SessionWriterLease { claimAcquired = true; await this.readOwnedLock(); await validateOpenTranscriptProof(this.transcriptPath, proof); + transitionStarted = true; await transitionExactPrimary( this.lockPath, this.lockRecordRaw, sealedCandidatePath, sealedRaw, handoffRetiredPath, + this.sessionId, + this.claimPath, + this.lockRecordRaw, ); transitionCommitted = true; await validateOpenTranscriptProof(this.transcriptPath, proof); @@ -1629,6 +1777,9 @@ export class SessionWriterLease { sealedRaw, handoffRetiredPath, this.lockRecordRaw, + this.sessionId, + this.claimPath, + this.lockRecordRaw, ); } catch (cleanupError) { cleanupFailures.push(cleanupError); @@ -1636,12 +1787,16 @@ export class SessionWriterLease { } if (claimState === 'exact') { try { - await removeTransitionClaimAfterFailure( - this.lockPath, - this.lockRecordRaw, - this.claimPath, - this.lockRecordRaw, - ); + if (transitionStarted) { + await removeTransitionClaimAfterFailure( + this.lockPath, + this.lockRecordRaw, + this.claimPath, + this.lockRecordRaw, + ); + } else { + await releaseTransitionClaim(this.claimPath, this.lockRecordRaw); + } } catch (cleanupError) { cleanupFailures.push(cleanupError); } From 34e884e194c382a96eab0bfe4dc50eaf20a758d6 Mon Sep 17 00:00:00 2001 From: doudouOUC Date: Wed, 29 Jul 2026 01:23:12 +0800 Subject: [PATCH 4/6] fix(core): bound session handoff candidate waits Fail closed when a claim-aware primary candidate is abandoned during a handoff transition, preserving the transition fence for authoritative recovery. Co-authored-by: Qwen-Coder --- .../certified-session-writer-handoff.md | 6 +- .../src/services/session-writer-lease.test.ts | 74 +++++++++++++++++++ .../core/src/services/session-writer-lease.ts | 20 ++++- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/docs/design/certified-session-writer-handoff.md b/docs/design/certified-session-writer-handoff.md index 6201c930ba8..b0263723bf3 100644 --- a/docs/design/certified-session-writer-handoff.md +++ b/docs/design/certified-session-writer-handoff.md @@ -89,8 +89,10 @@ creates the claim, then install its active candidate during the transition's primary-path gap. The transition recognizes that same-session schema-v2 active record as a claim-aware candidate, preserves the retired predecessor, and waits for the candidate's mandatory second check to remove it before retrying -the hard link. Unknown, malformed, or cross-session successors are never -removed or overwritten. +the hard link. This wait is bounded; if the candidate stalls or exits before +its second check, the transition fails unavailable while retaining its claim +and retired predecessor. Unknown, malformed, or cross-session successors are +never removed or overwritten. ## Sealing diff --git a/packages/core/src/services/session-writer-lease.test.ts b/packages/core/src/services/session-writer-lease.test.ts index 426b29679eb..aba34ea97ca 100644 --- a/packages/core/src/services/session-writer-lease.test.ts +++ b/packages/core/src/services/session-writer-lease.test.ts @@ -1405,6 +1405,36 @@ describe('SessionWriterLease', () => { }); }); + it('fails closed when a primary candidate is abandoned during sealing', async () => { + const fixture = await createFixture('sealed-abandoned-candidate-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const activeRaw = await fs.readFile(lockPath, 'utf8'); + const candidateRaw = JSON.stringify({ + ...(JSON.parse(activeRaw) as Record), + owner_id: 'abandoned-candidate', + }); + const retiredPath = `${lockPath}.handoff.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.renameFrom = lockPath; + transitionFault.renameTo = retiredPath; + transitionFault.afterRename = () => + fs.writeFile(lockPath, candidateRaw, 'utf8'); + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(candidateRaw); + await expect(fs.readFile(`${lockPath}.claim`, 'utf8')).resolves.toBe( + activeRaw, + ); + await expect(fs.readFile(retiredPath, 'utf8')).resolves.toBe(activeRaw); + }); + it('waits for a claim-aware primary candidate while rolling back sealing', async () => { const fixture = await createFixture('sealed-rollback-candidate-session'); await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); @@ -1449,6 +1479,50 @@ describe('SessionWriterLease', () => { ).rejects.toMatchObject({ code: 'ENOENT' }); }); + it('fails closed when a primary candidate is abandoned during rollback', async () => { + const fixture = await createFixture( + 'sealed-rollback-abandoned-candidate-session', + ); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.writeFile(fixture.transcriptPath, '{"record":"tail"}\n'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const activeRaw = await fs.readFile(lockPath, 'utf8'); + const candidateRaw = JSON.stringify({ + ...(JSON.parse(activeRaw) as Record), + owner_id: 'abandoned-rollback-candidate', + }); + const retiredPath = `${lockPath}.handoff.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkFrom = `${lockPath}.sealed-candidate.${encodeURIComponent( + first.ownerId, + )}`; + transitionFault.linkTo = lockPath; + transitionFault.afterLink = () => { + lstatFault.path = fixture.transcriptPath; + lstatFault.remainingFailures = 1; + unlinkFault.path = lockPath; + unlinkFault.throwAfterUnlink = true; + unlinkFault.afterUnlink = async () => { + unlinkFault.path = undefined; + await fs.writeFile(lockPath, candidateRaw, 'utf8'); + }; + }; + + await expect(first.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(candidateRaw); + await expect(fs.readFile(`${lockPath}.claim`, 'utf8')).resolves.toBe( + activeRaw, + ); + await expect(fs.readFile(retiredPath, 'utf8')).resolves.toBe(activeRaw); + }); + it('never overwrites a primary installed during the sealing transition', async () => { const fixture = await createFixture('sealed-successor-session'); const first = await SessionWriterLease.acquire(fixture.options); diff --git a/packages/core/src/services/session-writer-lease.ts b/packages/core/src/services/session-writer-lease.ts index 19fc7ffcce1..fcc86d142d1 100644 --- a/packages/core/src/services/session-writer-lease.ts +++ b/packages/core/src/services/session-writer-lease.ts @@ -16,6 +16,7 @@ const LEGACY_LOCK_SCHEMA_VERSION = 1; const LOCK_SCHEMA_VERSION = 2; const MALFORMED_RETRY_COUNT = 3; const MALFORMED_RETRY_DELAY_MS = 50; +const CLAIMED_PRIMARY_WAIT_ATTEMPTS = 20; const RELEASE_PRECHECK_ATTEMPTS = 3; const RELEASE_PRECHECK_RETRY_DELAY_MS = 50; const ACQUIRE_ATTEMPTS = 8; @@ -867,6 +868,17 @@ async function assertExactTransitionClaim( } } +async function waitForClaimedPrimaryCandidate(attempt: number): Promise { + if (attempt >= CLAIMED_PRIMARY_WAIT_ATTEMPTS) { + throw new SessionWriterUnavailableError({ + cause: new Error( + 'Session writer primary candidate did not release the claimed path', + ), + }); + } + await delay(MALFORMED_RETRY_DELAY_MS); +} + async function linkClaimedPrimary( lockPath: string, sourcePath: string, @@ -875,6 +887,7 @@ async function linkClaimedPrimary( claimPath: string, claimRaw: string, ): Promise { + let candidateWaitAttempts = 0; for (;;) { await assertExactTransitionClaim(claimPath, claimRaw); try { @@ -887,7 +900,7 @@ async function linkClaimedPrimary( // A schema-v2 acquirer can pass its first claim check immediately // before this transition creates the claim. It must remove its primary // candidate after the second check, so keep the predecessor and wait. - await delay(MALFORMED_RETRY_DELAY_MS); + await waitForClaimedPrimaryCandidate(++candidateWaitAttempts); continue; } if (state === 'other') throw new SessionWriterLostError(); @@ -905,12 +918,13 @@ async function removeClaimedPrimary( claimPath: string, claimRaw: string, ): Promise { + let candidateWaitAttempts = 0; for (;;) { await assertExactTransitionClaim(claimPath, claimRaw); const state = await inspectClaimedPrimary(lockPath, sourceRaw, sessionId); if (state === 'missing') return; if (state === 'candidate') { - await delay(MALFORMED_RETRY_DELAY_MS); + await waitForClaimedPrimaryCandidate(++candidateWaitAttempts); continue; } if (state === 'other') throw new SessionWriterLostError(); @@ -926,7 +940,7 @@ async function removeClaimedPrimary( ); if (afterState === 'missing') return; if (afterState === 'candidate') { - await delay(MALFORMED_RETRY_DELAY_MS); + await waitForClaimedPrimaryCandidate(++candidateWaitAttempts); continue; } if (afterState === 'other') throw new SessionWriterLostError(); From 4aa1abe93603124d052e0d5ad5696468784dac44 Mon Sep 17 00:00:00 2001 From: doudouOUC Date: Wed, 29 Jul 2026 01:45:27 +0800 Subject: [PATCH 5/6] fix(core): reconcile session handoff proof edges Reject dangling transcript paths and reconcile ambiguous lock-record link outcomes by exact bytes before continuing a certified handoff. Co-authored-by: Qwen-Coder --- .../certified-session-writer-handoff.md | 10 +- .../src/services/session-writer-lease.test.ts | 104 ++++++++++++++++++ .../core/src/services/session-writer-lease.ts | 26 ++++- 3 files changed, 135 insertions(+), 5 deletions(-) diff --git a/docs/design/certified-session-writer-handoff.md b/docs/design/certified-session-writer-handoff.md index b0263723bf3..fdd769ebcf6 100644 --- a/docs/design/certified-session-writer-handoff.md +++ b/docs/design/certified-session-writer-handoff.md @@ -17,6 +17,9 @@ managed child can replace its active lock with a sealed record. A trusted managed replacement can take ownership only after validating that record against the exact transcript requested by the new Config. +Transcript paths must be absent or resolve to the same regular file opened for +the proof. A dangling symlink is not treated as an absent transcript. + The protocol remains gated by `experimental.sessionWriterLease`, which is disabled by default and snapshotted at ACP process startup. Standalone ACP, interactive, and headless recorders do not gain certified takeover. Normal @@ -72,9 +75,10 @@ The fixed claim path is: It serializes the two transitions that temporarily remove the primary path: active-to-sealed and sealed-to-active. The claim is created with the existing -write-sync-and-hard-link primitive. A claim is never reclaimed by PID, -hostname, or age. Any pre-existing claim returns -`session_writer_unavailable`; manual cleanup is allowed only after an +write-sync-and-hard-link primitive. Ambiguous link errors are reconciled +against the exact claim bytes before the transition continues or cleans up. A +claim is never reclaimed by PID, hostname, or age. Any pre-existing claim +returns `session_writer_unavailable`; manual cleanup is allowed only after an authoritative external writer fence. Ordinary acquisition checks the claim before every attempt to install a diff --git a/packages/core/src/services/session-writer-lease.test.ts b/packages/core/src/services/session-writer-lease.test.ts index aba34ea97ca..ee3cc57385d 100644 --- a/packages/core/src/services/session-writer-lease.test.ts +++ b/packages/core/src/services/session-writer-lease.test.ts @@ -914,6 +914,57 @@ describe('SessionWriterLease', () => { ).rejects.toMatchObject({ code: 'ENOENT' }); }); + it('rejects a dangling transcript symlink introduced before sealing', async () => { + const fixture = await createFixture('dangling-seal-session'); + const lease = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const activeRaw = await fs.readFile(lockPath, 'utf8'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.symlink( + `${fixture.transcriptPath}.missing`, + fixture.transcriptPath, + ); + + await expect(lease.sealForHandoff()).rejects.toBeInstanceOf( + SessionWriterUnavailableError, + ); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(activeRaw); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('rejects a dangling transcript symlink before certified takeover', async () => { + const fixture = await createFixture('dangling-takeover-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + const sealedRaw = await fs.readFile(lockPath, 'utf8'); + await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); + await fs.symlink( + `${fixture.transcriptPath}.missing`, + fixture.transcriptPath, + ); + + await expect( + SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }), + ).rejects.toBeInstanceOf(SessionWriterUnavailableError); + await expect(fs.readFile(lockPath, 'utf8')).resolves.toBe(sealedRaw); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + it('detects an equal-length atomic transcript replacement', async () => { const fixture = await createFixture(); await fs.mkdir(path.dirname(fixture.transcriptPath), { recursive: true }); @@ -1221,6 +1272,29 @@ describe('SessionWriterLease', () => { }); }); + it('reconciles a sealing claim link error after effect', async () => { + const fixture = await createFixture('sealed-claim-link-session'); + const first = await SessionWriterLease.acquire(fixture.options); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + claimInstallFault.path = `${lockPath}.claim`; + claimInstallFault.afterInstall = () => { + throw Object.assign(new Error('injected error after claim link'), { + code: 'EIO', + }); + }; + + await expect(first.sealForHandoff()).resolves.toBeUndefined(); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'sealed', + }); + }); + it('reconciles a sealing claim unlink error after effect', async () => { const fixture = await createFixture('sealed-claim-unlink-session'); const first = await SessionWriterLease.acquire(fixture.options); @@ -1633,6 +1707,36 @@ describe('SessionWriterLease', () => { await replacement.release(); }); + it('reconciles a takeover claim link error after effect', async () => { + const fixture = await createFixture('takeover-claim-link-session'); + const first = await SessionWriterLease.acquire(fixture.options); + await first.sealForHandoff(); + const lockPath = getSessionWriterLockPath( + fixture.runtimeBaseDir, + fixture.options.sessionId, + ); + claimInstallFault.path = `${lockPath}.claim`; + claimInstallFault.afterInstall = () => { + throw Object.assign(new Error('injected error after claim link'), { + code: 'EIO', + }); + }; + + const replacement = await SessionWriterLease.acquire({ + ...fixture.options, + reclaimPolicy: 'never', + takeoverPolicy: 'certified', + }); + await expect(fs.lstat(`${lockPath}.claim`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect(JSON.parse(await fs.readFile(lockPath, 'utf8'))).toMatchObject({ + state: 'active', + owner_id: replacement.ownerId, + }); + await replacement.release(); + }); + it('reconciles a takeover claim unlink error after effect', async () => { const fixture = await createFixture('takeover-claim-unlink-session'); const first = await SessionWriterLease.acquire(fixture.options); diff --git a/packages/core/src/services/session-writer-lease.ts b/packages/core/src/services/session-writer-lease.ts index fcc86d142d1..174622004f4 100644 --- a/packages/core/src/services/session-writer-lease.ts +++ b/packages/core/src/services/session-writer-lease.ts @@ -397,6 +397,18 @@ function sameTranscriptState( ); } +async function assertTranscriptPathMissing(filePath: string): Promise { + try { + await fs.lstat(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + throw new SessionWriterUnavailableError({ + cause: new Error('Session transcript path is not a regular file'), + }); +} + async function getTranscriptState(filePath: string): Promise { let handle: fs.FileHandle | undefined; try { @@ -404,6 +416,7 @@ async function getTranscriptState(filePath: string): Promise { handle = await fs.open(filePath, 'r'); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + await assertTranscriptPathMissing(filePath); return { exists: false, byteLength: 0 }; } throw error; @@ -478,6 +491,7 @@ async function openTranscriptProof( handle = await fs.open(filePath, 'r'); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + await assertTranscriptPathMissing(filePath); return { state: { exists: false, byteLength: 0 }, sha256: createHash('sha256').digest('hex'), @@ -668,8 +682,9 @@ async function installLockRecord( const temporaryPath = `${lockPath}.${record.owner_id}.tmp`; let handle: fs.FileHandle | undefined; try { + const recordRaw = JSON.stringify(record); handle = await fs.open(temporaryPath, 'wx', 0o600); - await handle.writeFile(JSON.stringify(record), 'utf8'); + await handle.writeFile(recordRaw, 'utf8'); await handle.sync(); await handle.close(); handle = undefined; @@ -677,7 +692,14 @@ async function installLockRecord( await fs.link(temporaryPath, lockPath); return true; } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'EEXIST') return false; + const state = await inspectExactRecord(lockPath, recordRaw); + if (state === 'exact') return true; + if ( + state === 'other' || + (error as NodeJS.ErrnoException).code === 'EEXIST' + ) { + return false; + } throw error; } } catch (error) { From 5fa5c355ca9e21d1f6800c3628f1d72c53cdd134 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Wed, 29 Jul 2026 19:24:23 +0000 Subject: [PATCH 6/6] fix(core): preserve inspect error cause and document handoff fail-closed paths (#7976) --- .../certified-session-writer-handoff.md | 9 ++ .../core/src/services/chatRecordingService.ts | 6 ++ .../core/src/services/session-writer-lease.ts | 101 ++++++++++++------ 3 files changed, 81 insertions(+), 35 deletions(-) diff --git a/docs/design/certified-session-writer-handoff.md b/docs/design/certified-session-writer-handoff.md index fdd769ebcf6..45ee8db2010 100644 --- a/docs/design/certified-session-writer-handoff.md +++ b/docs/design/certified-session-writer-handoff.md @@ -186,6 +186,15 @@ accepts schema v1 active records, but an older reader does not understand schema v2. Rollback therefore requires draining all new writers and confirming that no active, sealed, or claim record from this protocol remains. +A writer that does not acquire a lease — for example a plain `qwen --resume` +session, because standalone, interactive, and headless recorders run outside +this protocol — can still append to a transcript a managed writer has sealed. +That append invalidates the sealed proof, so a later certified takeover of the +same session fails closed with `session_transcript_changed` and the daemon +stays fenced until an authoritative external writer fence clears the residual +record. Rollout must therefore keep non-lease writers away from any transcript +that participates in certified handoff. + Hashing is intentionally performed at seal and takeover instead of adding an incremental digest to every append. This keeps the first implementation small and makes the proof independent of process memory. A very large transcript may diff --git a/packages/core/src/services/chatRecordingService.ts b/packages/core/src/services/chatRecordingService.ts index 9c87dc091e8..d973554c3a2 100644 --- a/packages/core/src/services/chatRecordingService.ts +++ b/packages/core/src/services/chatRecordingService.ts @@ -1183,6 +1183,12 @@ export class ChatRecordingService { flushFailure = error; } if (this.handoffRequested && flushFailure !== undefined) { + // Fail closed: a handoff whose final flush did not durably land must + // neither seal (the proof would be incomplete) nor release (a successor + // could take over records that were never persisted). Unlike the + // release-then-throw normal-close path below, it retains the active lock + // so write ownership stays unambiguous until an external writer fence + // recovers it. this.state = 'integrity_failed'; throw flushFailure; } diff --git a/packages/core/src/services/session-writer-lease.ts b/packages/core/src/services/session-writer-lease.ts index 174622004f4..2a43dad890b 100644 --- a/packages/core/src/services/session-writer-lease.ts +++ b/packages/core/src/services/session-writer-lease.ts @@ -692,7 +692,7 @@ async function installLockRecord( await fs.link(temporaryPath, lockPath); return true; } catch (error) { - const state = await inspectExactRecord(lockPath, recordRaw); + const { state } = await inspectExactRecord(lockPath, recordRaw); if (state === 'exact') return true; if ( state === 'other' || @@ -789,19 +789,30 @@ async function removeExactRecord( } } +type ExactRecordState = 'exact' | 'missing' | 'other' | 'unknown'; + +interface RecordInspection { + state: State; + cause?: Error; +} + async function inspectExactRecord( candidatePath: string, expectedRaw: string, -): Promise<'exact' | 'missing' | 'other' | 'unknown'> { +): Promise> { try { const stat = await fs.lstat(candidatePath); - if (!stat.isFile() || stat.isSymbolicLink()) return 'other'; + if (!stat.isFile() || stat.isSymbolicLink()) return { state: 'other' }; const raw = await fs.readFile(candidatePath, 'utf8'); - return raw === expectedRaw ? 'exact' : 'other'; + return { state: raw === expectedRaw ? 'exact' : 'other' }; } catch (error) { - return (error as NodeJS.ErrnoException).code === 'ENOENT' - ? 'missing' - : 'unknown'; + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return { state: 'missing' }; + } + return { + state: 'unknown', + cause: error instanceof Error ? error : undefined, + }; } } @@ -811,7 +822,10 @@ async function removeTransitionClaimAfterFailure( claimPath: string, claimRaw: string, ): Promise { - const primaryState = await inspectExactRecord(lockPath, expectedPrimaryRaw); + const { state: primaryState } = await inspectExactRecord( + lockPath, + expectedPrimaryRaw, + ); if (primaryState !== 'exact') { throw new SessionWriterUnavailableError({ cause: new Error( @@ -840,7 +854,7 @@ async function releaseTransitionClaim( await fs.unlink(claimPath); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - const state = await inspectExactRecord(claimPath, claimRaw); + const { state } = await inspectExactRecord(claimPath, claimRaw); if (state === 'missing' || state === 'other') return; throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, @@ -859,22 +873,29 @@ async function inspectClaimedPrimary( lockPath: string, sourceRaw: string, sessionId: string, -): Promise { +): Promise> { try { const stat = await fs.lstat(lockPath); - if (!stat.isFile() || stat.isSymbolicLink()) return 'other'; + if (!stat.isFile() || stat.isSymbolicLink()) return { state: 'other' }; const raw = await fs.readFile(lockPath, 'utf8'); - if (raw === sourceRaw) return 'source'; + if (raw === sourceRaw) return { state: 'source' }; const record = parseLockRecord(raw); - return record?.schema_version === LOCK_SCHEMA_VERSION && - record.state === 'active' && - record.session_id === sessionId - ? 'candidate' - : 'other'; + return { + state: + record?.schema_version === LOCK_SCHEMA_VERSION && + record.state === 'active' && + record.session_id === sessionId + ? 'candidate' + : 'other', + }; } catch (error) { - return (error as NodeJS.ErrnoException).code === 'ENOENT' - ? 'missing' - : 'unknown'; + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return { state: 'missing' }; + } + return { + state: 'unknown', + cause: error instanceof Error ? error : undefined, + }; } } @@ -882,7 +903,7 @@ async function assertExactTransitionClaim( claimPath: string, claimRaw: string, ): Promise { - const claimState = await inspectExactRecord(claimPath, claimRaw); + const { state: claimState } = await inspectExactRecord(claimPath, claimRaw); if (claimState !== 'exact') { throw new SessionWriterUnavailableError({ cause: new Error('Session writer transition claim ownership was lost'), @@ -916,7 +937,11 @@ async function linkClaimedPrimary( await fs.link(sourcePath, lockPath); return; } catch (error) { - const state = await inspectClaimedPrimary(lockPath, sourceRaw, sessionId); + const { state } = await inspectClaimedPrimary( + lockPath, + sourceRaw, + sessionId, + ); if (state === 'source') return; if (state === 'candidate') { // A schema-v2 acquirer can pass its first claim check immediately @@ -943,19 +968,25 @@ async function removeClaimedPrimary( let candidateWaitAttempts = 0; for (;;) { await assertExactTransitionClaim(claimPath, claimRaw); - const state = await inspectClaimedPrimary(lockPath, sourceRaw, sessionId); + const { state, cause } = await inspectClaimedPrimary( + lockPath, + sourceRaw, + sessionId, + ); if (state === 'missing') return; if (state === 'candidate') { await waitForClaimedPrimaryCandidate(++candidateWaitAttempts); continue; } if (state === 'other') throw new SessionWriterLostError(); - if (state === 'unknown') throw new SessionWriterUnavailableError(); + if (state === 'unknown') { + throw new SessionWriterUnavailableError({ cause }); + } try { await fs.unlink(lockPath); return; } catch (error) { - const afterState = await inspectClaimedPrimary( + const { state: afterState } = await inspectClaimedPrimary( lockPath, sourceRaw, sessionId, @@ -986,16 +1017,16 @@ async function transitionExactPrimary( try { await fs.rename(lockPath, retiredPath); } catch (error) { - const [primaryExpectedState, retiredState] = await Promise.all([ + const [primaryExpected, retired] = await Promise.all([ inspectExactRecord(lockPath, expectedRaw), inspectExactRecord(retiredPath, expectedRaw), ]); - if (primaryExpectedState === 'exact' && retiredState === 'missing') { + if (primaryExpected.state === 'exact' && retired.state === 'missing') { throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, }); } - if (retiredState !== 'exact') { + if (retired.state !== 'exact') { throw new SessionWriterUnavailableError({ cause: error instanceof Error ? error : undefined, }); @@ -1051,11 +1082,11 @@ async function rollbackExactTransition( claimPath, claimRaw, ); - const retiredState = await inspectExactRecord(retiredPath, expectedRaw); - if (retiredState !== 'exact') { - throw retiredState === 'other' + const retired = await inspectExactRecord(retiredPath, expectedRaw); + if (retired.state !== 'exact') { + throw retired.state === 'other' ? new SessionWriterLostError() - : new SessionWriterUnavailableError(); + : new SessionWriterUnavailableError({ cause: retired.cause }); } try { await linkClaimedPrimary( @@ -1379,7 +1410,7 @@ export class SessionWriterLease { } catch (error) { const cleanupFailures: unknown[] = []; const claimState = claimAcquired - ? await inspectExactRecord(claimPath, claimRaw) + ? (await inspectExactRecord(claimPath, claimRaw)).state : 'missing'; if (claimState === 'unknown') { cleanupFailures.push( @@ -1795,7 +1826,7 @@ export class SessionWriterLease { } catch (error) { const cleanupFailures: unknown[] = []; const claimState = claimAcquired - ? await inspectExactRecord(this.claimPath, this.lockRecordRaw) + ? (await inspectExactRecord(this.claimPath, this.lockRecordRaw)).state : 'missing'; if (claimState === 'unknown') { cleanupFailures.push( @@ -1843,7 +1874,7 @@ export class SessionWriterLease { cleanupFailures.push(cleanupError); } if ( - (await inspectExactRecord(this.lockPath, this.lockRecordRaw)) !== + (await inspectExactRecord(this.lockPath, this.lockRecordRaw)).state !== 'exact' ) { this.released = true;