From 799dbf13915e049a451e0108dea7f088b5bc88c4 Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Sun, 2 Aug 2026 03:33:18 +0800 Subject: [PATCH 1/7] feat(metering): route history compaction through the canonical seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First of the two compaction call kinds #1679 left unrouted. `history_compact` was still writing a per-send row into the frozen `LlmCallRecord` table, which is the last place a model call is metered outside `ModelCallAttempt`. The summarizer already built a `ProviderRequestTracker` for capture and attempt diagnostics; it now also carries accounting, so the call settles the same way a main send does — one record per physical provider request, with `usageBasis` and `costBasis` instead of a cost that cannot say whether it is real. **Accounting is supplied per call, by the backend.** The host wires the summarizer once at composition time and cannot know `runId`, which is per-turn state only the backend holds. Rather than plumbing a turn-to-run resolver down through the kernel and the backend factory context so the host could look up something the backend already has, `AiSdkBackend.modelCallAccounting(callKind)` builds the identity and the caller passes it in at the moment the call is made. The main send path now uses the same factory, so the three call kinds share one construction instead of repeating it. The legacy writer goes out in the same commit: keeping both would have made history compaction the double-metered path that #1755 removed everywhere else. Co-Authored-By: Claude Opus 5 --- .../src/server/execution-model-composition.ts | 8 -- .../history-compact-summarizer.test.ts | 92 ++++++++++--------- packages/runtime/src/ai-sdk-backend.ts | 53 +++++++---- .../runtime/src/ai-sdk-compaction-contract.ts | 8 ++ packages/runtime/src/ai-sdk-compaction.ts | 18 ++++ .../runtime/src/history-compact-summarizer.ts | 45 +-------- 6 files changed, 113 insertions(+), 111 deletions(-) diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index a6f24a5006..720550d38d 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -647,14 +647,6 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom }, } : {}), - telemetry: { - connectionSlug: target.connection.slug, - providerId: target.connection.providerType, - modelId: target.model, - newId: randomUUID, - now: Date.now, - recordLlmCall: recordLlmUsage, - }, }), recordHistoryCompactCheckpoint: input.context.recordHistoryCompactCheckpoint, loadTurnRuntimeEvents: input.context.loadTurnRuntimeEvents, diff --git a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts index 04d15f7e0c..9305b3d4b2 100644 --- a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts +++ b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts @@ -4,11 +4,12 @@ * * Run: `npm --workspace @maka/runtime run test` */ +import { MockLanguageModelV4 } from 'ai/test'; import { describe, test } from 'node:test'; import assert from 'node:assert/strict'; import { expect } from '../test-helpers.js'; import type { RuntimeEvent, RuntimeEventContent } from '@maka/core/runtime-event'; -import type { LlmCallRecord } from '@maka/core/usage-stats/types'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import type { HistoryCompactSummaryInput } from '../ai-sdk-compaction-contract.js'; import { buildLlmHistorySummarizer, @@ -96,63 +97,64 @@ describe('buildLlmHistorySummarizer', () => { expect(seen?.maxOutputTokens).toBe(undefined); }); - test('attributes provider-reported usage to one history-compaction call', async () => { - const records: LlmCallRecord[] = []; + test('attributes provider-reported usage to one canonical history-compaction record', async () => { + // history_compact used to write a per-call row into the frozen table. It + // now settles through the same seam as a main send, so the record carries + // the run it belongs to and its cost basis (#1679). + const recorded: ModelCallAttempt[] = []; let now = 100; const summarize = buildLlmHistorySummarizer({ - resolveModel: () => 'fake-model', - generateText: async () => ({ - text: '## Goal\nX', - finishReason: 'stop', - usage: { - inputTokens: 7, - outputTokens: 3, - totalTokens: 10, - }, - }), - telemetry: { - connectionSlug: 'connection', - providerId: 'provider', - modelId: 'model', - newId: () => 'call-id', + resolveModel: () => + new MockLanguageModelV4({ + doGenerate: { + content: [{ type: 'text', text: '## Goal\nX' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 7, noCache: 7, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 3, text: 3, reasoning: 0 }, + }, + warnings: [], + }, + }), + providerRequestTracking: { now: () => { now += 10; return now; }, - recordLlmCall: (record) => { - records.push(record); - }, + newId: () => 'trace-id', + persistCapture: async () => ({ artifactId: 'artifact-1' }), + recordAttempt: () => {}, }, }); - await summarize( - inputWith([ev({ role: 'user', author: 'user', content: { kind: 'text', text: 'hi' } })]), - ); - - assert.deepEqual(records, [ - { + await summarize({ + ...inputWith([ev({ role: 'user', author: 'user', content: { kind: 'text', text: 'hi' } })]), + accounting: { sessionId: 'sess-1', - turnId: 'turn-1', - callKind: 'history_compact', - callId: 'history_compact_turn-1_call-id', + resolveRunId: () => 'run-1', connectionSlug: 'connection', providerId: 'provider', - modelId: 'model', - inputTokens: 7, - outputTokens: 3, - cacheHitInputTokens: 0, - cacheMissInputTokens: 7, - cacheMissInputSource: 'derived', - cachedInputTokens: 0, - cacheWriteInputTokens: 0, - reasoningTokens: 0, - totalTokens: 10, - rawFinishReason: 'stop', - latencyMs: 10, - status: 'success', - startedAt: 110, + callKind: 'history_compact', + record: (attempt) => { + recorded.push(attempt); + }, }, - ]); + }); + + const attempt = decodeModelCallAttempt(recorded[0]); + assert.equal(attempt.callKind, 'history_compact'); + assert.equal(attempt.sessionId, 'sess-1'); + assert.equal(attempt.runId, 'run-1'); + assert.equal(attempt.turnId, 'turn-1'); + assert.equal(attempt.connectionSlug, 'connection'); + assert.equal(attempt.providerId, 'provider'); + assert.equal(attempt.inputTokens, 7); + assert.equal(attempt.outputTokens, 3); + assert.equal(attempt.usageBasis, 'reported'); + // No pricing was wired, so the record says the price is unknown rather + // than claiming the summarization was free. + assert.equal(attempt.costBasis, 'unpriced'); + assert.equal(attempt.costUsd, undefined); }); test('produces schema-valid tool-result messages (toolName + wrapped output) and does not fall back', async () => { diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 6bc853290b..077cddd8db 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -160,9 +160,10 @@ import { toolSchemaCharsForDiagnostics, type RequestShapeDiagnostic, } from './request-shape.js'; -import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; +import type { ModelCallAttempt, ModelCallKind } from '@maka/core/model-call-attempt'; import { ProviderRequestTracker, + type ModelCallAccountingInput, type ProviderRequestAttemptRecord, type ProviderRequestCaptureRecord, type ProviderRequestUsage, @@ -574,6 +575,7 @@ export class AiSdkBackend implements AgentBackend { now: this.now, modelAdapter: this.modelAdapter, computeCostUsd: (usage) => this.computeTokenUsageCostUsd(usage), + modelCallAccounting: (callKind) => this.modelCallAccounting(callKind), materializeRuntimeReplayPlan: (plan) => this.materializeRuntimeReplayPlan(plan), canReplayProviderNative: (plan) => this.canReplayProviderNative(plan), appendTurnTailPrompt: (content, turnTailPrompt) => @@ -811,22 +813,10 @@ export class AiSdkBackend implements AgentBackend { newId: this.newId, persistCapture: recordProviderRequestCapture!, recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), - ...(this.input.recordModelCallAttempt - ? { - accounting: { - sessionId: this.sessionId, - resolveRunId: () => this.currentRunId ?? undefined, - connectionSlug: this.input.connection.slug, - providerId: this.input.connection.providerType, - callKind: 'main' as const, - record: this.input.recordModelCallAttempt, - resolveCost: (usage: ProviderRequestUsage) => this.resolveModelCallCost(usage), - ...(this.input.assertModelCallAccountingReady - ? { assertReady: this.input.assertModelCallAccountingReady } - : {}), - }, - } - : {}), + ...(() => { + const accounting = this.modelCallAccounting('main'); + return accounting ? { accounting } : {}; + })(), }) : undefined; @@ -2073,6 +2063,35 @@ export class AiSdkBackend implements AgentBackend { * record then carries `costBasis: 'unpriced'`, which is not the same claim as * a call that was free. */ + /** + * Accounting identity for one call kind (#1679). + * + * Owned here rather than by the host that wires the summarizers, because the + * only field a host cannot supply is the one that changes per turn: `runId` + * lives on this backend. A host configures the capture and attempt plumbing + * once; the run a record belongs to is resolved at the moment the call is + * actually made. + * + * Absent when there is no canonical sink, which leaves the corresponding + * tracker purely diagnostic. + */ + modelCallAccounting(callKind: ModelCallKind): ModelCallAccountingInput | undefined { + const record = this.input.recordModelCallAttempt; + if (!record) return undefined; + return { + sessionId: this.sessionId, + resolveRunId: () => this.currentRunId ?? undefined, + connectionSlug: this.input.connection.slug, + providerId: this.input.connection.providerType, + callKind, + record, + resolveCost: (usage: ProviderRequestUsage) => this.resolveModelCallCost(usage), + ...(this.input.assertModelCallAccountingReady + ? { assertReady: this.input.assertModelCallAccountingReady } + : {}), + }; + } + private resolveModelCallCost(usage: ProviderRequestUsage): ResolvedModelCallCost | undefined { try { const pricing = (this.input.lookupPricing ?? getBuiltinPricing)( diff --git a/packages/runtime/src/ai-sdk-compaction-contract.ts b/packages/runtime/src/ai-sdk-compaction-contract.ts index 3670c9b6bf..5c8c5bc813 100644 --- a/packages/runtime/src/ai-sdk-compaction-contract.ts +++ b/packages/runtime/src/ai-sdk-compaction-contract.ts @@ -2,6 +2,7 @@ import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { LlmCallRecord } from '@maka/core/usage-stats/types'; +import type { ModelCallAccountingInput } from './provider-request-telemetry.js'; import type { ActiveFullCompactBlock } from './active-full-compact.js'; import type { ActiveToolResultArchiveCandidate } from './active-tool-result-prune.js'; import type { @@ -120,6 +121,13 @@ export interface HistoryCompactSummaryInput { newlyFoldedRuntimeEvents?: RuntimeEvent[]; requestShapeHashBefore?: string; abortSignal?: AbortSignal; + /** + * Accounting identity for this summarization call (#1679). Supplied per call + * rather than baked into the summarizer, because the host that configures the + * summarizer cannot know which run is active — `runId` is per-turn state the + * backend holds. + */ + accounting?: ModelCallAccountingInput; } export type HistoryCompactSummarizer = ( input: HistoryCompactSummaryInput, diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 47e5ee8ce3..763bd8df18 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -87,6 +87,11 @@ import { type RuntimeEventModelReplayPlan, } from './model-history.js'; import { toolSchemaCharsForDiagnostics } from './request-shape.js'; +import type { ModelCallKind } from '@maka/core/model-call-attempt'; +import { + ProviderRequestTracker, + type ModelCallAccountingInput, +} from './provider-request-telemetry.js'; import { estimateNextRequestTokens, exceedsHighWater, @@ -101,6 +106,11 @@ export interface AiSdkCompactionDeps { now: () => number; modelAdapter: ModelAdapter; computeCostUsd: (usage: NormalizedAiSdkUsage) => number | undefined; + /** + * Accounting identity for a compaction call, resolved by the backend because + * `runId` changes per turn and only it holds the current one (#1679). + */ + modelCallAccounting: (callKind: ModelCallKind) => ModelCallAccountingInput | undefined; materializeRuntimeReplayPlan: (plan: RuntimeEventModelReplayPlan) => Promise; canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; appendTurnTailPrompt: ( @@ -115,6 +125,9 @@ export class AiSdkCompaction { private readonly now: () => number; private readonly modelAdapter: ModelAdapter; private readonly computeCostUsd: (usage: NormalizedAiSdkUsage) => number | undefined; + private readonly modelCallAccounting: ( + callKind: ModelCallKind, + ) => ModelCallAccountingInput | undefined; private readonly materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, ) => Promise; @@ -131,6 +144,7 @@ export class AiSdkCompaction { this.now = deps.now; this.modelAdapter = deps.modelAdapter; this.computeCostUsd = deps.computeCostUsd; + this.modelCallAccounting = deps.modelCallAccounting; this.materializeRuntimeReplayPlan = deps.materializeRuntimeReplayPlan; this.canReplayProviderNative = deps.canReplayProviderNative; this.appendTurnTailPrompt = deps.appendTurnTailPrompt; @@ -386,6 +400,7 @@ export class AiSdkCompaction { const summarizer = this.input.summarizeHistoryCompact; const recorder = this.input.recordHistoryCompactCheckpoint; if (!summarizer || !recorder) return { diagnosticPatch: {} }; + const historyCompactAccounting = this.modelCallAccounting('history_compact'); const foldedIds = new Set(input.draftBlock.coverage.runtimeEventIds); const foldedRuntimeEvents = input.priorRuntimeContext.filter((event) => foldedIds.has(event.id), @@ -461,6 +476,7 @@ export class AiSdkCompaction { newlyFoldedRuntimeEvents, requestShapeHashBefore: input.requestShapeHashBefore, abortSignal: input.abortSignal, + ...(historyCompactAccounting ? { accounting: historyCompactAccounting } : {}), }), ); if (!summary?.trim()) { @@ -1466,6 +1482,7 @@ export class AiSdkCompaction { abortSignal, } = input; const summarizer = this.input.summarizeHistoryCompact!; + const midTurnAccounting = this.modelCallAccounting('history_compact'); const recorder = this.input.recordHistoryCompactCheckpoint!; const loadTurnRuntimeEvents = this.input.loadTurnRuntimeEvents!; const policy = this.input.contextBudget!; @@ -1563,6 +1580,7 @@ export class AiSdkCompaction { ...(previousCheckpoint ? { previousCheckpoint } : {}), newlyFoldedRuntimeEvents: [...newlyFoldedRuntimeEvents], ...(abortSignal ? { abortSignal } : {}), + ...(midTurnAccounting ? { accounting: midTurnAccounting } : {}), }), ); }, diff --git a/packages/runtime/src/history-compact-summarizer.ts b/packages/runtime/src/history-compact-summarizer.ts index 07bda16f8c..d46d801840 100644 --- a/packages/runtime/src/history-compact-summarizer.ts +++ b/packages/runtime/src/history-compact-summarizer.ts @@ -44,15 +44,6 @@ export interface BuildLlmHistorySummarizerOptions { generateText?: AiSdkGenerateTextLike; /** Physical provider-call capture and attempt tracking for generated summaries. */ providerRequestTracking?: Omit; - /** Usage attribution for the auxiliary history-compaction call. */ - telemetry?: { - connectionSlug: string; - providerId: string; - modelId: string; - newId: () => string; - now: () => number; - recordLlmCall: LlmTelemetryRecorder; - }; } // Conversation-summarization prompt (sectioned, modelled on pi/opencode): @@ -111,6 +102,10 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti ...options.providerRequestTracking, traceId: options.providerRequestTracking.newId(), turnId: input.turnId, + // Per call, not per summarizer: the host wires the capture and + // attempt plumbing once, but only the caller knows the run this + // summarization belongs to. + ...(input.accounting ? { accounting: input.accounting } : {}), }) : undefined; const ai = @@ -135,7 +130,6 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti }, }) : options.resolveModel(); - const startedAt = options.telemetry?.now(); const result = await generateText({ model, instructions: SUMMARIZATION_SYSTEM_PROMPT, @@ -145,9 +139,6 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti : {}), ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), }); - if (options.telemetry && startedAt !== undefined) { - recordHistoryCompactCall(options.telemetry, input, startedAt, result); - } if (rawFinishReasonString(result.finishReason) === 'length') { throw new HistoryCompactSummarizerError('output_length'); } @@ -159,34 +150,6 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti }; } -function recordHistoryCompactCall( - telemetry: NonNullable, - input: HistoryCompactSummaryInput, - startedAt: number, - result: Awaited>, -): void { - const usage = normalizeAiSdkUsage(result.usage, { rawFinishReason: result.finishReason }); - if (!usage) return; - const completedAt = telemetry.now(); - try { - telemetry.recordLlmCall({ - sessionId: input.sessionId, - turnId: input.turnId, - callKind: 'history_compact', - callId: `history_compact_${input.turnId}_${telemetry.newId()}`, - connectionSlug: telemetry.connectionSlug, - providerId: telemetry.providerId, - modelId: telemetry.modelId, - ...llmCallUsageFields(usage), - latencyMs: Math.max(0, completedAt - startedAt), - status: 'success', - startedAt, - }); - } catch { - // Usage telemetry is diagnostic. The summary remains authoritative. - } -} - interface AiSdkTextModule { generateText: AiSdkGenerateTextLike; wrapLanguageModel(input: Record): unknown; From e7071413b754e45f217a5b421c0f8d85699de91a Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Sun, 2 Aug 2026 12:04:20 +0800 Subject: [PATCH 2/7] feat(metering): route semantic compaction through the canonical seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second of the two compaction call kinds #1679 left unrouted, and the last model call anywhere that was metered by hand. `semantic_compact` had no tracker at all: it called `generateCompactSummary` directly and then built an `LlmCallRecord` around the returned usage. It now settles the way every other provider request does. `generateCompactSummary` takes an optional `ProviderRequestTracker` and wraps the model with a `wrapGenerate` middleware — the exact mirror of what `startStream` already does with `wrapStream`, and in the one place that already owns "attach a tracker to a model". The success, failure, and abort paths all settle inside the tracker, so the hand-rolled try/catch that recorded three ways goes with it. The backend hands the summarizer a *built* tracker rather than the capture, attempt, and id sinks it is made of: compaction has no business assembling metering identity, and a half-wired tracker is what produces records nothing can attribute. `createProviderRequestTracker` also absorbed the main send's own tracker construction, so this kind was added without a second copy of it. One trace per turn rather than per call, so a step that summarizes is a step of that trace and a retried summarization is another attempt of the same logical call. Built on first use — most turns never summarize, and an unused trace id is a trace that never happened. The legacy writer goes out in the same commit, along with the now-unused `computeCostUsd` dep: cost is resolved at settlement by the seam's own `resolveCost`, with a basis attached. One behavioural difference worth stating: the old row copied the SDK's normalized `cacheRead` through as a cache hit. The canonical record attributes cache tokens only when the provider's own payload claims them, so a provider that reports none now yields an absent field instead of a number no provider ever said. `@maka/runtime` 2648/2661 (the 4 failures are the documented local `rg` noise). Co-Authored-By: Claude Opus 5 --- .../src/__tests__/ai-sdk-backend.test.ts | 92 ++++++------- packages/runtime/src/ai-sdk-backend.ts | 74 +++++++---- packages/runtime/src/ai-sdk-compaction.ts | 125 +++++++----------- packages/runtime/src/model-adapter.ts | 40 +++++- 4 files changed, 170 insertions(+), 161 deletions(-) diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index c02c2ce72c..20ae02ab7c 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -74,6 +74,7 @@ import type { ProviderRequestAttemptRecord, ProviderRequestCaptureRecord, } from '../provider-request-telemetry.js'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; describe('AiSdkBackend model history', () => { @@ -8043,54 +8044,11 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(recordedBlocks[0]?.blockId, 'afcompact-sync-test'); }); - test('does not record semantic compact usage when provider usage is unavailable', () => { - const llmRecords: LlmCallRecord[] = []; - const backend = createTestAiSdkBackend({ - sessionId: 'session-1', - header: header(), - appendMessage: async () => {}, - connection: connection(), - apiKey: 'sk-test', - modelId: 'mock-model-id', - modelFactory: () => completionModel(), - tools: [], - newId: idGenerator(), - now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, - }); - - ( - backend as unknown as { - compaction: { - recordSemanticCompactSummaryCall(input: { - callId: string; - turnId: string; - modelId: string; - startedAt: number; - latencyMs: number; - status: LlmCallRecord['status']; - }): void; - }; - } - ).compaction.recordSemanticCompactSummaryCall({ - callId: 'semantic-1', - turnId: 'turn-1', - modelId: 'mock-model-id', - startedAt: 1, - latencyMs: 2, - status: 'error', - }); - - assert.deepEqual(llmRecords, []); - }); - - test('semantic compact records a separate no-tools summarizer LLM call', async () => { + test('semantic compact records a separate no-tools summarizer model call', async () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; + const modelCalls: ModelCallAttempt[] = []; const recordedBlocks: SemanticCompactBlock[] = []; const recordedActiveFullBlocks: ActiveFullCompactBlock[] = []; const largeBody = 'SEMANTIC_COMPACT_RAW_TOOL_OUTPUT'.repeat(180); @@ -8224,8 +8182,9 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-semantic-capture' }), + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); }, recordSemanticCompactBlock: (block) => { recordedBlocks.push(block); @@ -8235,7 +8194,9 @@ describe('AiSdkBackend usage telemetry', () => { }, }); - for await (const event of backend.send(durable.input())) { + // A canonical record belongs to a run; the send carries the one the + // harness's runtime events already claim. + for await (const event of backend.send(durable.input({ runId: 'run-1' }))) { durable.record(event); events.push(event); } @@ -8304,13 +8265,34 @@ describe('AiSdkBackend usage telemetry', () => { 'projection replay after active pruning must retain the exact single user anchor', ); - const semanticRecord = llmRecords.find((record) => record.callKind === 'semantic_compact'); - assert.ok(semanticRecord, 'expected semantic compact LLM record'); - assert.match(semanticRecord.callId ?? '', /^semantic_compact_turn-1_2_/); - assert.equal(semanticRecord.inputTokens, 21); - assert.equal(semanticRecord.outputTokens, 13); - assert.equal(semanticRecord.cacheHitInputTokens, 2); - assert.equal(semanticRecord.totalTokens, 34); + // The summarization is one physical provider request, metered through the + // same canonical seam as the send it interrupts (#1679) — not a hand-built + // row in the frozen usage table. + const semanticAttempt = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'semantic_compact'); + assert.ok(semanticAttempt, 'expected a canonical semantic compact record'); + assert.equal(semanticAttempt.sessionId, 'session-1'); + assert.equal(semanticAttempt.runId, 'run-1'); + assert.equal(semanticAttempt.turnId, 'turn-1'); + assert.equal( + semanticAttempt.step, + 2, + 'the record carries the send step whose projection triggered the summarization', + ); + assert.equal(semanticAttempt.status, 'completed'); + assert.equal(semanticAttempt.inputTokens, 21); + assert.equal(semanticAttempt.outputTokens, 13); + assert.equal(semanticAttempt.usageBasis, 'reported'); + // The old row copied the SDK's normalized `cacheRead` through as a cache + // hit. The canonical record only attributes cache tokens the provider's own + // payload claims, and this mock ships none — so absent, not zero. + assert.equal(semanticAttempt.cacheReadInputTokens, undefined); + assert.equal( + modelCalls.filter((attempt) => attempt.callKind === 'semantic_compact').length, + 1, + 'one summarization is one record', + ); const usageEvent = events.find((event) => event.type === 'token_usage') as | (Extract & { diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 077cddd8db..9c56c1bba0 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -574,8 +574,9 @@ export class AiSdkBackend implements AgentBackend { sessionId: this.sessionId, now: this.now, modelAdapter: this.modelAdapter, - computeCostUsd: (usage) => this.computeTokenUsageCostUsd(usage), modelCallAccounting: (callKind) => this.modelCallAccounting(callKind), + createProviderRequestTracker: (trackerInput) => + this.createProviderRequestTracker(trackerInput), materializeRuntimeReplayPlan: (plan) => this.materializeRuntimeReplayPlan(plan), canReplayProviderNative: (plan) => this.canReplayProviderNative(plan), appendTurnTailPrompt: (content, turnTailPrompt) => @@ -799,26 +800,12 @@ export class AiSdkBackend implements AgentBackend { toSandboxRunTraceProjection(this.input.sandboxDiagnosticsSnapshot), ); } - const recordProviderRequestCapture = this.input.recordProviderRequestCapture; - const providerRequestTraceId = recordProviderRequestCapture ? this.newId() : undefined; - const providerRequestTracker = providerRequestTraceId - ? new ProviderRequestTracker({ - traceId: providerRequestTraceId, - turnId, - contextWindow: resolveSelectedModelContextWindow( - this.input.connection, - this.input.modelId, - ), - now: this.now, - newId: this.newId, - persistCapture: recordProviderRequestCapture!, - recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), - ...(() => { - const accounting = this.modelCallAccounting('main'); - return accounting ? { accounting } : {}; - })(), - }) - : undefined; + const providerRequestTracker = this.createProviderRequestTracker({ + turnId, + callKind: 'main', + modelId: this.input.modelId, + }); + const providerRequestTraceId = providerRequestTracker?.traceId; // --- Resolve model (API key already attached at construct time) --- let model: unknown; @@ -2054,15 +2041,36 @@ export class AiSdkBackend implements AgentBackend { } /** - * Resolves cost for a canonical accounting record at settlement time, together - * with the rates it was computed against. + * One tracker for one physical provider call kind (#1679). * - * The basis travels with the amount because a figure recomputed later from - * whatever pricing is current would silently drift from what the call actually - * cost. An unresolvable price returns `undefined` rather than zero — the - * record then carries `costBasis: 'unpriced'`, which is not the same claim as - * a call that was free. + * Auxiliary calls get the same capture, attempt, and accounting plumbing the + * main send uses, built here because the sinks and the current run live on + * this backend. Callers receive a ready tracker rather than the ingredients: + * a half-wired tracker is what produces records nothing can attribute. + * + * Absent when capture is not wired, which leaves the call untracked exactly + * as it was before. */ + private createProviderRequestTracker(input: { + turnId: string; + callKind: ModelCallKind; + modelId: string; + }): ProviderRequestTracker | undefined { + const persistCapture = this.input.recordProviderRequestCapture; + if (!persistCapture) return undefined; + const accounting = this.modelCallAccounting(input.callKind); + return new ProviderRequestTracker({ + traceId: this.newId(), + turnId: input.turnId, + contextWindow: resolveSelectedModelContextWindow(this.input.connection, input.modelId), + now: this.now, + newId: this.newId, + persistCapture, + recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), + ...(accounting ? { accounting } : {}), + }); + } + /** * Accounting identity for one call kind (#1679). * @@ -2092,6 +2100,16 @@ export class AiSdkBackend implements AgentBackend { }; } + /** + * Resolves cost for a canonical accounting record at settlement time, together + * with the rates it was computed against. + * + * The basis travels with the amount because a figure recomputed later from + * whatever pricing is current would silently drift from what the call actually + * cost. An unresolvable price returns `undefined` rather than zero — the + * record then carries `costBasis: 'unpriced'`, which is not the same claim as + * a call that was free. + */ private resolveModelCallCost(usage: ProviderRequestUsage): ResolvedModelCallCost | undefined { try { const pricing = (this.input.lookupPricing ?? getBuiltinPricing)( diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 763bd8df18..4039dfa1df 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -15,7 +15,7 @@ import type { BackendCompactHistoryResult, BackendSendInput, } from '@maka/core/backend-types'; -import type { ContextBudgetDiagnostic, LlmCallRecord } from '@maka/core/usage-stats/types'; +import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; import type { AiSdkCompactionCapabilities } from './ai-sdk-compaction-contract.js'; import { @@ -49,12 +49,7 @@ import { import { createHash } from 'node:crypto'; import type { ModelMessage } from './model-protocol.js'; -import { - normalizeAiSdkUsage, - type ModelAdapter, - type NormalizedAiSdkUsage, -} from './model-adapter.js'; -import { llmCallUsageFields } from './telemetry/llm-call-usage.js'; +import { normalizeAiSdkUsage, type ModelAdapter } from './model-adapter.js'; import type { RequestProjection, RequestProjectionContext, @@ -88,9 +83,9 @@ import { } from './model-history.js'; import { toolSchemaCharsForDiagnostics } from './request-shape.js'; import type { ModelCallKind } from '@maka/core/model-call-attempt'; -import { +import type { + ModelCallAccountingInput, ProviderRequestTracker, - type ModelCallAccountingInput, } from './provider-request-telemetry.js'; import { estimateNextRequestTokens, @@ -105,12 +100,21 @@ export interface AiSdkCompactionDeps { sessionId: string; now: () => number; modelAdapter: ModelAdapter; - computeCostUsd: (usage: NormalizedAiSdkUsage) => number | undefined; /** * Accounting identity for a compaction call, resolved by the backend because * `runId` changes per turn and only it holds the current one (#1679). */ modelCallAccounting: (callKind: ModelCallKind) => ModelCallAccountingInput | undefined; + /** + * A ready tracker for a compaction call that has none of its own. The backend + * hands over the built tracker rather than the capture, attempt, and id sinks + * it is made of: compaction has no business assembling metering identity. + */ + createProviderRequestTracker: (input: { + turnId: string; + callKind: ModelCallKind; + modelId: string; + }) => ProviderRequestTracker | undefined; materializeRuntimeReplayPlan: (plan: RuntimeEventModelReplayPlan) => Promise; canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; appendTurnTailPrompt: ( @@ -124,10 +128,14 @@ export class AiSdkCompaction { private readonly sessionId: string; private readonly now: () => number; private readonly modelAdapter: ModelAdapter; - private readonly computeCostUsd: (usage: NormalizedAiSdkUsage) => number | undefined; private readonly modelCallAccounting: ( callKind: ModelCallKind, ) => ModelCallAccountingInput | undefined; + private readonly createProviderRequestTracker: (input: { + turnId: string; + callKind: ModelCallKind; + modelId: string; + }) => ProviderRequestTracker | undefined; private readonly materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, ) => Promise; @@ -143,8 +151,8 @@ export class AiSdkCompaction { this.sessionId = deps.sessionId; this.now = deps.now; this.modelAdapter = deps.modelAdapter; - this.computeCostUsd = deps.computeCostUsd; this.modelCallAccounting = deps.modelCallAccounting; + this.createProviderRequestTracker = deps.createProviderRequestTracker; this.materializeRuntimeReplayPlan = deps.materializeRuntimeReplayPlan; this.canReplayProviderNative = deps.canReplayProviderNative; this.appendTurnTailPrompt = deps.appendTurnTailPrompt; @@ -1003,6 +1011,24 @@ export class AiSdkCompaction { compactCallTotalTokens: 0, acceptedEstimatedTokensSaved: 0, }; + // One auxiliary trace per turn rather than per call: a step that summarizes + // is a step of that trace, so a retried summarization is another attempt of + // the same logical call. Built on first use — most turns never summarize, + // and an unused trace id is a trace that never happened. + const summarizerModelId = policy.summarizerModel ?? this.input.modelId; + let summaryTracker: ProviderRequestTracker | undefined; + let summaryTrackerBuilt = false; + const resolveSummaryTracker = (): ProviderRequestTracker | undefined => { + if (!summaryTrackerBuilt) { + summaryTrackerBuilt = true; + summaryTracker = this.createProviderRequestTracker({ + turnId, + callKind: 'semantic_compact', + modelId: summarizerModelId, + }); + } + return summaryTracker; + }; return async (options) => { const activeToolsForStep = options.activeTools; const dryRun = policy.mode === 'validate_only' || policy.mode === 'prepare_step_dry_run'; @@ -1018,7 +1044,6 @@ export class AiSdkCompaction { modelId: policy.summarizerModel, }) : model; - const summarizerModelId = policy.summarizerModel ?? this.input.modelId; const rewritten = await rewriteSemanticCompactInMessages({ sessionId: this.sessionId, turnId, @@ -1037,38 +1062,19 @@ export class AiSdkCompaction { : {}), abortSignal: abortSignal, summarizer: async (request) => { - const startedAt = this.now(); - const callId = `semantic_compact_${turnId}_${options.stepNumber}_${startedAt}`; - try { - const result = await this.modelAdapter.generateCompactSummary({ - model: summarizerModel, - system: request.system, - messages: request.messages, - maxOutputTokens: request.maxOutputTokens, - abortSignal: request.abortSignal, - }); - this.recordSemanticCompactSummaryCall({ - callId, - turnId, - modelId: summarizerModelId, - startedAt, - latencyMs: Math.max(0, this.now() - startedAt), - usage: result.usage, - status: 'success', - }); - return result; - } catch (error) { - this.recordSemanticCompactSummaryCall({ - callId, - turnId, - modelId: summarizerModelId, - startedAt, - latencyMs: Math.max(0, this.now() - startedAt), - status: request.abortSignal?.aborted ? 'aborted' : 'error', - errorClass: this.modelAdapter.classifyError(error), - }); - throw error; - } + // The tracker settles this call itself, on the success, failure, and + // abort paths alike, so the summarization is metered as the physical + // provider request it is instead of a hand-built row (#1679). + const tracker = resolveSummaryTracker(); + tracker?.setStep(options.stepNumber); + return await this.modelAdapter.generateCompactSummary({ + model: summarizerModel, + system: request.system, + messages: request.messages, + maxOutputTokens: request.maxOutputTokens, + abortSignal: request.abortSignal, + ...(tracker ? { providerRequestTracker: tracker } : {}), + }); }, }); onDiagnosticPatch?.({ @@ -1150,35 +1156,6 @@ export class AiSdkCompaction { }; } - private recordSemanticCompactSummaryCall(input: { - callId: string; - turnId: string; - modelId: string; - startedAt: number; - latencyMs: number; - usage?: NormalizedAiSdkUsage; - status: LlmCallRecord['status']; - errorClass?: string; - }): void { - if (!input.usage) return; - const costUsd = this.computeCostUsd(input.usage); - this.input.recordLlmCall?.({ - sessionId: this.sessionId, - turnId: input.turnId, - callKind: 'semantic_compact', - callId: input.callId, - connectionSlug: this.input.connection.slug, - providerId: this.input.connection.providerType, - modelId: input.modelId, - ...llmCallUsageFields(input.usage), - latencyMs: input.latencyMs, - status: input.status, - ...(input.errorClass ? { errorClass: input.errorClass } : {}), - startedAt: input.startedAt, - ...(costUsd !== undefined ? { costUsd } : {}), - }); - } - private recordSemanticCompactBlock(block: SemanticCompactBlock): void { const recorder = this.input.recordSemanticCompactBlock; if (!recorder) return; diff --git a/packages/runtime/src/model-adapter.ts b/packages/runtime/src/model-adapter.ts index 3449c9b517..48e5c0f6e4 100644 --- a/packages/runtime/src/model-adapter.ts +++ b/packages/runtime/src/model-adapter.ts @@ -38,7 +38,10 @@ import { errorPresentationFromClass, providerRetryMetadata, } from './provider-error-classification.js'; -import type { ProviderRequestTracker } from './provider-request-telemetry.js'; +import type { + ProviderGenerateResult, + ProviderRequestTracker, +} from './provider-request-telemetry.js'; import { createKimiOpenAiTransportState, kimiReasoningFieldProviderOptions, @@ -86,6 +89,12 @@ export interface CompactSummaryRequest { messages: readonly ModelMessage[]; maxOutputTokens: number; abortSignal?: AbortSignal; + /** + * Physical provider-call tracking for this summarization. Attaching it here + * rather than at the call site keeps "wrap a model with a tracker" in the one + * place that already owns it for streams. + */ + providerRequestTracker?: ProviderRequestTracker; } export interface CompactSummaryResult { @@ -106,7 +115,7 @@ export interface ModelAdapterStreamInput { toolCall: RepairableAiSdkToolCall; error: unknown; }) => RepairableAiSdkToolCall | null | Promise; - /** Main-agent provider-call tracker. Auxiliary model calls intentionally omit it. */ + /** Main-agent provider-call tracker. Auxiliary calls track their own generates. */ providerRequestTracker?: ProviderRequestTracker; } @@ -120,6 +129,12 @@ interface ProviderMiddlewareStreamInput { model: { provider: string; modelId: string }; } +interface ProviderMiddlewareGenerateInput { + doGenerate: () => PromiseLike; + params: Record & { abortSignal?: AbortSignal }; + model: { provider: string; modelId: string }; +} + export class ModelAdapter { private readonly kimiOpenAiTransportState = createKimiOpenAiTransportState(); @@ -257,7 +272,7 @@ export class ModelAdapter { `Failed to load 'ai' package. Run \`npm install ai\`. Inner: ${(err as Error).message}`, ); }); - const { generateText } = ai as unknown as { + const { generateText, wrapLanguageModel } = ai as unknown as { generateText: (opts: Record) => Promise<{ text?: string; usage?: AiSdkUsageLike; @@ -265,10 +280,27 @@ export class ModelAdapter { providerMetadata?: unknown; finalStep?: { response?: { id?: string } }; }>; + wrapLanguageModel: (input: Record) => unknown; }; + const trackedModel = input.providerRequestTracker + ? wrapLanguageModel({ + model: input.model, + middleware: { + wrapGenerate: async ({ doGenerate, params, model }: ProviderMiddlewareGenerateInput) => + await input.providerRequestTracker!.trackGenerate({ + providerId: model.provider, + modelId: model.modelId, + params, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), + doGenerate, + }), + }, + }) + : input.model; + const result = await generateText({ - model: input.model, + model: trackedModel, instructions: input.system, messages: input.messages, maxOutputTokens: input.maxOutputTokens, From 25d58f949084755933c5f3c02403c07c21a44d1a Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Sun, 2 Aug 2026 12:12:05 +0800 Subject: [PATCH 3/7] refactor(metering): drop the compaction path's legacy usage writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With both compaction kinds routed, nothing in the send path writes a `LlmCallRecord` any more. `AiSdkCompactionCapabilities.recordLlmCall` and its `LlmTelemetryRecorder` type had no writer left above them, and the Host and Desktop each still wired a recorder into a backend field nothing read — dead plumbing that a future change could just as easily have brought back to life. Five backend tests still collected the records into arrays nothing asserted on; those go too. **`recordLlmCall` itself stays, and #1679's plan to delete it here was wrong.** `goal_evaluation` is the fourth call kind, and `createHostGoalEvaluator` still writes it to the frozen table. It cannot follow the compaction kinds through this seam as-is: a `ModelCallAttempt` is identified by `(sessionId, runId, turnId)`, and the Host evaluates a goal against a `sessionId` alone — there is no run or turn at that layer to attribute the call to. Giving it one is a design question for the RFC, not something to smuggle into a routing change. The practical consequence is worth stating plainly, because #1679 currently says otherwise: `provenance.legacyRecords` will *not* fall to zero as the old table ages out. Goal evaluations keep landing there, so the merged read path stays load-bearing until that kind is routed too. `@maka/runtime` 2648/2661 (4 = documented local `rg` noise), `@maka/runtime-host` 488/488, `@maka/desktop` 1090/1128 — identical to this worktree's baseline on the parent commit (the 38 are its missing Astryx peer dep, not this change). Co-Authored-By: Claude Opus 5 --- apps/desktop/src/main/session-stream.ts | 8 +++---- .../src/server/execution-model-composition.ts | 6 ------ .../src/__tests__/ai-sdk-backend.test.ts | 21 ------------------- packages/runtime/src/ai-sdk-backend.ts | 1 - .../runtime/src/ai-sdk-compaction-contract.ts | 5 ----- .../runtime/src/history-compact-summarizer.ts | 8 ++----- 6 files changed, 5 insertions(+), 44 deletions(-) diff --git a/apps/desktop/src/main/session-stream.ts b/apps/desktop/src/main/session-stream.ts index 66b5640c98..a6661e36d6 100644 --- a/apps/desktop/src/main/session-stream.ts +++ b/apps/desktop/src/main/session-stream.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { SessionChangedReason, SessionEvent } from '@maka/core'; -import type { LlmCallRecord, ToolInvocationRecord } from '@maka/core/usage-stats/types'; +import type { ToolInvocationRecord } from '@maka/core/usage-stats/types'; import { AiSdkBackend, buildDefaultContextBudgetPolicy, @@ -11,7 +11,6 @@ import { loadHistoryCompactBlocksFromArtifacts, loadSynthesisCacheBlocksFromArtifacts, persistSynthesisCacheBlocksToArtifacts, - recordLlmCall, recordToolInvocation, renderPlanExecutionPrompt, renderInterruptedPlanContext, @@ -85,8 +84,8 @@ export interface AiSdkBackendFactoryDeps extends DesktopBackendToolSurfaceDeps { * seams that resolve AFTER the registration point are injected as accessors: * `getRuntime` (the SessionManager is constructed after registration) and * `getLookupPricing` (a mutable pricing lookup reassigned by usage IPC + startup; - * read live per `recordLlmCall`, snapshotted once for the `lookupPricing` field — - * matching the original module-`let` closure semantics exactly). + * snapshotted once for the `lookupPricing` field — matching the original + * module-`let` closure semantics exactly). */ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): BackendFactory { const { @@ -281,7 +280,6 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen }, shellRunContextSummary: ctx.shellRunContextSummary, lookupPricing: getLookupPricing(), - recordLlmCall: (event: LlmCallRecord) => recordLlmCall({ repo: telemetryRepo, lookupPricing: getLookupPricing() }, event), // One canonical record, one commit point (#1679): the AgentRun stream is // the only durable authority, and the ledger is a projection written only // after the authority holds the record. A failed projection marks the run diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index 720550d38d..6927ee5db8 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -501,15 +501,10 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom } }; const telemetry = { - insertLlmCall: (record: Parameters[0]) => - persistTelemetry(() => input.usage.telemetry.recordLlmCall(record)), insertToolInvocation: ( record: Parameters[0], ) => persistTelemetry(() => input.usage.telemetry.recordToolInvocation(record)), }; - const recordLlmUsage = (event: Parameters[1]) => { - void recordLlmCall({ repo: telemetry, lookupPricing: pricing }, event); - }; /** * One canonical record, one commit point (#1679). * @@ -658,7 +653,6 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom turnTailPrompt: modelComposition.turnTailPrompt, shellRunContextSummary: input.context.shellRunContextSummary, lookupPricing: pricing, - recordLlmCall: recordLlmUsage, recordModelCallAttempt, assertModelCallAccountingReady, recordToolInvocation: (event) => recordToolInvocation({ repo: telemetry }, event), diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 20ae02ab7c..ed8247bde8 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -20,7 +20,6 @@ import { projectRuntimeEventsToStoredMessages } from '../runtime-event-read-mode import { materializeSession } from '../materializer.js'; import type { InvocationContext } from '../invocation-context.js'; import type { AssistantMessage, StoredMessage, ToolResultMessage } from '@maka/core/session'; -import type { LlmCallRecord } from '@maka/core/usage-stats/types'; import { z } from 'zod'; import { AiSdkBackend, @@ -7486,7 +7485,6 @@ describe('AiSdkBackend usage telemetry', () => { test('records cumulative usage checkpoints across tool-loop steps and turns', async () => { const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const usageCheckpoints: Array<{ inputTokens: number; outputTokens: number }> = []; const firstTurn = durableTurnHarness('turn-1', 'hi'); const secondTurn = durableTurnHarness('turn-2', 'continue'); @@ -7571,9 +7569,6 @@ describe('AiSdkBackend usage telemetry', () => { : secondTurn.loadTurnRuntimeEvents(turnId), newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record: LlmCallRecord) => { - llmRecords.push(record); - }, recordUsageCheckpoint: async (usage: { inputTokens: number; outputTokens: number }) => { usageCheckpoints.push(usage); }, @@ -7728,7 +7723,6 @@ describe('AiSdkBackend usage telemetry', () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const largeBody = 'SECRET_PAYLOAD_SHOULD_BE_ARCHIVED'.repeat(200); let streamCalls = 0; const prompts: unknown[] = []; @@ -7820,9 +7814,6 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, }); for await (const event of backend.send(durable.input())) { @@ -7857,7 +7848,6 @@ describe('AiSdkBackend usage telemetry', () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const recordedBlocks: ActiveFullCompactBlock[] = []; const largeBody = 'ACTIVE_FULL_COMPACT_RAW_TOOL_OUTPUT'.repeat(200); let streamCalls = 0; @@ -7937,9 +7927,6 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, recordActiveFullCompactBlock: (block) => { recordedBlocks.push(block); }, @@ -8438,7 +8425,6 @@ describe('AiSdkBackend usage telemetry', () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const largeBody = 'VALIDATE_ONLY_RAW_TOOL_OUTPUT'.repeat(80); let streamCalls = 0; const model = new MockLanguageModelV4({ @@ -8512,9 +8498,6 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, }); for await (const event of backend.send(durable.input())) { @@ -8560,7 +8543,6 @@ describe('AiSdkBackend usage telemetry', () => { test('normalizes cache and reasoning tokens to messages, events, and telemetry', async () => { const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const runTraceEvents: Array<{ type: string; data?: Record }> = []; let pricingLookupCalls = 0; const pricing = { @@ -8620,9 +8602,6 @@ describe('AiSdkBackend usage telemetry', () => { pricingLookupCalls += 1; return modelKey === pricing.modelKey ? pricing : null; }, - recordLlmCall: (record) => { - llmRecords.push(record); - }, recordRunTrace: (event) => { runTraceEvents.push(event); }, diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 9c56c1bba0..c853c08dbc 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -267,7 +267,6 @@ export type { HistoryCompactWriter, HistoryCompactWriteInput, HistoryCompactWriteResult, - LlmTelemetryRecorder, SemanticCompactBlockRecorder, SynthesisCacheLoader, SynthesisCacheLoadInput, diff --git a/packages/runtime/src/ai-sdk-compaction-contract.ts b/packages/runtime/src/ai-sdk-compaction-contract.ts index 5c8c5bc813..3a09f77813 100644 --- a/packages/runtime/src/ai-sdk-compaction-contract.ts +++ b/packages/runtime/src/ai-sdk-compaction-contract.ts @@ -1,6 +1,5 @@ import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { LlmCallRecord } from '@maka/core/usage-stats/types'; import type { ModelCallAccountingInput } from './provider-request-telemetry.js'; import type { ActiveFullCompactBlock } from './active-full-compact.js'; @@ -17,8 +16,6 @@ import type { HistoryCompactCheckpoint } from './history-compact-checkpoint.js'; import type { ModelFactory } from './model-adapter.js'; import type { SemanticCompactBlock } from './semantic-compact.js'; -export type LlmTelemetryRecorder = (record: LlmCallRecord) => void; - export type ToolResultArchiveRecorderInput = ( | StaleToolResultArchiveCandidate | (ActiveToolResultArchiveCandidate & { runtimeEventId: string }) @@ -153,8 +150,6 @@ export interface AiSdkCompactionCapabilities { modelFactory: ModelFactory; /** Optional prior-history budget. Keeps whole turns to preserve tool-call/result pairs. */ contextBudget?: ContextBudgetPolicy; - /** Optional fire-and-forget LLM telemetry hook. */ - recordLlmCall?: LlmTelemetryRecorder; /** * Optional archive writer for replay-only stale tool-result pruning. The * runtime rewrites only candidates whose original body has been durably diff --git a/packages/runtime/src/history-compact-summarizer.ts b/packages/runtime/src/history-compact-summarizer.ts index d46d801840..13225792c5 100644 --- a/packages/runtime/src/history-compact-summarizer.ts +++ b/packages/runtime/src/history-compact-summarizer.ts @@ -1,18 +1,14 @@ import { rawFinishReasonString, type ModelMessage } from './model-protocol.js'; import { buildRuntimeEventModelReplayPlan } from './model-history.js'; import { toolResultOutput } from './tool-result-output.js'; -import type { - HistoryCompactSummaryInput, - LlmTelemetryRecorder, -} from './ai-sdk-compaction-contract.js'; +import type { HistoryCompactSummaryInput } from './ai-sdk-compaction-contract.js'; import { HistoryCompactSummarizerError } from './history-compact-error.js'; -import { normalizeAiSdkUsage, type AiSdkUsageLike } from './model-adapter.js'; +import type { AiSdkUsageLike } from './model-adapter.js'; import { ProviderRequestTracker, type ProviderGenerateResult, type ProviderRequestTrackerInput, } from './provider-request-telemetry.js'; -import { llmCallUsageFields } from './telemetry/llm-call-usage.js'; export { HistoryCompactSummarizerError } from './history-compact-error.js'; From c241a4b17005a97b35bbe776557dddb6b912c082 Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Sun, 2 Aug 2026 16:58:02 +0800 Subject: [PATCH 4/7] fix(metering): stop gating metering on the capture sink, and pin what it settles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1 on #1877. The P1 and the P2s that were about this PR's own behaviour, plus every P3. **Metering no longer depends on a diagnostic.** `createProviderRequestTracker` returned undefined without `recordProviderRequestCapture`, so a deployment with capture off — a reachable config; both hosts wire it conditionally — silently stopped metering compaction, which the deleted `recordLlmCall` had metered unconditionally. Capture is now optional inside the tracker: `preparedCapture` is pure, so `requestHash`, `requestBytes`, and `segments` survive without a sink, and only the artifact join keys (`captureId`, `captureArtifactId`) go absent. The tracker is built when there is either sink to feed. The Host now wires the summarizer's tracking unconditionally too, and the Desktop wires it at all — it never had, so the accounting the backend computed for a Desktop history compaction was handed to a summarizer that had nowhere to settle it. **The "one place owns attaching a tracker" claim is now true rather than asserted.** `withProviderGenerateTracking` is shared by `generateCompactSummary` and `buildLlmHistorySummarizer`; `ProviderMiddlewareGenerateInput` is declared once. Also fixed: history_compact records carried no `contextWindow` while semantic_compact did. **Tests for what was only claimed:** - `usageBasis: 'missing'` — the branch had no test anywhere. A call the provider reported no usage for records `missing`, not zero tokens, and stays unpriced whatever the resolver would have said. - Metering with capture switched off, asserting the attempt still carries the locally-computed request shape and no artifact ids. - Cache attribution end-to-end: the semantic mock now ships a `raw` provider payload, so the assertion moved from "absent because this mock claims nothing" to `cacheReadInputTokens: 2` through the provider branch. The no-`raw` rule stays pinned at the unit level. - A dry-run (`validate_only`) semantic compaction really is a billed call: the summarizer runs to completion and only then is its block refused. Confirmed by the test, which is why it is worth pinning — a mode named "dry run" that bills is what a later reader would assume otherwise. - Mid-turn history compaction settles a canonical record end-to-end through the backend glue, with the real summarizer against a mock provider, asserting the live `runId` resolves — a stubbed resolver cannot show that. P3 cleanups: two dead imports, and the Usage IPC comment that still claimed the frozen table receives compaction calls. `@maka/runtime` 2684/2696 (3 suites = documented local `rg` noise), `@maka/runtime-host` 524/524, `@maka/desktop` 1101/1139 (38 = this worktree's missing Astryx peer dep, unchanged from the parent commit). Co-Authored-By: Claude Opus 5 --- apps/desktop/src/main/session-stream.ts | 49 ++++-- apps/desktop/src/main/usage-ipc-main.ts | 8 +- .../src/server/execution-model-composition.ts | 28 +-- .../src/__tests__/ai-sdk-backend.test.ts | 161 +++++++++++++----- .../mid-turn-capacity-backend.test.ts | 79 +++++++++ .../provider-request-telemetry.test.ts | 72 +++++++- packages/runtime/src/ai-sdk-backend.ts | 10 +- packages/runtime/src/ai-sdk-compaction.ts | 2 +- .../runtime/src/history-compact-summarizer.ts | 27 +-- packages/runtime/src/model-adapter.ts | 27 +-- .../runtime/src/provider-request-telemetry.ts | 67 +++++++- 11 files changed, 406 insertions(+), 124 deletions(-) diff --git a/apps/desktop/src/main/session-stream.ts b/apps/desktop/src/main/session-stream.ts index a6661e36d6..56fde471ce 100644 --- a/apps/desktop/src/main/session-stream.ts +++ b/apps/desktop/src/main/session-stream.ts @@ -135,6 +135,26 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen mode: effectivePermissionMode, cwd: ctx.header.cwd, }); + // Hoisted so the auxiliary summarizer can share them with the send path: + // capture is optional plumbing, the context window is a property of the + // model both calls run against. + const providerRequestCapture = ctx.recordProviderRequestCapture + ? createProviderRequestCaptureRecorder({ + persistArtifact: async (capture) => { + const artifact = await persistProviderRequestCaptureArtifact(artifactStore, { + sessionId: ctx.sessionId, + turnId: capture.turnId, + captureId: capture.captureId, + step: capture.step, + serializedRequest: capture.serializedRequest, + now: Date.now(), + }); + return { artifactId: artifact.id }; + }, + recordLedger: ctx.recordProviderRequestCapture, + }) + : undefined; + const summarizerContextWindow = resolveSelectedModelContextWindow(connection, model); return new AiSdkBackend({ sessionId: ctx.sessionId, @@ -325,6 +345,18 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen resolveModel: () => getAIModel({ connection, apiKey: apiKey ?? '', modelId: model, fetch: modelFetch }), providerOptions: buildProviderOptions(connection, model, ctx.header.thinkingLevel), + // Without this the accounting the backend computes for a history + // compaction is handed to a summarizer that has nowhere to settle it, + // and the call goes unmetered. Capture joins in only when configured. + providerRequestTracking: { + now: Date.now, + newId: randomUUID, + ...(providerRequestCapture ? { persistCapture: providerRequestCapture } : {}), + recordAttempt: ctx.recordProviderRequestAttempt ?? (() => {}), + ...(summarizerContextWindow !== undefined + ? { contextWindow: summarizerContextWindow } + : {}), + }, }), loadSynthesisCache: (event) => loadSynthesisCacheBlocksFromArtifacts(artifactStore, event), writeSynthesisCache: (event) => persistSynthesisCacheBlocksToArtifacts(artifactStore, event, { @@ -338,22 +370,9 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen }, }), recordRunTrace: ctx.recordRunTrace, - ...(ctx.recordProviderRequestCapture + ...(providerRequestCapture ? { - recordProviderRequestCapture: createProviderRequestCaptureRecorder({ - persistArtifact: async (capture) => { - const artifact = await persistProviderRequestCaptureArtifact(artifactStore, { - sessionId: ctx.sessionId, - turnId: capture.turnId, - captureId: capture.captureId, - step: capture.step, - serializedRequest: capture.serializedRequest, - now: Date.now(), - }); - return { artifactId: artifact.id }; - }, - recordLedger: ctx.recordProviderRequestCapture, - }), + recordProviderRequestCapture: providerRequestCapture, recordProviderRequestAttempt: ctx.recordProviderRequestAttempt, } : {}), diff --git a/apps/desktop/src/main/usage-ipc-main.ts b/apps/desktop/src/main/usage-ipc-main.ts index 45a0bdf67c..acc6fff792 100644 --- a/apps/desktop/src/main/usage-ipc-main.ts +++ b/apps/desktop/src/main/usage-ipc-main.ts @@ -42,9 +42,11 @@ const USAGE_REPAIR_RUNS_PER_QUERY = 16; export function registerUsageIpc(deps: UsageIpcDeps): void { /** * Usage answers sum two sources (#1679): the canonical model-call ledger and - * the frozen `LlmCallRecord` table, which still receives the compaction calls - * that have not been routed through the canonical seam yet. Every merged - * result carries the provenance that qualifies it. + * the frozen `LlmCallRecord` table. Both compaction kinds now settle through + * the canonical seam; what still lands in the frozen table is historical rows + * and `goal_evaluation`, the one kind the seam cannot yet identify (it has no + * run or turn at the Host layer). Every merged result carries the provenance + * that qualifies it. */ const canonicalUsage = async (query: UsageQuery, now: number): Promise => { // Fold in whatever the authority holds and this read model is behind on diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index 6927ee5db8..4d90c7ee74 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -554,6 +554,12 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom } }; let artifactDrainRequested = false; + // The summarizer runs on the session's own connection and model, so its + // attempts are measured against the same window the send is. + const summarizerContextWindow = resolveSelectedModelContextWindow( + target.connection, + target.model, + ); const providerRequestCapture = input.context.recordProviderRequestCapture ? createProviderRequestCaptureRecorder({ persistArtifact: async (capture) => { @@ -632,16 +638,18 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom modelId: target.model, }), providerOptions, - ...(providerRequestCapture - ? { - providerRequestTracking: { - now: Date.now, - newId: randomUUID, - persistCapture: providerRequestCapture, - recordAttempt: recordProviderRequestAttempt, - }, - } - : {}), + // Wired unconditionally: this is what carries the summarization's + // accounting, and metering must not depend on the capture sink being + // configured. Capture joins in only when it is. + providerRequestTracking: { + now: Date.now, + newId: randomUUID, + ...(providerRequestCapture ? { persistCapture: providerRequestCapture } : {}), + recordAttempt: recordProviderRequestAttempt, + ...(summarizerContextWindow !== undefined + ? { contextWindow: summarizerContextWindow } + : {}), + }, }), recordHistoryCompactCheckpoint: input.context.recordHistoryCompactCheckpoint, loadTurnRuntimeEvents: input.context.loadTurnRuntimeEvents, diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index ed8247bde8..33993d5d75 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -8031,16 +8031,16 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(recordedBlocks[0]?.blockId, 'afcompact-sync-test'); }); - test('semantic compact records a separate no-tools summarizer model call', async () => { - const durable = durableTurnHarness('turn-1', 'hi'); - const messages: unknown[] = []; - const events: SessionEvent[] = []; - const modelCalls: ModelCallAttempt[] = []; - const recordedBlocks: SemanticCompactBlock[] = []; - const recordedActiveFullBlocks: ActiveFullCompactBlock[] = []; - const largeBody = 'SEMANTIC_COMPACT_RAW_TOOL_OUTPUT'.repeat(180); + /** + * One model whose stream drives a turn into semantic compaction: two tool + * steps, then a plain finish. Shared by the accept and dry-run cases so both + * exercise the same summarization, and only the policy differs. + */ + function semanticCompactFixtureModel(): { + model: MockLanguageModelV4; + streamCalls: () => number; + } { let streamCalls = 0; - let archiveCalls = 0; const model = new MockLanguageModelV4({ doGenerate: { content: [ @@ -8059,6 +8059,14 @@ describe('AiSdkBackend usage telemetry', () => { usage: { inputTokens: { total: 21, noCache: 19, cacheRead: 2, cacheWrite: 0 }, outputTokens: { total: 13, text: 13, reasoning: 0 }, + // The provider's own payload, which is the only thing the canonical + // record will attribute cache tokens from. + raw: { + input_tokens: 19, + output_tokens: 13, + cache_read_input_tokens: 2, + cache_creation_input_tokens: 0, + }, }, warnings: [], }, @@ -8117,6 +8125,49 @@ describe('AiSdkBackend usage telemetry', () => { }; }, }); + return { model, streamCalls: () => streamCalls }; + } + + function semanticCompactContextBudget(mode: 'replace' | 'validate_only') { + return { + charsPerToken: 1, + activeToolResultPrune: { enabled: true, maxCurrentResultEstimatedTokens: 1 }, + semanticCompact: { + enabled: true, + mode, + minStepNumber: 1, + minRecentMessages: 0, + maxActiveEstimatedTokens: 1, + highWaterRatio: 0.1, + minSafePrefixEstimatedTokens: 1, + minNewPrefixEstimatedTokens: 1, + maxSummaryEstimatedTokens: 1024, + minSavingsTokens: 1, + minSavingsRatio: 0, + }, + activeFullCompact: { + enabled: true, + minStepNumber: 1, + maxActiveEstimatedTokens: 1_000_000, + highWaterRatio: 0.1, + minRecentMessages: 0, + maxSummaryEstimatedTokens: 1024, + }, + } as const; + } + + const SEMANTIC_COMPACT_LARGE_BODY = 'SEMANTIC_COMPACT_RAW_TOOL_OUTPUT'.repeat(180); + + test('semantic compact records a separate no-tools summarizer model call', async () => { + const durable = durableTurnHarness('turn-1', 'hi'); + const messages: unknown[] = []; + const events: SessionEvent[] = []; + const modelCalls: ModelCallAttempt[] = []; + const recordedBlocks: SemanticCompactBlock[] = []; + const recordedActiveFullBlocks: ActiveFullCompactBlock[] = []; + const largeBody = SEMANTIC_COMPACT_LARGE_BODY; + let archiveCalls = 0; + const { model, streamCalls } = semanticCompactFixtureModel(); const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), @@ -8137,31 +8188,7 @@ describe('AiSdkBackend usage telemetry', () => { }), }, ], - contextBudget: { - charsPerToken: 1, - activeToolResultPrune: { enabled: true, maxCurrentResultEstimatedTokens: 1 }, - semanticCompact: { - enabled: true, - mode: 'replace', - minStepNumber: 1, - minRecentMessages: 0, - maxActiveEstimatedTokens: 1, - highWaterRatio: 0.1, - minSafePrefixEstimatedTokens: 1, - minNewPrefixEstimatedTokens: 1, - maxSummaryEstimatedTokens: 1024, - minSavingsTokens: 1, - minSavingsRatio: 0, - }, - activeFullCompact: { - enabled: true, - minStepNumber: 1, - maxActiveEstimatedTokens: 1_000_000, - highWaterRatio: 0.1, - minRecentMessages: 0, - maxSummaryEstimatedTokens: 1024, - }, - }, + contextBudget: semanticCompactContextBudget('replace'), archiveToolResult: async () => { archiveCalls += 1; return { artifactId: 'archived-covered-semantic-result' }; @@ -8188,7 +8215,7 @@ describe('AiSdkBackend usage telemetry', () => { events.push(event); } - assert.equal(streamCalls, 3); + assert.equal(streamCalls(), 3); assert.equal(model.doGenerateCalls.length, 1); assert.match( JSON.stringify(model.doGenerateCalls[0]?.prompt), @@ -8271,10 +8298,9 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(semanticAttempt.inputTokens, 21); assert.equal(semanticAttempt.outputTokens, 13); assert.equal(semanticAttempt.usageBasis, 'reported'); - // The old row copied the SDK's normalized `cacheRead` through as a cache - // hit. The canonical record only attributes cache tokens the provider's own - // payload claims, and this mock ships none — so absent, not zero. - assert.equal(semanticAttempt.cacheReadInputTokens, undefined); + // Cache tokens are attributed from the provider's own payload, never from + // the SDK's normalized view — which is what the old row copied through. + assert.equal(semanticAttempt.cacheReadInputTokens, 2); assert.equal( modelCalls.filter((attempt) => attempt.callKind === 'semantic_compact').length, 1, @@ -8300,6 +8326,63 @@ describe('AiSdkBackend usage telemetry', () => { ); }); + test('a dry-run semantic compaction is still a real, billed model call', async () => { + // `validate_only` declines the *projection*, not the summarization: the + // summarizer runs to completion and only then is its block refused. That + // call reaches a provider and is charged for, so it settles a canonical + // record like any other. Pinned because a mode named "dry run" that bills + // is exactly the kind of thing a later reader would assume otherwise. + const durable = durableTurnHarness('turn-1', 'hi'); + const modelCalls: ModelCallAttempt[] = []; + const recordedBlocks: SemanticCompactBlock[] = []; + const { model } = semanticCompactFixtureModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [ + { + name: 'Read', + description: 'Read description', + parameters: z.object({ path: z.string() }), + impl: async ({ path }) => ({ + body: path === 'large.log' ? SEMANTIC_COMPACT_LARGE_BODY : 'FRESH_SEMANTIC_TAIL_RESULT', + }), + }, + ], + contextBudget: semanticCompactContextBudget('validate_only'), + archiveToolResult: async () => ({ artifactId: 'archived-covered-semantic-result' }), + loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, + newId: idGenerator(), + now: monotonicClock(), + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-semantic-capture' }), + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); + }, + recordSemanticCompactBlock: (block) => { + recordedBlocks.push(block); + }, + }); + + for await (const event of backend.send(durable.input({ runId: 'run-1' }))) { + durable.record(event); + } + + assert.equal(model.doGenerateCalls.length, 1, 'the dry run still calls the summarizer'); + assert.equal(recordedBlocks.length, 0, 'and still refuses to accept the block it produced'); + const dryRunAttempt = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'semantic_compact'); + assert.ok(dryRunAttempt, 'a declined projection does not make the call free'); + assert.equal(dryRunAttempt.status, 'completed'); + assert.equal(dryRunAttempt.inputTokens, 21); + assert.equal(dryRunAttempt.outputTokens, 13); + }); + test('active full compact keeps the accepted boundary projection across later AI SDK steps', async () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index b43fef64c1..50712dde00 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -21,6 +21,8 @@ import { evaluateHistoryCompactCheckpointReplay } from '../history-compact.js'; import type { HistoryCompactCheckpoint } from '../history-compact-checkpoint.js'; import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; import { HistoryCompactSummarizerError } from '../history-compact-error.js'; +import { buildLlmHistorySummarizer } from '../history-compact-summarizer.js'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; const RAW_SPAN_ONE = 'RAW_SPAN_ONE_'.repeat(24); @@ -41,6 +43,8 @@ interface MidTurnFixture { anchor: RuntimeEvent; /** The fixture's durable RuntimeEvent ledger for the current turn/run. */ ledger: RuntimeEvent[]; + /** Canonical accounting records settled during the turn (#1679). */ + modelCalls: ModelCallAttempt[]; ledgerReads: number; events: SessionEvent[]; messages: unknown[]; @@ -75,6 +79,12 @@ interface MidTurnFixtureOptions { activeToolResultPrune?: boolean; /** Enable semantic compaction so it competes with the capacity hook. */ semanticCompact?: boolean; + /** + * Summarize through the real `buildLlmHistorySummarizer` against a mock + * provider, so the compaction settles a canonical record instead of the + * stubbed string the other cases return. + */ + meteredSummarizer?: boolean; /** Override the checkpoint recorder (e.g. to simulate a write failure). */ record?: (checkpoint: HistoryCompactCheckpoint) => void; /** Make the prior turns large so folding them rescues an over-window turn. */ @@ -242,6 +252,30 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { ledger.push(mapped); }; + const modelCalls: ModelCallAttempt[] = []; + const summarizerModel = new MockLanguageModelV4({ + doGenerate: { + content: [{ type: 'text', text: 'MID_TURN_SUMMARY_SENTINEL' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 31, noCache: 31, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 7, text: 7, reasoning: 0 }, + raw: { input_tokens: 31, output_tokens: 7 }, + }, + warnings: [], + }, + }); + let summarizerIds = 0; + const meteredSummarize = buildLlmHistorySummarizer({ + resolveModel: () => summarizerModel, + providerRequestTracking: { + now: () => 5_000, + newId: () => `summarizer-id-${++summarizerIds}`, + persistCapture: async () => ({ artifactId: 'artifact-mid-turn-capture' }), + recordAttempt: () => {}, + }, + }); + const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), @@ -333,9 +367,20 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { summarizeHistoryCompact: async (input) => { fixture.summarizerCalls += 1; summarizedSources.push(JSON.stringify(input.source.foldedRuntimeEvents)); + // The real summarizer is what carries the accounting the backend hands + // it, so the metered case must go through it rather than a stub. + if (options.meteredSummarizer) return await meteredSummarize(input); const summary = options.summarize ? await options.summarize() : 'MID_TURN_SUMMARY_SENTINEL'; return summary; }, + ...(options.meteredSummarizer + ? { + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-mid-turn-capture' }), + recordModelCallAttempt: (attempt: ModelCallAttempt) => { + modelCalls.push(attempt); + }, + } + : {}), recordHistoryCompactCheckpoint: (checkpoint) => { if (options.record) return options.record(checkpoint); recorded.push(checkpoint); @@ -378,6 +423,7 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { priorEvents, anchor, ledger, + modelCalls, events, messages, llmCalls, @@ -487,6 +533,39 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { assert.equal(fit.fits, true); }); + test('a mid-turn compaction settles a canonical record for the run it interrupts', async () => { + // End-to-end over the backend glue, not the summarizer in isolation: the + // accounting identity is built inside `AiSdkCompaction` from the backend's + // live `runId`, which only exists while a send is in flight. A stubbed + // resolver in a unit test cannot show that the real one resolves. + const fixture = buildFixture({ meteredSummarizer: true }); + await runFixtureTurn(fixture, consumer); + + assert.equal(fixture.summarizerCalls, 1); + const attempts = fixture.modelCalls.map((attempt) => decodeModelCallAttempt(attempt)); + const compaction = attempts.find((attempt) => attempt.callKind === 'history_compact'); + assert.ok(compaction, 'the compaction call must settle its own canonical record'); + assert.equal(compaction.sessionId, 'session-1'); + assert.equal(compaction.runId, 'run-1', 'attributed to the run the send is executing'); + assert.equal(compaction.turnId, 'turn-1'); + assert.equal(compaction.status, 'completed'); + assert.equal(compaction.usageBasis, 'reported'); + assert.equal(compaction.inputTokens, 31); + assert.equal(compaction.outputTokens, 7); + // The send's own steps meter separately: one auxiliary call is not folded + // into the turn it interrupts. + assert.equal( + attempts.filter((attempt) => attempt.callKind === 'history_compact').length, + 1, + 'one summarization is one record', + ); + assert.equal( + attempts.some((attempt) => attempt.callKind === 'main'), + true, + 'and the send it interrupts still records its own', + ); + }); + test('recovery re-projection with ctx.branch replays the checkpoint without the raw span', async () => { const fixture = buildFixture({ branch: 'lane-7' }); await runFixtureTurn(fixture, consumer); diff --git a/packages/runtime/src/__tests__/provider-request-telemetry.test.ts b/packages/runtime/src/__tests__/provider-request-telemetry.test.ts index 44ade3b870..9f2d42892d 100644 --- a/packages/runtime/src/__tests__/provider-request-telemetry.test.ts +++ b/packages/runtime/src/__tests__/provider-request-telemetry.test.ts @@ -719,6 +719,9 @@ describe('canonical model-call accounting', () => { resolveCost?: telemetry.ModelCallAccountingInput['resolveCost']; assertReady?: () => void; resolveRunId?: () => string | undefined; + /** Models a deployment with request capture switched off. */ + withoutCapture?: boolean; + recordAttempt?: (attempt: telemetry.ProviderRequestAttemptRecord) => void; }): telemetry.ProviderRequestTracker { let n = 0; return new telemetry.ProviderRequestTracker({ @@ -726,8 +729,10 @@ describe('canonical model-call accounting', () => { turnId: 'turn-1', now: () => 1_000 + n, newId: () => `id-${++n}`, - persistCapture: async () => ({ artifactId: 'artifact-1' }), - recordAttempt: () => {}, + ...(overrides.withoutCapture + ? {} + : { persistCapture: async () => ({ artifactId: 'artifact-1' }) }), + recordAttempt: overrides.recordAttempt ?? (() => {}), accounting: { sessionId: 'session-1', resolveRunId: overrides.resolveRunId ?? (() => 'run-1'), @@ -770,6 +775,69 @@ describe('canonical model-call accounting', () => { assert.equal(attempt.attempt, 0); }); + test('a call the provider reported no usage for records usageBasis missing', async () => { + // The alternative is a record claiming zero tokens, which is a measurement + // nobody made. `missing` says the call happened and the meter did not read. + const recorded: ModelCallAttempt[] = []; + const tracker = accountingTracker({ + record: (a) => { + recorded.push(a); + }, + resolveCost: () => ({ costUsd: 0.002, pricingRevision: 4 }), + }); + + await tracker.trackGenerate({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('hello'), + doGenerate: async () => ({ finishReason: 'stop' }), + }); + + const attempt = decodeModelCallAttempt(recorded[0]); + assert.equal(attempt.status, 'completed'); + assert.equal(attempt.usageBasis, 'missing'); + assert.equal(attempt.inputTokens, undefined); + assert.equal(attempt.outputTokens, undefined); + // No usage means nothing to price, whatever the resolver would have said. + assert.equal(attempt.costBasis, 'unpriced'); + assert.equal(attempt.costUsd, undefined); + }); + + test('metering survives a deployment with request capture switched off', async () => { + // Capture is a diagnostic. A record that cannot be joined to a stored + // request body is still a record of a call that really was billed, so the + // canonical seam must not be gated on the capture sink being configured. + const recorded: ModelCallAttempt[] = []; + const attempts: telemetry.ProviderRequestAttemptRecord[] = []; + const tracker = accountingTracker({ + withoutCapture: true, + record: (a) => { + recorded.push(a); + }, + recordAttempt: (a) => { + attempts.push(a); + }, + }); + + const result = await tracker.trackStream({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('hello'), + doStream: async () => ({ stream: streamOf([finishPart()]) }), + }); + await drain(result.stream); + + const attempt = decodeModelCallAttempt(recorded[0]); + assert.equal(attempt.usageBasis, 'reported'); + assert.equal(attempt.captureArtifactId, undefined, 'there is no artifact to point at'); + // The request shape is computed locally, so it does not need the sink. + assert.equal(attempts.length, 1); + assert.equal(attempts[0]?.captureId, undefined); + assert.equal(attempts[0]?.captureArtifactId, undefined); + assert.ok((attempts[0]?.requestHash?.length ?? 0) > 0); + assert.ok((attempts[0]?.requestBytes ?? 0) > 0); + }); + test('an unresolvable price records unpriced rather than zero', async () => { const recorded: ModelCallAttempt[] = []; const tracker = accountingTracker({ diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index c853c08dbc..9899235dbb 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -92,7 +92,6 @@ import type { UserContent, } from './model-protocol.js'; import { z } from 'zod'; -import { llmCallUsageFields } from './telemetry/llm-call-usage.js'; import { AsyncEventQueue } from './async-queue.js'; import { StreamWatchdog, formatStreamWatchdogError } from './stream-watchdog.js'; @@ -2047,8 +2046,9 @@ export class AiSdkBackend implements AgentBackend { * this backend. Callers receive a ready tracker rather than the ingredients: * a half-wired tracker is what produces records nothing can attribute. * - * Absent when capture is not wired, which leaves the call untracked exactly - * as it was before. + * Absent only when there is nothing to feed: no capture sink *and* no + * canonical sink. Metering deliberately does not depend on capture — capture + * is a diagnostic, and a deployment that turns it off must still be billed. */ private createProviderRequestTracker(input: { turnId: string; @@ -2056,15 +2056,15 @@ export class AiSdkBackend implements AgentBackend { modelId: string; }): ProviderRequestTracker | undefined { const persistCapture = this.input.recordProviderRequestCapture; - if (!persistCapture) return undefined; const accounting = this.modelCallAccounting(input.callKind); + if (!persistCapture && !accounting) return undefined; return new ProviderRequestTracker({ traceId: this.newId(), turnId: input.turnId, contextWindow: resolveSelectedModelContextWindow(this.input.connection, input.modelId), now: this.now, newId: this.newId, - persistCapture, + ...(persistCapture ? { persistCapture } : {}), recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), ...(accounting ? { accounting } : {}), }); diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 4039dfa1df..4ef96ea463 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -49,7 +49,7 @@ import { import { createHash } from 'node:crypto'; import type { ModelMessage } from './model-protocol.js'; -import { normalizeAiSdkUsage, type ModelAdapter } from './model-adapter.js'; +import type { ModelAdapter } from './model-adapter.js'; import type { RequestProjection, RequestProjectionContext, diff --git a/packages/runtime/src/history-compact-summarizer.ts b/packages/runtime/src/history-compact-summarizer.ts index 13225792c5..7d6c19cc67 100644 --- a/packages/runtime/src/history-compact-summarizer.ts +++ b/packages/runtime/src/history-compact-summarizer.ts @@ -6,7 +6,7 @@ import { HistoryCompactSummarizerError } from './history-compact-error.js'; import type { AiSdkUsageLike } from './model-adapter.js'; import { ProviderRequestTracker, - type ProviderGenerateResult, + withProviderGenerateTracking, type ProviderRequestTrackerInput, } from './provider-request-telemetry.js'; @@ -25,12 +25,6 @@ export type AiSdkGenerateTextLike = ( options: AiSdkGenerateTextOptions, ) => Promise<{ text: string; finishReason?: unknown; usage?: AiSdkUsageLike }>; -interface ProviderMiddlewareGenerateInput { - doGenerate: () => PromiseLike; - params: Record & { abortSignal?: AbortSignal }; - model: { provider: string; modelId: string }; -} - export interface BuildLlmHistorySummarizerOptions { /** Resolve the AI SDK model used for summarization. Reuses the session model. */ resolveModel: () => unknown; @@ -108,22 +102,11 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti options.generateText && !providerRequestTracker ? undefined : await loadAiSdkTextModule(); const generateText = options.generateText ?? ai!.generateText; const model = providerRequestTracker - ? ai!.wrapLanguageModel({ + ? withProviderGenerateTracking({ model: options.resolveModel(), - middleware: { - wrapGenerate: async ({ - doGenerate, - params, - model: providerModel, - }: ProviderMiddlewareGenerateInput) => - await providerRequestTracker.trackGenerate({ - providerId: providerModel.provider, - modelId: providerModel.modelId, - params, - abortSignal: input.abortSignal, - doGenerate, - }), - }, + wrapLanguageModel: ai!.wrapLanguageModel, + tracker: providerRequestTracker, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), }) : options.resolveModel(); const result = await generateText({ diff --git a/packages/runtime/src/model-adapter.ts b/packages/runtime/src/model-adapter.ts index 48e5c0f6e4..c1ba117607 100644 --- a/packages/runtime/src/model-adapter.ts +++ b/packages/runtime/src/model-adapter.ts @@ -38,9 +38,9 @@ import { errorPresentationFromClass, providerRetryMetadata, } from './provider-error-classification.js'; -import type { - ProviderGenerateResult, - ProviderRequestTracker, +import { + withProviderGenerateTracking, + type ProviderRequestTracker, } from './provider-request-telemetry.js'; import { createKimiOpenAiTransportState, @@ -129,12 +129,6 @@ interface ProviderMiddlewareStreamInput { model: { provider: string; modelId: string }; } -interface ProviderMiddlewareGenerateInput { - doGenerate: () => PromiseLike; - params: Record & { abortSignal?: AbortSignal }; - model: { provider: string; modelId: string }; -} - export class ModelAdapter { private readonly kimiOpenAiTransportState = createKimiOpenAiTransportState(); @@ -284,18 +278,11 @@ export class ModelAdapter { }; const trackedModel = input.providerRequestTracker - ? wrapLanguageModel({ + ? withProviderGenerateTracking({ model: input.model, - middleware: { - wrapGenerate: async ({ doGenerate, params, model }: ProviderMiddlewareGenerateInput) => - await input.providerRequestTracker!.trackGenerate({ - providerId: model.provider, - modelId: model.modelId, - params, - ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), - doGenerate, - }), - }, + wrapLanguageModel, + tracker: input.providerRequestTracker, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), }) : input.model; diff --git a/packages/runtime/src/provider-request-telemetry.ts b/packages/runtime/src/provider-request-telemetry.ts index b20f937914..2e4d0024ac 100644 --- a/packages/runtime/src/provider-request-telemetry.ts +++ b/packages/runtime/src/provider-request-telemetry.ts @@ -63,8 +63,13 @@ export interface ProviderRequestAttemptRecord extends ProviderRequestUsage { turnId: string; step: number; attempt: number; - captureId: string; - captureArtifactId: string; + /** + * Present only when a capture sink is wired. The request shape below is + * computed locally and always present; these two are the join keys to the + * persisted artifact, so they are absent when there is nothing to join to. + */ + captureId?: string; + captureArtifactId?: string; providerId: string; modelId: string; contextWindow?: number; @@ -96,7 +101,12 @@ export interface ProviderRequestTrackerInput { contextWindow?: number; now: () => number; newId: () => string; - persistCapture: ( + /** + * Request-body capture sink. Optional because capture is a diagnostic, and + * metering must not depend on one: a deployment with capture switched off + * still settles canonical records, it just has no artifact to join them to. + */ + persistCapture?: ( capture: ProviderRequestCaptureRecord, ) => Promise>; recordAttempt: (attempt: ProviderRequestAttemptRecord) => void | Promise; @@ -163,6 +173,42 @@ export interface TrackProviderGenerateInput { doGenerate: () => PromiseLike; } +interface ProviderMiddlewareGenerateInput { + doGenerate: () => PromiseLike; + params: Record & { abortSignal?: AbortSignal }; + model: { provider: string; modelId: string }; +} + +/** + * Wraps a language model so its single `generate` call is tracked. + * + * The AI SDK's `wrapLanguageModel` is passed in rather than imported: both + * callers load the `ai` package dynamically, and there is no reason for this + * module to load it a third time. Non-streaming counterpart of what + * `ModelAdapter.startStream` does with `wrapStream`, and the only place either + * auxiliary caller attaches a tracker to a model. + */ +export function withProviderGenerateTracking(input: { + model: unknown; + wrapLanguageModel: (input: Record) => unknown; + tracker: ProviderRequestTracker; + abortSignal?: AbortSignal; +}): unknown { + return input.wrapLanguageModel({ + model: input.model, + middleware: { + wrapGenerate: async ({ doGenerate, params, model }: ProviderMiddlewareGenerateInput) => + await input.tracker.trackGenerate({ + providerId: model.provider, + modelId: model.modelId, + params, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), + doGenerate, + }), + }, + }); +} + export function createProviderRequestCaptureRecorder( input: ProviderRequestCaptureRecorderInput, ): ( @@ -192,7 +238,8 @@ export interface ProviderGenerateResult { interface StoredCapture { capture: ProviderRequestCaptureRecord; - ref: ProviderRequestCaptureRef; + /** Absent when no capture sink is wired: there is no artifact to point at. */ + ref?: ProviderRequestCaptureRef; } const CANONICAL_USAGE_FIELDS = [ @@ -388,8 +435,9 @@ export class ProviderRequestTracker { turnId: this.input.turnId, step, attempt, - captureId: capture.ref.captureId, - captureArtifactId: capture.ref.artifactId, + ...(capture.ref + ? { captureId: capture.ref.captureId, captureArtifactId: capture.ref.artifactId } + : {}), providerId: input.providerId, modelId: input.modelId, ...(contextWindow !== undefined ? { contextWindow } : {}), @@ -518,6 +566,7 @@ export class ProviderRequestTracker { const existing = this.captures.get(key); if (existing) return await existing; + const persistCapture = this.input.persistCapture; const pending = (async (): Promise => { const captureId = this.input.newId(); const capture: ProviderRequestCaptureRecord = { @@ -529,7 +578,11 @@ export class ProviderRequestTracker { providerId: input.providerId, modelId: input.modelId, }; - const persisted = await this.input.persistCapture(capture); + // The request shape on `capture` is computed here and needs no sink. Only + // the artifact join keys depend on one, so without it the attempt still + // carries hash, bytes, and segments — it just points at nothing. + if (!persistCapture) return { capture }; + const persisted = await persistCapture(capture); return { capture, ref: { captureId, artifactId: persisted.artifactId } }; })(); this.captures.set(key, pending); From fb03fa5ceb5df18e2c9354b08105e05663004407 Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Sun, 2 Aug 2026 17:02:30 +0800 Subject: [PATCH 5/7] fix(headless): narrow the trace analyzer to optional capture ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fallout from making capture optional on `ProviderRequestAttemptRecord`: the headless provider-request trace analyzer joins attempts to captures by id, and the id is no longer guaranteed to be there. Behaviour is unchanged for every trace this tool actually reads. It analyses a capture ledger, so an attempt that cannot name a capture is incomplete by its own definition and fails with the same diagnostic it already produced — and the decoder above it already rejects such records as `invalid_attempt` before the join runs. Only the type needed narrowing. Caught by CI, not by me: I checked the blast radius of the optional fields in `core`, `storage`, `runtime-host`, and `desktop`, and did not grep `headless`. Co-Authored-By: Claude Opus 5 --- packages/headless/src/provider-request-trace.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/headless/src/provider-request-trace.ts b/packages/headless/src/provider-request-trace.ts index 08ea63d7bf..366347b381 100644 --- a/packages/headless/src/provider-request-trace.ts +++ b/packages/headless/src/provider-request-trace.ts @@ -271,8 +271,12 @@ export function assertProviderRequestTraceComplete( if (!identities.some((candidate) => candidate.turnId === attempt.turnId)) { fail(`attempt ${attempt.attemptId} has another turn id`); } + // Capture ids are optional on the record since #1679 — an attempt made in a + // deployment with capture switched off has none. This analysis reads a + // capture ledger, so an attempt that cannot name one is incomplete here; + // the decoder above already rejects such records as invalid_attempt. const capture = - captures.get(attempt.captureId) ?? + (attempt.captureId !== undefined ? captures.get(attempt.captureId) : undefined) ?? fail(`attempt ${attempt.attemptId} does not match its request capture`); if (!attemptMatchesCapture(attempt, capture)) { fail(`attempt ${attempt.attemptId} does not match its request capture`); From bfdf9586e502eb61ed0382bb4b553d3ebd58c7c2 Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Mon, 3 Aug 2026 00:43:58 +0800 Subject: [PATCH 6/7] fix(metering): name the run for a manual compaction, and price each call as its own model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 2 on #1877. Both findings were correct, and the first was a live bug I had argued my way out of. **Manual compaction was unmetered, and my "nothing reaches this path" was wrong.** Desktop `sessions:compact` and CLI `/compact` both reach `RuntimeKernel.compactSession()`, which opens an `AgentRun` and then called `compactHistory` without its id. Outside `send()` there is no `currentRunId`, so the summarization settled nothing while the old `recordLlmCall` had metered it. `BackendCompactHistoryInput.runId` is now **required**, not optional: the failure mode is silence, and an optional field is one a caller can forget in exactly the situation that produced this bug. The kernel passes `run.runId`; the compaction threads it into the accounting identity for that one call. Seventeen test call sites had to name a run, which is the type doing its job. **A configured summarizer model was priced as the session model.** Cost resolution looked up `${providerType}:${this.input.modelId}` regardless of which model served the request, so with `MAKA_CONTEXT_SEMANTIC_COMPACT_MODEL` set we stored one model's id beside another model's `pricingRates` — precisely what recording the rates exists to prevent. `resolveModelCallCost` now takes the call's model id, supplied through the same identity the tracker already carries. **One bug the new test caught in my own fix:** the compaction dep was wired as `(callKind) => this.modelCallAccounting(callKind)`, which silently dropped the new identity argument. TypeScript accepts a narrower function, so the manual compaction kept recording nothing and the types stayed green. Only the end-to-end assertion showed it. Tests, both as asked: `SessionManager.compactSession()` settles exactly one `history_compact` record carrying the run the kernel opened, driven through a real `AiSdkBackend` and a real summarizer; and a semantic compaction with a distinct summarizer model records that model's own rates while the send's own steps keep the session model's. `@maka/runtime` 2686/2698 (3 suites = documented local `rg` noise), `@maka/runtime-host` 524/524, `@maka/desktop` 1101/1139 (38 = this worktree's missing Astryx peer dep). All runtime-consuming packages build. Co-Authored-By: Claude Opus 5 --- packages/core/src/backend-types.ts | 7 ++ .../src/__tests__/ai-sdk-backend.test.ts | 114 ++++++++++++++++- .../src/__tests__/session-manager.test.ts | 117 ++++++++++++++++++ packages/runtime/src/ai-sdk-backend.ts | 39 ++++-- packages/runtime/src/ai-sdk-compaction.ts | 19 ++- packages/runtime/src/runtime-kernel.ts | 1 + 6 files changed, 287 insertions(+), 10 deletions(-) diff --git a/packages/core/src/backend-types.ts b/packages/core/src/backend-types.ts index 0f26b7b0ef..88020661c4 100644 --- a/packages/core/src/backend-types.ts +++ b/packages/core/src/backend-types.ts @@ -129,6 +129,13 @@ export interface SteeringLease { export interface BackendCompactHistoryInput { turnId: string; + /** + * The run this compaction belongs to. Required, not optional: a manual + * compaction is a real model call, and a call whose run cannot be named is a + * call nothing can bill (#1679). Unlike `send`, this path has no turn state to + * infer it from, so the caller that opened the run states it. + */ + runId: string; runtimeContext: readonly RuntimeEvent[]; /** Override the configured recent-turn tail for an explicit recovery compaction. */ minRecentTurns?: number; diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 33993d5d75..bf76ad04f8 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -3979,6 +3979,7 @@ describe('AiSdkBackend model history', () => { const runtimeContext = [...oldEvents, recentEvent]; const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext, }); @@ -3995,6 +3996,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-overflow-recovery', + runId: 'run-1', runtimeContext, minRecentTurns: 0, }); @@ -4040,6 +4042,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'default-policy-manual-old-1', @@ -4097,6 +4100,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'manual-v2-old-1', @@ -4186,6 +4190,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...oldEvents, runtimeTextEvent({ @@ -4263,6 +4268,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...oldEvents, runtimeTextEvent({ @@ -4345,6 +4351,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...oldEvents, runtimeTextEvent({ @@ -4391,6 +4398,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'manual-v2-envelope-old-1', @@ -4448,6 +4456,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'manual-v2-larger-old-1', @@ -4509,6 +4518,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'output-length-old', @@ -4600,6 +4610,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...covered, runtimeTextEvent({ @@ -4646,6 +4657,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4690,6 +4702,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4759,6 +4772,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: oldEvents, }); @@ -4819,6 +4833,7 @@ describe('AiSdkBackend model history', () => { const compactPromise = backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4904,6 +4919,7 @@ describe('AiSdkBackend model history', () => { const compactPromise = backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4982,6 +4998,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -8036,12 +8053,13 @@ describe('AiSdkBackend usage telemetry', () => { * steps, then a plain finish. Shared by the accept and dry-run cases so both * exercise the same summarization, and only the policy differs. */ - function semanticCompactFixtureModel(): { + function semanticCompactFixtureModel(modelId?: string): { model: MockLanguageModelV4; streamCalls: () => number; } { let streamCalls = 0; const model = new MockLanguageModelV4({ + ...(modelId ? { modelId } : {}), doGenerate: { content: [ { @@ -8383,6 +8401,100 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(dryRunAttempt.outputTokens, 13); }); + test('a separate summarizer model is priced as itself, not as the session model', async () => { + // The record already named the model that served the request. Pricing it + // against a different model's rates would store an id and a `pricingRates` + // that contradict each other — the exact thing recording the rates is for. + const summarizerPricing = { + modelKey: 'anthropic:summarizer-model-id', + inputUsdPer1M: 1, + outputUsdPer1M: 2, + cacheReadUsdPer1M: 0.1, + cacheWriteUsdPer1M: 1, + }; + const mainPricing = { + modelKey: 'anthropic:mock-model-id', + inputUsdPer1M: 1_000, + outputUsdPer1M: 2_000, + cacheReadUsdPer1M: 100, + cacheWriteUsdPer1M: 1_000, + }; + const pricingKeysAsked: string[] = []; + const durable = durableTurnHarness('turn-1', 'hi'); + const modelCalls: ModelCallAttempt[] = []; + const { model } = semanticCompactFixtureModel(); + // A real `modelFactory` builds the summarizer against the configured id, so + // the record names it; the fixture has to do the same or it would prove + // nothing about which model the rates belong to. + const { model: summarizerOnlyModel } = semanticCompactFixtureModel('summarizer-model-id'); + const budget = semanticCompactContextBudget('replace'); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: ({ modelId }) => + modelId === 'summarizer-model-id' ? summarizerOnlyModel : model, + tools: [ + { + name: 'Read', + description: 'Read description', + parameters: z.object({ path: z.string() }), + impl: async ({ path }) => ({ + body: path === 'large.log' ? SEMANTIC_COMPACT_LARGE_BODY : 'FRESH_SEMANTIC_TAIL_RESULT', + }), + }, + ], + contextBudget: { + ...budget, + semanticCompact: { ...budget.semanticCompact, summarizerModel: 'summarizer-model-id' }, + }, + archiveToolResult: async () => ({ artifactId: 'archived-covered-semantic-result' }), + loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, + newId: idGenerator(), + now: monotonicClock(), + lookupPricing: (modelKey) => { + pricingKeysAsked.push(modelKey); + if (modelKey === summarizerPricing.modelKey) return summarizerPricing; + if (modelKey === mainPricing.modelKey) return mainPricing; + return null; + }, + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-semantic-capture' }), + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); + }, + recordSemanticCompactBlock: () => {}, + }); + + for await (const event of backend.send(durable.input({ runId: 'run-1' }))) { + durable.record(event); + } + + const summarization = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'semantic_compact'); + assert.ok(summarization, 'expected a canonical semantic compact record'); + assert.equal(summarization.modelId, 'summarizer-model-id'); + assert.equal(summarization.costBasis, 'priced'); + assert.deepEqual( + summarization.pricingRates, + summarizerPricing, + 'the rates stored are the ones the summarizer model actually bills at', + ); + assert.equal( + pricingKeysAsked.includes(summarizerPricing.modelKey), + true, + 'the cost lookup asks for the model that served the request', + ); + // The session model is still priced as itself for its own steps. + const mainCall = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'main'); + assert.equal(mainCall?.pricingRates?.modelKey, mainPricing.modelKey); + }); + test('active full compact keeps the accepted boundary projection across later AI SDK steps', async () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index cbcceb1178..cbade0bc55 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -77,6 +77,8 @@ import { buildHistoryCompactCheckpoint, type HistoryCompactCheckpoint, } from '../history-compact-checkpoint.js'; +import { buildLlmHistorySummarizer } from '../history-compact-summarizer.js'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { AGENT_WORKSPACE_WORKTREE, IMPLEMENTATION_AGENT_DEFINITION, @@ -3758,6 +3760,121 @@ describe('SessionManager manual compaction', () => { await manager.stopSession(session.id, { source: 'stop_button' }); }); + test('manual compaction settles one canonical record for the run the kernel opened', async () => { + // `sessions:compact` and CLI `/compact` both land here. The call has no + // send to inherit a run from, so the kernel states the run it opened and + // the record must carry it — otherwise a real, billed summarization is + // silently unmetered (#1679). + const store = new MemorySessionStore(); + const runStore = new MemoryAgentRunStore(); + const backends = new BackendRegistry(); + const modelCalls: ModelCallAttempt[] = []; + const summarizerModel = new MockLanguageModelV4({ + doGenerate: { + content: [{ type: 'text', text: 'MANUAL_COMPACT_SUMMARY' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 41, noCache: 41, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 9, text: 9, reasoning: 0 }, + raw: { input_tokens: 41, output_tokens: 9 }, + }, + warnings: [], + }, + }); + let summarizerIds = 0; + backends.register('fake', (ctx) => + createTestAiSdkBackend({ + sessionId: ctx.sessionId, + header: ctx.header, + appendMessage: async () => {}, + connection: { + slug: 'mock-main', + providerType: 'anthropic', + defaultModel: 'mock-model-id', + }, + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => + new MockLanguageModelV4({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: 'ok '.repeat(80) }, + { type: 'text-end', id: 'text-1' }, + { + type: 'finish', + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 100, noCache: 100, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 10, text: 10, reasoning: 0 }, + }, + }, + ] as LanguageModelV4StreamPart[], + initialDelayInMs: null, + chunkDelayInMs: null, + }), + }), + }), + tools: [], + newId: nextId(), + now: nextNow(1), + contextBudget: { + name: 'manual-compact-accounting', + maxHistoryEstimatedTokens: 10_000, + minRecentTurns: 1, + charsPerToken: 1, + }, + summarizeHistoryCompact: buildLlmHistorySummarizer({ + resolveModel: () => summarizerModel, + providerRequestTracking: { + now: () => 7_000, + newId: () => `manual-summarizer-${++summarizerIds}`, + recordAttempt: () => {}, + }, + }), + recordHistoryCompactCheckpoint: () => {}, + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); + }, + }), + ); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: runStore, + backends, + newId: nextId(), + now: nextNow(13_000), + }); + const session = await manager.createSession( + makeInput({ backend: 'fake', permissionMode: 'bypass' }), + ); + + await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'first '.repeat(400) })); + await drain(manager.sendMessage(session.id, { turnId: 'turn-2', text: 'second' })); + await drain(manager.compactSession(session.id, { turnId: 'turn-compact' })); + + const compactRun = (await runStore.listSessionRuns(session.id)).find( + (run) => run.turnId === 'turn-compact', + ); + assert.ok(compactRun, 'the kernel opens a run for a manual compaction'); + const compactions = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .filter((attempt) => attempt.callKind === 'history_compact'); + assert.equal(compactions.length, 1, 'one manual compaction is one record'); + assert.equal( + compactions[0]?.runId, + compactRun.runId, + 'attributed to the run the kernel opened, not to whatever ran last', + ); + assert.equal(compactions[0]?.turnId, 'turn-compact'); + assert.equal(compactions[0]?.inputTokens, 41); + assert.equal(compactions[0]?.usageBasis, 'reported'); + await manager.stopSession(session.id, { source: 'stop_button' }); + }); + test('persists one visible warning when manual compaction fails open', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 9899235dbb..f6d2ee85ea 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -572,7 +572,7 @@ export class AiSdkBackend implements AgentBackend { sessionId: this.sessionId, now: this.now, modelAdapter: this.modelAdapter, - modelCallAccounting: (callKind) => this.modelCallAccounting(callKind), + modelCallAccounting: (callKind, identity) => this.modelCallAccounting(callKind, identity), createProviderRequestTracker: (trackerInput) => this.createProviderRequestTracker(trackerInput), materializeRuntimeReplayPlan: (plan) => this.materializeRuntimeReplayPlan(plan), @@ -2054,9 +2054,13 @@ export class AiSdkBackend implements AgentBackend { turnId: string; callKind: ModelCallKind; modelId: string; + runId?: string; }): ProviderRequestTracker | undefined { const persistCapture = this.input.recordProviderRequestCapture; - const accounting = this.modelCallAccounting(input.callKind); + const accounting = this.modelCallAccounting(input.callKind, { + modelId: input.modelId, + ...(input.runId ? { runId: input.runId } : {}), + }); if (!persistCapture && !accounting) return undefined; return new ProviderRequestTracker({ traceId: this.newId(), @@ -2082,17 +2086,29 @@ export class AiSdkBackend implements AgentBackend { * Absent when there is no canonical sink, which leaves the corresponding * tracker purely diagnostic. */ - modelCallAccounting(callKind: ModelCallKind): ModelCallAccountingInput | undefined { + modelCallAccounting( + callKind: ModelCallKind, + identity?: { + /** + * States the run explicitly for a call made outside `send()`, where there + * is no live turn to resolve it from — a manual history compaction is one. + */ + runId?: string; + /** The model this call actually runs against; priced as that model. */ + modelId?: string; + }, + ): ModelCallAccountingInput | undefined { const record = this.input.recordModelCallAttempt; if (!record) return undefined; + const modelId = identity?.modelId ?? this.input.modelId; return { sessionId: this.sessionId, - resolveRunId: () => this.currentRunId ?? undefined, + resolveRunId: () => identity?.runId ?? this.currentRunId ?? undefined, connectionSlug: this.input.connection.slug, providerId: this.input.connection.providerType, callKind, record, - resolveCost: (usage: ProviderRequestUsage) => this.resolveModelCallCost(usage), + resolveCost: (usage: ProviderRequestUsage) => this.resolveModelCallCost(usage, modelId), ...(this.input.assertModelCallAccountingReady ? { assertReady: this.input.assertModelCallAccountingReady } : {}), @@ -2108,11 +2124,20 @@ export class AiSdkBackend implements AgentBackend { * cost. An unresolvable price returns `undefined` rather than zero — the * record then carries `costBasis: 'unpriced'`, which is not the same claim as * a call that was free. + * + * Priced against the model that actually served the request, which is not + * always the session's model: a configured semantic-compact summarizer runs + * on its own. Recording one model's id beside another model's rates would + * make the stored amount unauditable in exactly the way `pricingRates` exists + * to prevent. */ - private resolveModelCallCost(usage: ProviderRequestUsage): ResolvedModelCallCost | undefined { + private resolveModelCallCost( + usage: ProviderRequestUsage, + modelId: string, + ): ResolvedModelCallCost | undefined { try { const pricing = (this.input.lookupPricing ?? getBuiltinPricing)( - `${this.input.connection.providerType}:${this.input.modelId}`, + `${this.input.connection.providerType}:${modelId}`, ); if (pricing === null) return undefined; const costUsd = computeCost( diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 4ef96ea463..18744202ca 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -104,7 +104,10 @@ export interface AiSdkCompactionDeps { * Accounting identity for a compaction call, resolved by the backend because * `runId` changes per turn and only it holds the current one (#1679). */ - modelCallAccounting: (callKind: ModelCallKind) => ModelCallAccountingInput | undefined; + modelCallAccounting: ( + callKind: ModelCallKind, + identity?: { runId?: string; modelId?: string }, + ) => ModelCallAccountingInput | undefined; /** * A ready tracker for a compaction call that has none of its own. The backend * hands over the built tracker rather than the capture, attempt, and id sinks @@ -114,6 +117,7 @@ export interface AiSdkCompactionDeps { turnId: string; callKind: ModelCallKind; modelId: string; + runId?: string; }) => ProviderRequestTracker | undefined; materializeRuntimeReplayPlan: (plan: RuntimeEventModelReplayPlan) => Promise; canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; @@ -130,11 +134,13 @@ export class AiSdkCompaction { private readonly modelAdapter: ModelAdapter; private readonly modelCallAccounting: ( callKind: ModelCallKind, + identity?: { runId?: string; modelId?: string }, ) => ModelCallAccountingInput | undefined; private readonly createProviderRequestTracker: (input: { turnId: string; callKind: ModelCallKind; modelId: string; + runId?: string; }) => ProviderRequestTracker | undefined; private readonly materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, @@ -396,6 +402,11 @@ export class AiSdkCompaction { public async writeHistoryCompactCheckpoint(input: { requestShapeHashBefore?: string; turnId: string; + /** + * Present for a manual compaction, which runs outside `send()` and so has + * no live run for the backend to resolve. Absent mid-send, where it does. + */ + runId?: string; contextBudget: ContextBudgetPolicy; priorRuntimeContext: readonly RuntimeEvent[]; draftBlock: HistoryCompactBlock; @@ -408,7 +419,9 @@ export class AiSdkCompaction { const summarizer = this.input.summarizeHistoryCompact; const recorder = this.input.recordHistoryCompactCheckpoint; if (!summarizer || !recorder) return { diagnosticPatch: {} }; - const historyCompactAccounting = this.modelCallAccounting('history_compact'); + const historyCompactAccounting = this.modelCallAccounting('history_compact', { + ...(input.runId ? { runId: input.runId } : {}), + }); const foldedIds = new Set(input.draftBlock.coverage.runtimeEventIds); const foldedRuntimeEvents = input.priorRuntimeContext.filter((event) => foldedIds.has(event.id), @@ -782,6 +795,8 @@ export class AiSdkCompaction { } const writePatch = await this.writeHistoryCompactCheckpoint({ turnId: input.turnId, + // A manual compaction names its own run: nothing else can (#1679). + runId: input.runId, contextBudget: writeContextBudget, priorRuntimeContext: runtimeContext, draftBlock: draftBlocks[0]!, diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 89db4e48b7..4e5dbe6d7d 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -978,6 +978,7 @@ export class RuntimeKernel implements RuntimeKernelLike { this.assertRunCanDispatch(run, begin.backend); const result = await begin.backend.compactHistory({ turnId: run.turnId, + runId: run.runId, runtimeContext: begin.runtimeContext, ...(input.minRecentTurns !== undefined ? { minRecentTurns: input.minRecentTurns } : {}), }); From 6503ad189e65c783b3b8c1a62af737df0347e461 Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Mon, 3 Aug 2026 11:42:51 +0800 Subject: [PATCH 7/7] refactor(metering): give the backend sole ownership of the compaction tracker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 3 on #1877. The blocker was real: the CLI composition never passed `ctx.recordModelCallAttempt` to `AiSdkBackend` and never configured the summarizer's tracking, so `/compact` got the new `runId` and still had nothing to settle into. Harbor had the same gap for `semantic_compact`. Both are fixed here, but the reviewer's structural point is the one worth acting on: this is the fifth way the same split has failed, so the split goes. `history_compact` now gets its tracker from the same `AiSdkBackend.createProviderRequestTracker` that `main` and `semantic_compact` already use, and `HistoryCompactSummaryInput` carries a *ready* tracker instead of an accounting identity. `BuildLlmHistorySummarizerOptions.providerRequestTracking` is gone, and with it the Host's and Desktop's hand-assembled copies of the tracker's inputs — including the `contextWindow` plumbing added one round ago, which the backend now supplies for free because it knows the model. What a product owes accounting is now exactly one thing: the canonical sink. Nothing else can be half-wired, because nothing else is a product's to wire. `AiSdkCompactionDeps.modelCallAccounting` went with it — one factory, one seam. **Composition tests, because the last round's test proved the wrong thing.** The `SessionManager.compactSession()` test supplied both dependencies itself, so it demonstrated that the runtime works when everything is wired, not that the CLI wires it. The two new tests build backends through the real CLI and Harbor factories and assert the sink reaches the caller's recorder. Both were confirmed to fail with their production wiring reverted. CLI's gap turned out to be wider than the review described: with no sink passed at all, `main` sends were unmetered too, and had been since the frozen table's writer was removed — CLI never wired `recordLlmCall` either, so this composition root has simply never been inside accounting. Left out deliberately, as agreed: `goal_evaluation` routing, a global `errorClass`, and anything Inspector-shaped. `@maka/runtime` 2700/2712 (3 suites = documented local `rg` noise), `@maka/runtime-host` 563/563, `maka-agent` (CLI) 467/467, `@maka/headless` harbor-cell 102/102, `@maka/desktop` 1156/1194 (38 = this worktree's missing Astryx peer dep). Co-Authored-By: Claude Opus 5 --- apps/desktop/src/main/session-stream.ts | 18 +--- .../src/__tests__/runtime-bootstrap.test.ts | 82 +++++++++++++++++++ packages/cli/src/runtime-bootstrap.ts | 4 + .../src/__tests__/harbor-cell.test.ts | 75 +++++++++++++++++ packages/headless/src/harbor-cell.ts | 5 ++ .../src/server/execution-model-composition.ts | 18 ---- .../history-compact-summarizer.test.ts | 34 ++++---- .../mid-turn-capacity-backend.test.ts | 7 -- .../src/__tests__/session-manager.test.ts | 6 -- packages/runtime/src/ai-sdk-backend.ts | 1 - .../runtime/src/ai-sdk-compaction-contract.ts | 14 ++-- packages/runtime/src/ai-sdk-compaction.ts | 34 +++----- .../runtime/src/history-compact-summarizer.ts | 22 +---- 13 files changed, 212 insertions(+), 108 deletions(-) diff --git a/apps/desktop/src/main/session-stream.ts b/apps/desktop/src/main/session-stream.ts index 56fde471ce..cbb6f0534f 100644 --- a/apps/desktop/src/main/session-stream.ts +++ b/apps/desktop/src/main/session-stream.ts @@ -135,9 +135,8 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen mode: effectivePermissionMode, cwd: ctx.header.cwd, }); - // Hoisted so the auxiliary summarizer can share them with the send path: - // capture is optional plumbing, the context window is a property of the - // model both calls run against. + // Hoisted out of the backend input so the shape stays readable; the + // auxiliary summarizer no longer needs any of it (#1679). const providerRequestCapture = ctx.recordProviderRequestCapture ? createProviderRequestCaptureRecorder({ persistArtifact: async (capture) => { @@ -154,7 +153,6 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen recordLedger: ctx.recordProviderRequestCapture, }) : undefined; - const summarizerContextWindow = resolveSelectedModelContextWindow(connection, model); return new AiSdkBackend({ sessionId: ctx.sessionId, @@ -345,18 +343,6 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen resolveModel: () => getAIModel({ connection, apiKey: apiKey ?? '', modelId: model, fetch: modelFetch }), providerOptions: buildProviderOptions(connection, model, ctx.header.thinkingLevel), - // Without this the accounting the backend computes for a history - // compaction is handed to a summarizer that has nowhere to settle it, - // and the call goes unmetered. Capture joins in only when configured. - providerRequestTracking: { - now: Date.now, - newId: randomUUID, - ...(providerRequestCapture ? { persistCapture: providerRequestCapture } : {}), - recordAttempt: ctx.recordProviderRequestAttempt ?? (() => {}), - ...(summarizerContextWindow !== undefined - ? { contextWindow: summarizerContextWindow } - : {}), - }, }), loadSynthesisCache: (event) => loadSynthesisCacheBlocksFromArtifacts(artifactStore, event), writeSynthesisCache: (event) => persistSynthesisCacheBlocksToArtifacts(artifactStore, event, { diff --git a/packages/cli/src/__tests__/runtime-bootstrap.test.ts b/packages/cli/src/__tests__/runtime-bootstrap.test.ts index 66470acbbf..943fe0116a 100644 --- a/packages/cli/src/__tests__/runtime-bootstrap.test.ts +++ b/packages/cli/src/__tests__/runtime-bootstrap.test.ts @@ -4,6 +4,10 @@ import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; +import { + MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + type ModelCallAttempt, +} from '@maka/core/model-call-attempt'; import { createConnectionStore, createFileCredentialStore, @@ -38,6 +42,29 @@ import { resolveCliStreamConnectTimeoutMs, } from '../runtime-bootstrap.js'; +function modelCallAttemptFixture(): ModelCallAttempt { + return { + schemaVersion: MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + logicalCallId: 'call-1', + attemptId: 'attempt-1', + traceId: 'trace-1', + sessionId: 'session-1', + runId: 'run-1', + turnId: 'turn-1', + step: 0, + attempt: 0, + callKind: 'history_compact', + providerId: 'ollama', + modelId: 'llama3.2', + startedAt: 1, + completedAt: 2, + latencyMs: 1, + status: 'completed', + usageBasis: 'missing', + costBasis: 'unpriced', + }; +} + describe('Maka CLI runtime bootstrap', () => { test('parses the CLI stream connect timeout override', () => { assert.equal(resolveCliStreamConnectTimeoutMs({}), undefined); @@ -205,6 +232,61 @@ describe('Maka CLI runtime bootstrap', () => { }); }); + test('forwards the canonical metering sink from the backend context', async () => { + // The CLI factory used to wire capture and attempt diagnostics but not the + // canonical sink, so `/compact` and ordinary sends produced no + // `ModelCallAttempt` at all — the kernel offered one and nothing took it. + await withWorkspace(async (workspaceRoot) => { + const connectionStore = createConnectionStore(workspaceRoot); + await connectionStore.create({ + slug: 'local', + name: 'Local Ollama', + providerType: 'ollama', + defaultModel: 'llama3.2', + }); + const context = await createMakaCliRuntimeContext({ + surface: 'tui', + workspaceRoot, + cwd: '/repo', + }); + try { + const session = await context.runtime.createSession({ + cwd: context.cwd, + backend: 'ai-sdk', + llmConnectionSlug: context.target.connection.slug, + model: context.target.model, + permissionMode: 'explore', + name: 'metering-sink', + }); + const runtimeDeps = (context.runtime as unknown as RuntimeWithPrivateDeps).deps; + const header = await runtimeDeps.store.readHeader(session.id); + const recorded: ModelCallAttempt[] = []; + const backend = await runtimeDeps.backends.build('ai-sdk', { + sessionId: session.id, + workspaceRoot, + header, + store: runtimeDeps.store, + recordModelCallAttempt: (attempt: ModelCallAttempt) => { + recorded.push(attempt); + return Promise.resolve(); + }, + }); + const backendInput = (backend as unknown as { input: AiSdkBackendInput }).input; + + assert.equal( + typeof backendInput.recordModelCallAttempt, + 'function', + 'the composition must pass the sink the kernel offers', + ); + await backendInput.recordModelCallAttempt?.(modelCallAttemptFixture()); + assert.equal(recorded.length, 1, 'and it must reach the context, not a local stub'); + assert.equal(recorded[0]?.callKind, 'history_compact'); + } finally { + await context.close(); + } + }); + }); + test('uses an explicit connection and forwards one-shot limits and invocation results', async () => { await withWorkspace(async (workspaceRoot) => { const connectionStore = createConnectionStore(workspaceRoot); diff --git a/packages/cli/src/runtime-bootstrap.ts b/packages/cli/src/runtime-bootstrap.ts index 4bd3730e3f..81ed705b6c 100644 --- a/packages/cli/src/runtime-bootstrap.ts +++ b/packages/cli/src/runtime-bootstrap.ts @@ -744,6 +744,10 @@ export async function createMakaCliRuntimeContext( }), providerOptions: buildProviderOptions(ready.connection, ready.model, header.thinkingLevel), }), + // The canonical metering sink (#1679). Without it this composition root + // produces diagnostics and no accounting at all — for `/compact` and for + // ordinary sends alike. + ...(ctx.recordModelCallAttempt ? { recordModelCallAttempt: ctx.recordModelCallAttempt } : {}), recordHistoryCompactCheckpoint: ctx.recordHistoryCompactCheckpoint, loadTurnRuntimeEvents: ctx.loadTurnRuntimeEvents, allowMidTurnHistoryCompaction: ctx.allowMidTurnHistoryCompaction, diff --git a/packages/headless/src/__tests__/harbor-cell.test.ts b/packages/headless/src/__tests__/harbor-cell.test.ts index e1ad68d526..a51f76b8b3 100644 --- a/packages/headless/src/__tests__/harbor-cell.test.ts +++ b/packages/headless/src/__tests__/harbor-cell.test.ts @@ -16,6 +16,10 @@ import type { import type { BackendSendInput, BackendStopMode } from '@maka/core/backend-types'; import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; import { createSessionStore } from '@maka/storage'; +import { + MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + type ModelCallAttempt, +} from '@maka/core/model-call-attempt'; import { BackendRegistry, PiAgentBackend, @@ -1985,6 +1989,54 @@ describe('runHarborCell', () => { }); }); + test('Harbor ai-sdk backend registration forwards the canonical metering sink', async () => { + // The controller has always exposed `recordModelCallAttempt`; this + // composition never passed it on, so Harbor produced diagnostic attempts + // with no canonical record behind them (#1679). + await withDirs(async ({ workspaceDir, artifactStore }) => { + const registry = new BackendRegistry(); + const toolExecutor = fakeToolExecutor(); + const register = buildAiSdkCellBackendRegistration({ + provider: 'openai', + model: 'gpt-5.6-sol', + env: { OPENAI_API_KEY: 'test-key' }, + now: () => 123, + newId: () => 'id', + }); + await registerProjectedAiSdkBackend(register, registry, { + config: { + id: 'harbor-ai-sdk', + backend: 'ai-sdk', + llmConnectionSlug: 'openai', + model: 'gpt-5.6-sol', + systemPrompt: DEFAULT_HEADLESS_SYSTEM_PROMPT, + }, + task: { id: 'harbor-cell', instruction: 'solve', workspaceDir }, + storageRoot: workspaceDir, + workspaceDir, + artifactStore, + realBackendIsolation: { kind: 'external', label: 'Harbor task container', toolExecutor }, + toolExecutor, + ...createHeadlessSessionCapabilityBridge().capabilities, + }); + + const recorded: ModelCallAttempt[] = []; + const backend = await registry.build('ai-sdk', { + ...backendContext(workspaceDir), + recordModelCallAttempt: (attempt: ModelCallAttempt) => { + recorded.push(attempt); + return Promise.resolve(); + }, + }); + const backendInput = (backend as unknown as { input: AiSdkBackendInput }).input; + + assert.equal(typeof backendInput.recordModelCallAttempt, 'function'); + await backendInput.recordModelCallAttempt?.(harborModelCallAttemptFixture()); + assert.equal(recorded.length, 1, 'the sink must reach the controller'); + assert.equal(recorded[0]?.callKind, 'semantic_compact'); + }); + }); + test('Harbor ai-sdk backend registration exposes native file tools to the provider schema', async () => { await withDirs(async ({ workspaceDir, artifactStore }) => { const registry = new BackendRegistry(); @@ -4780,6 +4832,29 @@ function sha256(text: string): string { return createHash('sha256').update(text).digest('hex'); } +function harborModelCallAttemptFixture(): ModelCallAttempt { + return { + schemaVersion: MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + logicalCallId: 'call-1', + attemptId: 'attempt-1', + traceId: 'trace-1', + sessionId: 'session-1', + runId: 'run-1', + turnId: 'turn-1', + step: 0, + attempt: 0, + callKind: 'semantic_compact', + providerId: 'openai', + modelId: 'gpt-5.6-sol', + startedAt: 1, + completedAt: 2, + latencyMs: 1, + status: 'completed', + usageBasis: 'missing', + costBasis: 'unpriced', + }; +} + function backendContext(workspaceDir: string): BackendFactoryContext { return { sessionId: 'session-1', diff --git a/packages/headless/src/harbor-cell.ts b/packages/headless/src/harbor-cell.ts index 98d5a821d2..92f157db98 100644 --- a/packages/headless/src/harbor-cell.ts +++ b/packages/headless/src/harbor-cell.ts @@ -1192,6 +1192,11 @@ export function buildAiSdkCellBackendRegistration(input: { newId: input.newId, now: input.now, recordRunTrace: ctx.recordRunTrace, + // The canonical metering sink (#1679); the controller has always + // exposed it, this composition just never passed it through. + ...(ctx.recordModelCallAttempt + ? { recordModelCallAttempt: ctx.recordModelCallAttempt } + : {}), ...(ctx.recordProviderRequestCapture ? { recordProviderRequestCapture: createProviderRequestCaptureRecorder({ diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index 4d90c7ee74..b601f2200a 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -554,12 +554,6 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom } }; let artifactDrainRequested = false; - // The summarizer runs on the session's own connection and model, so its - // attempts are measured against the same window the send is. - const summarizerContextWindow = resolveSelectedModelContextWindow( - target.connection, - target.model, - ); const providerRequestCapture = input.context.recordProviderRequestCapture ? createProviderRequestCaptureRecorder({ persistArtifact: async (capture) => { @@ -638,18 +632,6 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom modelId: target.model, }), providerOptions, - // Wired unconditionally: this is what carries the summarization's - // accounting, and metering must not depend on the capture sink being - // configured. Capture joins in only when it is. - providerRequestTracking: { - now: Date.now, - newId: randomUUID, - ...(providerRequestCapture ? { persistCapture: providerRequestCapture } : {}), - recordAttempt: recordProviderRequestAttempt, - ...(summarizerContextWindow !== undefined - ? { contextWindow: summarizerContextWindow } - : {}), - }, }), recordHistoryCompactCheckpoint: input.context.recordHistoryCompactCheckpoint, loadTurnRuntimeEvents: input.context.loadTurnRuntimeEvents, diff --git a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts index 9305b3d4b2..c98e1fbd12 100644 --- a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts +++ b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts @@ -10,6 +10,7 @@ import assert from 'node:assert/strict'; import { expect } from '../test-helpers.js'; import type { RuntimeEvent, RuntimeEventContent } from '@maka/core/runtime-event'; import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; +import { ProviderRequestTracker } from '../provider-request-telemetry.js'; import type { HistoryCompactSummaryInput } from '../ai-sdk-compaction-contract.js'; import { buildLlmHistorySummarizer, @@ -116,7 +117,14 @@ describe('buildLlmHistorySummarizer', () => { warnings: [], }, }), - providerRequestTracking: { + }); + + // The backend hands over a built tracker; the summarizer assembles nothing. + await summarize({ + ...inputWith([ev({ role: 'user', author: 'user', content: { kind: 'text', text: 'hi' } })]), + providerRequestTracker: new ProviderRequestTracker({ + traceId: 'trace-id', + turnId: 'turn-1', now: () => { now += 10; return now; @@ -124,21 +132,17 @@ describe('buildLlmHistorySummarizer', () => { newId: () => 'trace-id', persistCapture: async () => ({ artifactId: 'artifact-1' }), recordAttempt: () => {}, - }, - }); - - await summarize({ - ...inputWith([ev({ role: 'user', author: 'user', content: { kind: 'text', text: 'hi' } })]), - accounting: { - sessionId: 'sess-1', - resolveRunId: () => 'run-1', - connectionSlug: 'connection', - providerId: 'provider', - callKind: 'history_compact', - record: (attempt) => { - recorded.push(attempt); + accounting: { + sessionId: 'sess-1', + resolveRunId: () => 'run-1', + connectionSlug: 'connection', + providerId: 'provider', + callKind: 'history_compact', + record: (attempt: ModelCallAttempt) => { + recorded.push(attempt); + }, }, - }, + }), }); const attempt = decodeModelCallAttempt(recorded[0]); diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index 50712dde00..bcd6ae8f2f 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -265,15 +265,8 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { warnings: [], }, }); - let summarizerIds = 0; const meteredSummarize = buildLlmHistorySummarizer({ resolveModel: () => summarizerModel, - providerRequestTracking: { - now: () => 5_000, - newId: () => `summarizer-id-${++summarizerIds}`, - persistCapture: async () => ({ artifactId: 'artifact-mid-turn-capture' }), - recordAttempt: () => {}, - }, }); const backend = createTestAiSdkBackend({ diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index cbade0bc55..ecb24c6f8e 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -3781,7 +3781,6 @@ describe('SessionManager manual compaction', () => { warnings: [], }, }); - let summarizerIds = 0; backends.register('fake', (ctx) => createTestAiSdkBackend({ sessionId: ctx.sessionId, @@ -3828,11 +3827,6 @@ describe('SessionManager manual compaction', () => { }, summarizeHistoryCompact: buildLlmHistorySummarizer({ resolveModel: () => summarizerModel, - providerRequestTracking: { - now: () => 7_000, - newId: () => `manual-summarizer-${++summarizerIds}`, - recordAttempt: () => {}, - }, }), recordHistoryCompactCheckpoint: () => {}, recordModelCallAttempt: (attempt) => { diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index f6d2ee85ea..a9760b3fd9 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -572,7 +572,6 @@ export class AiSdkBackend implements AgentBackend { sessionId: this.sessionId, now: this.now, modelAdapter: this.modelAdapter, - modelCallAccounting: (callKind, identity) => this.modelCallAccounting(callKind, identity), createProviderRequestTracker: (trackerInput) => this.createProviderRequestTracker(trackerInput), materializeRuntimeReplayPlan: (plan) => this.materializeRuntimeReplayPlan(plan), diff --git a/packages/runtime/src/ai-sdk-compaction-contract.ts b/packages/runtime/src/ai-sdk-compaction-contract.ts index 3a09f77813..eafed7f31b 100644 --- a/packages/runtime/src/ai-sdk-compaction-contract.ts +++ b/packages/runtime/src/ai-sdk-compaction-contract.ts @@ -1,7 +1,7 @@ import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { ModelCallAccountingInput } from './provider-request-telemetry.js'; +import type { ProviderRequestTracker } from './provider-request-telemetry.js'; import type { ActiveFullCompactBlock } from './active-full-compact.js'; import type { ActiveToolResultArchiveCandidate } from './active-tool-result-prune.js'; import type { @@ -119,12 +119,14 @@ export interface HistoryCompactSummaryInput { requestShapeHashBefore?: string; abortSignal?: AbortSignal; /** - * Accounting identity for this summarization call (#1679). Supplied per call - * rather than baked into the summarizer, because the host that configures the - * summarizer cannot know which run is active — `runId` is per-turn state the - * backend holds. + * Physical-call tracking for this summarization, built by the backend (#1679). + * + * A *ready* tracker, not the parts to assemble one: the products that wire a + * summarizer cannot know the run a call belongs to, and every root that had to + * assemble it independently eventually forgot a piece. Absent when the product + * supplied no canonical sink, which leaves the call untracked. */ - accounting?: ModelCallAccountingInput; + providerRequestTracker?: ProviderRequestTracker; } export type HistoryCompactSummarizer = ( input: HistoryCompactSummaryInput, diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 18744202ca..4337293ca5 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -83,10 +83,7 @@ import { } from './model-history.js'; import { toolSchemaCharsForDiagnostics } from './request-shape.js'; import type { ModelCallKind } from '@maka/core/model-call-attempt'; -import type { - ModelCallAccountingInput, - ProviderRequestTracker, -} from './provider-request-telemetry.js'; +import type { ProviderRequestTracker } from './provider-request-telemetry.js'; import { estimateNextRequestTokens, exceedsHighWater, @@ -100,14 +97,6 @@ export interface AiSdkCompactionDeps { sessionId: string; now: () => number; modelAdapter: ModelAdapter; - /** - * Accounting identity for a compaction call, resolved by the backend because - * `runId` changes per turn and only it holds the current one (#1679). - */ - modelCallAccounting: ( - callKind: ModelCallKind, - identity?: { runId?: string; modelId?: string }, - ) => ModelCallAccountingInput | undefined; /** * A ready tracker for a compaction call that has none of its own. The backend * hands over the built tracker rather than the capture, attempt, and id sinks @@ -132,10 +121,6 @@ export class AiSdkCompaction { private readonly sessionId: string; private readonly now: () => number; private readonly modelAdapter: ModelAdapter; - private readonly modelCallAccounting: ( - callKind: ModelCallKind, - identity?: { runId?: string; modelId?: string }, - ) => ModelCallAccountingInput | undefined; private readonly createProviderRequestTracker: (input: { turnId: string; callKind: ModelCallKind; @@ -157,7 +142,6 @@ export class AiSdkCompaction { this.sessionId = deps.sessionId; this.now = deps.now; this.modelAdapter = deps.modelAdapter; - this.modelCallAccounting = deps.modelCallAccounting; this.createProviderRequestTracker = deps.createProviderRequestTracker; this.materializeRuntimeReplayPlan = deps.materializeRuntimeReplayPlan; this.canReplayProviderNative = deps.canReplayProviderNative; @@ -419,7 +403,11 @@ export class AiSdkCompaction { const summarizer = this.input.summarizeHistoryCompact; const recorder = this.input.recordHistoryCompactCheckpoint; if (!summarizer || !recorder) return { diagnosticPatch: {} }; - const historyCompactAccounting = this.modelCallAccounting('history_compact', { + // One tracker for this summarization, built where every input lives. + const historyCompactTracker = this.createProviderRequestTracker({ + turnId: input.turnId, + callKind: 'history_compact', + modelId: this.input.modelId, ...(input.runId ? { runId: input.runId } : {}), }); const foldedIds = new Set(input.draftBlock.coverage.runtimeEventIds); @@ -497,7 +485,7 @@ export class AiSdkCompaction { newlyFoldedRuntimeEvents, requestShapeHashBefore: input.requestShapeHashBefore, abortSignal: input.abortSignal, - ...(historyCompactAccounting ? { accounting: historyCompactAccounting } : {}), + ...(historyCompactTracker ? { providerRequestTracker: historyCompactTracker } : {}), }), ); if (!summary?.trim()) { @@ -1474,7 +1462,11 @@ export class AiSdkCompaction { abortSignal, } = input; const summarizer = this.input.summarizeHistoryCompact!; - const midTurnAccounting = this.modelCallAccounting('history_compact'); + const midTurnTracker = this.createProviderRequestTracker({ + turnId, + callKind: 'history_compact', + modelId: this.input.modelId, + }); const recorder = this.input.recordHistoryCompactCheckpoint!; const loadTurnRuntimeEvents = this.input.loadTurnRuntimeEvents!; const policy = this.input.contextBudget!; @@ -1572,7 +1564,7 @@ export class AiSdkCompaction { ...(previousCheckpoint ? { previousCheckpoint } : {}), newlyFoldedRuntimeEvents: [...newlyFoldedRuntimeEvents], ...(abortSignal ? { abortSignal } : {}), - ...(midTurnAccounting ? { accounting: midTurnAccounting } : {}), + ...(midTurnTracker ? { providerRequestTracker: midTurnTracker } : {}), }), ); }, diff --git a/packages/runtime/src/history-compact-summarizer.ts b/packages/runtime/src/history-compact-summarizer.ts index 7d6c19cc67..6bafd48ed9 100644 --- a/packages/runtime/src/history-compact-summarizer.ts +++ b/packages/runtime/src/history-compact-summarizer.ts @@ -4,11 +4,7 @@ import { toolResultOutput } from './tool-result-output.js'; import type { HistoryCompactSummaryInput } from './ai-sdk-compaction-contract.js'; import { HistoryCompactSummarizerError } from './history-compact-error.js'; import type { AiSdkUsageLike } from './model-adapter.js'; -import { - ProviderRequestTracker, - withProviderGenerateTracking, - type ProviderRequestTrackerInput, -} from './provider-request-telemetry.js'; +import { withProviderGenerateTracking } from './provider-request-telemetry.js'; export { HistoryCompactSummarizerError } from './history-compact-error.js'; @@ -32,8 +28,6 @@ export interface BuildLlmHistorySummarizerOptions { providerOptions?: Record; /** Injectable `generateText` for tests; defaults to the real AI SDK export. */ generateText?: AiSdkGenerateTextLike; - /** Physical provider-call capture and attempt tracking for generated summaries. */ - providerRequestTracking?: Omit; } // Conversation-summarization prompt (sectioned, modelled on pi/opencode): @@ -87,17 +81,9 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti ], }); } - const providerRequestTracker = options.providerRequestTracking - ? new ProviderRequestTracker({ - ...options.providerRequestTracking, - traceId: options.providerRequestTracking.newId(), - turnId: input.turnId, - // Per call, not per summarizer: the host wires the capture and - // attempt plumbing once, but only the caller knows the run this - // summarization belongs to. - ...(input.accounting ? { accounting: input.accounting } : {}), - }) - : undefined; + // Handed over whole by the backend, which owns every input a tracker + // needs — including the run, which no summarizer wiring can know (#1679). + const providerRequestTracker = input.providerRequestTracker; const ai = options.generateText && !providerRequestTracker ? undefined : await loadAiSdkTextModule(); const generateText = options.generateText ?? ai!.generateText;