diff --git a/apps/desktop/src/main/session-stream.ts b/apps/desktop/src/main/session-stream.ts index 66b5640c98..cbb6f0534f 100644 --- a/apps/desktop/src/main/session-stream.ts +++ b/apps/desktop/src/main/session-stream.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { SessionChangedReason, SessionEvent } from '@maka/core'; -import type { LlmCallRecord, ToolInvocationRecord } from '@maka/core/usage-stats/types'; +import type { ToolInvocationRecord } from '@maka/core/usage-stats/types'; import { AiSdkBackend, buildDefaultContextBudgetPolicy, @@ -11,7 +11,6 @@ import { loadHistoryCompactBlocksFromArtifacts, loadSynthesisCacheBlocksFromArtifacts, persistSynthesisCacheBlocksToArtifacts, - recordLlmCall, recordToolInvocation, renderPlanExecutionPrompt, renderInterruptedPlanContext, @@ -85,8 +84,8 @@ export interface AiSdkBackendFactoryDeps extends DesktopBackendToolSurfaceDeps { * seams that resolve AFTER the registration point are injected as accessors: * `getRuntime` (the SessionManager is constructed after registration) and * `getLookupPricing` (a mutable pricing lookup reassigned by usage IPC + startup; - * read live per `recordLlmCall`, snapshotted once for the `lookupPricing` field — - * matching the original module-`let` closure semantics exactly). + * snapshotted once for the `lookupPricing` field — matching the original + * module-`let` closure semantics exactly). */ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): BackendFactory { const { @@ -136,6 +135,24 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen mode: effectivePermissionMode, cwd: ctx.header.cwd, }); + // Hoisted out of the backend input so the shape stays readable; the + // auxiliary summarizer no longer needs any of it (#1679). + const providerRequestCapture = ctx.recordProviderRequestCapture + ? createProviderRequestCaptureRecorder({ + persistArtifact: async (capture) => { + const artifact = await persistProviderRequestCaptureArtifact(artifactStore, { + sessionId: ctx.sessionId, + turnId: capture.turnId, + captureId: capture.captureId, + step: capture.step, + serializedRequest: capture.serializedRequest, + now: Date.now(), + }); + return { artifactId: artifact.id }; + }, + recordLedger: ctx.recordProviderRequestCapture, + }) + : undefined; return new AiSdkBackend({ sessionId: ctx.sessionId, @@ -281,7 +298,6 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen }, shellRunContextSummary: ctx.shellRunContextSummary, lookupPricing: getLookupPricing(), - recordLlmCall: (event: LlmCallRecord) => recordLlmCall({ repo: telemetryRepo, lookupPricing: getLookupPricing() }, event), // One canonical record, one commit point (#1679): the AgentRun stream is // the only durable authority, and the ledger is a projection written only // after the authority holds the record. A failed projection marks the run @@ -340,22 +356,9 @@ export function createAiSdkBackendFactory(deps: AiSdkBackendFactoryDeps): Backen }, }), recordRunTrace: ctx.recordRunTrace, - ...(ctx.recordProviderRequestCapture + ...(providerRequestCapture ? { - recordProviderRequestCapture: createProviderRequestCaptureRecorder({ - persistArtifact: async (capture) => { - const artifact = await persistProviderRequestCaptureArtifact(artifactStore, { - sessionId: ctx.sessionId, - turnId: capture.turnId, - captureId: capture.captureId, - step: capture.step, - serializedRequest: capture.serializedRequest, - now: Date.now(), - }); - return { artifactId: artifact.id }; - }, - recordLedger: ctx.recordProviderRequestCapture, - }), + recordProviderRequestCapture: providerRequestCapture, recordProviderRequestAttempt: ctx.recordProviderRequestAttempt, } : {}), diff --git a/apps/desktop/src/main/usage-ipc-main.ts b/apps/desktop/src/main/usage-ipc-main.ts index 45a0bdf67c..acc6fff792 100644 --- a/apps/desktop/src/main/usage-ipc-main.ts +++ b/apps/desktop/src/main/usage-ipc-main.ts @@ -42,9 +42,11 @@ const USAGE_REPAIR_RUNS_PER_QUERY = 16; export function registerUsageIpc(deps: UsageIpcDeps): void { /** * Usage answers sum two sources (#1679): the canonical model-call ledger and - * the frozen `LlmCallRecord` table, which still receives the compaction calls - * that have not been routed through the canonical seam yet. Every merged - * result carries the provenance that qualifies it. + * the frozen `LlmCallRecord` table. Both compaction kinds now settle through + * the canonical seam; what still lands in the frozen table is historical rows + * and `goal_evaluation`, the one kind the seam cannot yet identify (it has no + * run or turn at the Host layer). Every merged result carries the provenance + * that qualifies it. */ const canonicalUsage = async (query: UsageQuery, now: number): Promise => { // Fold in whatever the authority holds and this read model is behind on diff --git a/packages/cli/src/__tests__/runtime-bootstrap.test.ts b/packages/cli/src/__tests__/runtime-bootstrap.test.ts index 66470acbbf..943fe0116a 100644 --- a/packages/cli/src/__tests__/runtime-bootstrap.test.ts +++ b/packages/cli/src/__tests__/runtime-bootstrap.test.ts @@ -4,6 +4,10 @@ import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; +import { + MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + type ModelCallAttempt, +} from '@maka/core/model-call-attempt'; import { createConnectionStore, createFileCredentialStore, @@ -38,6 +42,29 @@ import { resolveCliStreamConnectTimeoutMs, } from '../runtime-bootstrap.js'; +function modelCallAttemptFixture(): ModelCallAttempt { + return { + schemaVersion: MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + logicalCallId: 'call-1', + attemptId: 'attempt-1', + traceId: 'trace-1', + sessionId: 'session-1', + runId: 'run-1', + turnId: 'turn-1', + step: 0, + attempt: 0, + callKind: 'history_compact', + providerId: 'ollama', + modelId: 'llama3.2', + startedAt: 1, + completedAt: 2, + latencyMs: 1, + status: 'completed', + usageBasis: 'missing', + costBasis: 'unpriced', + }; +} + describe('Maka CLI runtime bootstrap', () => { test('parses the CLI stream connect timeout override', () => { assert.equal(resolveCliStreamConnectTimeoutMs({}), undefined); @@ -205,6 +232,61 @@ describe('Maka CLI runtime bootstrap', () => { }); }); + test('forwards the canonical metering sink from the backend context', async () => { + // The CLI factory used to wire capture and attempt diagnostics but not the + // canonical sink, so `/compact` and ordinary sends produced no + // `ModelCallAttempt` at all — the kernel offered one and nothing took it. + await withWorkspace(async (workspaceRoot) => { + const connectionStore = createConnectionStore(workspaceRoot); + await connectionStore.create({ + slug: 'local', + name: 'Local Ollama', + providerType: 'ollama', + defaultModel: 'llama3.2', + }); + const context = await createMakaCliRuntimeContext({ + surface: 'tui', + workspaceRoot, + cwd: '/repo', + }); + try { + const session = await context.runtime.createSession({ + cwd: context.cwd, + backend: 'ai-sdk', + llmConnectionSlug: context.target.connection.slug, + model: context.target.model, + permissionMode: 'explore', + name: 'metering-sink', + }); + const runtimeDeps = (context.runtime as unknown as RuntimeWithPrivateDeps).deps; + const header = await runtimeDeps.store.readHeader(session.id); + const recorded: ModelCallAttempt[] = []; + const backend = await runtimeDeps.backends.build('ai-sdk', { + sessionId: session.id, + workspaceRoot, + header, + store: runtimeDeps.store, + recordModelCallAttempt: (attempt: ModelCallAttempt) => { + recorded.push(attempt); + return Promise.resolve(); + }, + }); + const backendInput = (backend as unknown as { input: AiSdkBackendInput }).input; + + assert.equal( + typeof backendInput.recordModelCallAttempt, + 'function', + 'the composition must pass the sink the kernel offers', + ); + await backendInput.recordModelCallAttempt?.(modelCallAttemptFixture()); + assert.equal(recorded.length, 1, 'and it must reach the context, not a local stub'); + assert.equal(recorded[0]?.callKind, 'history_compact'); + } finally { + await context.close(); + } + }); + }); + test('uses an explicit connection and forwards one-shot limits and invocation results', async () => { await withWorkspace(async (workspaceRoot) => { const connectionStore = createConnectionStore(workspaceRoot); diff --git a/packages/cli/src/runtime-bootstrap.ts b/packages/cli/src/runtime-bootstrap.ts index 4bd3730e3f..81ed705b6c 100644 --- a/packages/cli/src/runtime-bootstrap.ts +++ b/packages/cli/src/runtime-bootstrap.ts @@ -744,6 +744,10 @@ export async function createMakaCliRuntimeContext( }), providerOptions: buildProviderOptions(ready.connection, ready.model, header.thinkingLevel), }), + // The canonical metering sink (#1679). Without it this composition root + // produces diagnostics and no accounting at all — for `/compact` and for + // ordinary sends alike. + ...(ctx.recordModelCallAttempt ? { recordModelCallAttempt: ctx.recordModelCallAttempt } : {}), recordHistoryCompactCheckpoint: ctx.recordHistoryCompactCheckpoint, loadTurnRuntimeEvents: ctx.loadTurnRuntimeEvents, allowMidTurnHistoryCompaction: ctx.allowMidTurnHistoryCompaction, diff --git a/packages/core/src/backend-types.ts b/packages/core/src/backend-types.ts index 0f26b7b0ef..88020661c4 100644 --- a/packages/core/src/backend-types.ts +++ b/packages/core/src/backend-types.ts @@ -129,6 +129,13 @@ export interface SteeringLease { export interface BackendCompactHistoryInput { turnId: string; + /** + * The run this compaction belongs to. Required, not optional: a manual + * compaction is a real model call, and a call whose run cannot be named is a + * call nothing can bill (#1679). Unlike `send`, this path has no turn state to + * infer it from, so the caller that opened the run states it. + */ + runId: string; runtimeContext: readonly RuntimeEvent[]; /** Override the configured recent-turn tail for an explicit recovery compaction. */ minRecentTurns?: number; diff --git a/packages/headless/src/__tests__/harbor-cell.test.ts b/packages/headless/src/__tests__/harbor-cell.test.ts index e1ad68d526..a51f76b8b3 100644 --- a/packages/headless/src/__tests__/harbor-cell.test.ts +++ b/packages/headless/src/__tests__/harbor-cell.test.ts @@ -16,6 +16,10 @@ import type { import type { BackendSendInput, BackendStopMode } from '@maka/core/backend-types'; import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; import { createSessionStore } from '@maka/storage'; +import { + MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + type ModelCallAttempt, +} from '@maka/core/model-call-attempt'; import { BackendRegistry, PiAgentBackend, @@ -1985,6 +1989,54 @@ describe('runHarborCell', () => { }); }); + test('Harbor ai-sdk backend registration forwards the canonical metering sink', async () => { + // The controller has always exposed `recordModelCallAttempt`; this + // composition never passed it on, so Harbor produced diagnostic attempts + // with no canonical record behind them (#1679). + await withDirs(async ({ workspaceDir, artifactStore }) => { + const registry = new BackendRegistry(); + const toolExecutor = fakeToolExecutor(); + const register = buildAiSdkCellBackendRegistration({ + provider: 'openai', + model: 'gpt-5.6-sol', + env: { OPENAI_API_KEY: 'test-key' }, + now: () => 123, + newId: () => 'id', + }); + await registerProjectedAiSdkBackend(register, registry, { + config: { + id: 'harbor-ai-sdk', + backend: 'ai-sdk', + llmConnectionSlug: 'openai', + model: 'gpt-5.6-sol', + systemPrompt: DEFAULT_HEADLESS_SYSTEM_PROMPT, + }, + task: { id: 'harbor-cell', instruction: 'solve', workspaceDir }, + storageRoot: workspaceDir, + workspaceDir, + artifactStore, + realBackendIsolation: { kind: 'external', label: 'Harbor task container', toolExecutor }, + toolExecutor, + ...createHeadlessSessionCapabilityBridge().capabilities, + }); + + const recorded: ModelCallAttempt[] = []; + const backend = await registry.build('ai-sdk', { + ...backendContext(workspaceDir), + recordModelCallAttempt: (attempt: ModelCallAttempt) => { + recorded.push(attempt); + return Promise.resolve(); + }, + }); + const backendInput = (backend as unknown as { input: AiSdkBackendInput }).input; + + assert.equal(typeof backendInput.recordModelCallAttempt, 'function'); + await backendInput.recordModelCallAttempt?.(harborModelCallAttemptFixture()); + assert.equal(recorded.length, 1, 'the sink must reach the controller'); + assert.equal(recorded[0]?.callKind, 'semantic_compact'); + }); + }); + test('Harbor ai-sdk backend registration exposes native file tools to the provider schema', async () => { await withDirs(async ({ workspaceDir, artifactStore }) => { const registry = new BackendRegistry(); @@ -4780,6 +4832,29 @@ function sha256(text: string): string { return createHash('sha256').update(text).digest('hex'); } +function harborModelCallAttemptFixture(): ModelCallAttempt { + return { + schemaVersion: MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + logicalCallId: 'call-1', + attemptId: 'attempt-1', + traceId: 'trace-1', + sessionId: 'session-1', + runId: 'run-1', + turnId: 'turn-1', + step: 0, + attempt: 0, + callKind: 'semantic_compact', + providerId: 'openai', + modelId: 'gpt-5.6-sol', + startedAt: 1, + completedAt: 2, + latencyMs: 1, + status: 'completed', + usageBasis: 'missing', + costBasis: 'unpriced', + }; +} + function backendContext(workspaceDir: string): BackendFactoryContext { return { sessionId: 'session-1', diff --git a/packages/headless/src/harbor-cell.ts b/packages/headless/src/harbor-cell.ts index 98d5a821d2..92f157db98 100644 --- a/packages/headless/src/harbor-cell.ts +++ b/packages/headless/src/harbor-cell.ts @@ -1192,6 +1192,11 @@ export function buildAiSdkCellBackendRegistration(input: { newId: input.newId, now: input.now, recordRunTrace: ctx.recordRunTrace, + // The canonical metering sink (#1679); the controller has always + // exposed it, this composition just never passed it through. + ...(ctx.recordModelCallAttempt + ? { recordModelCallAttempt: ctx.recordModelCallAttempt } + : {}), ...(ctx.recordProviderRequestCapture ? { recordProviderRequestCapture: createProviderRequestCaptureRecorder({ diff --git a/packages/headless/src/provider-request-trace.ts b/packages/headless/src/provider-request-trace.ts index 08ea63d7bf..366347b381 100644 --- a/packages/headless/src/provider-request-trace.ts +++ b/packages/headless/src/provider-request-trace.ts @@ -271,8 +271,12 @@ export function assertProviderRequestTraceComplete( if (!identities.some((candidate) => candidate.turnId === attempt.turnId)) { fail(`attempt ${attempt.attemptId} has another turn id`); } + // Capture ids are optional on the record since #1679 — an attempt made in a + // deployment with capture switched off has none. This analysis reads a + // capture ledger, so an attempt that cannot name one is incomplete here; + // the decoder above already rejects such records as invalid_attempt. const capture = - captures.get(attempt.captureId) ?? + (attempt.captureId !== undefined ? captures.get(attempt.captureId) : undefined) ?? fail(`attempt ${attempt.attemptId} does not match its request capture`); if (!attemptMatchesCapture(attempt, capture)) { fail(`attempt ${attempt.attemptId} does not match its request capture`); diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index a6f24a5006..b601f2200a 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -501,15 +501,10 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom } }; const telemetry = { - insertLlmCall: (record: Parameters[0]) => - persistTelemetry(() => input.usage.telemetry.recordLlmCall(record)), insertToolInvocation: ( record: Parameters[0], ) => persistTelemetry(() => input.usage.telemetry.recordToolInvocation(record)), }; - const recordLlmUsage = (event: Parameters[1]) => { - void recordLlmCall({ repo: telemetry, lookupPricing: pricing }, event); - }; /** * One canonical record, one commit point (#1679). * @@ -637,24 +632,6 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom modelId: target.model, }), providerOptions, - ...(providerRequestCapture - ? { - providerRequestTracking: { - now: Date.now, - newId: randomUUID, - persistCapture: providerRequestCapture, - recordAttempt: recordProviderRequestAttempt, - }, - } - : {}), - telemetry: { - connectionSlug: target.connection.slug, - providerId: target.connection.providerType, - modelId: target.model, - newId: randomUUID, - now: Date.now, - recordLlmCall: recordLlmUsage, - }, }), recordHistoryCompactCheckpoint: input.context.recordHistoryCompactCheckpoint, loadTurnRuntimeEvents: input.context.loadTurnRuntimeEvents, @@ -666,7 +643,6 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom turnTailPrompt: modelComposition.turnTailPrompt, shellRunContextSummary: input.context.shellRunContextSummary, lookupPricing: pricing, - recordLlmCall: recordLlmUsage, recordModelCallAttempt, assertModelCallAccountingReady, recordToolInvocation: (event) => recordToolInvocation({ repo: telemetry }, event), diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index c02c2ce72c..bf76ad04f8 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -20,7 +20,6 @@ import { projectRuntimeEventsToStoredMessages } from '../runtime-event-read-mode import { materializeSession } from '../materializer.js'; import type { InvocationContext } from '../invocation-context.js'; import type { AssistantMessage, StoredMessage, ToolResultMessage } from '@maka/core/session'; -import type { LlmCallRecord } from '@maka/core/usage-stats/types'; import { z } from 'zod'; import { AiSdkBackend, @@ -74,6 +73,7 @@ import type { ProviderRequestAttemptRecord, ProviderRequestCaptureRecord, } from '../provider-request-telemetry.js'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; describe('AiSdkBackend model history', () => { @@ -3979,6 +3979,7 @@ describe('AiSdkBackend model history', () => { const runtimeContext = [...oldEvents, recentEvent]; const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext, }); @@ -3995,6 +3996,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-overflow-recovery', + runId: 'run-1', runtimeContext, minRecentTurns: 0, }); @@ -4040,6 +4042,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'default-policy-manual-old-1', @@ -4097,6 +4100,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'manual-v2-old-1', @@ -4186,6 +4190,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...oldEvents, runtimeTextEvent({ @@ -4263,6 +4268,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...oldEvents, runtimeTextEvent({ @@ -4345,6 +4351,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...oldEvents, runtimeTextEvent({ @@ -4391,6 +4398,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'manual-v2-envelope-old-1', @@ -4448,6 +4456,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'manual-v2-larger-old-1', @@ -4509,6 +4518,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'output-length-old', @@ -4600,6 +4610,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ ...covered, runtimeTextEvent({ @@ -4646,6 +4657,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4690,6 +4702,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4759,6 +4772,7 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: oldEvents, }); @@ -4819,6 +4833,7 @@ describe('AiSdkBackend model history', () => { const compactPromise = backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4904,6 +4919,7 @@ describe('AiSdkBackend model history', () => { const compactPromise = backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -4982,6 +4998,7 @@ describe('AiSdkBackend model history', () => { await backend.compactHistory({ turnId: 'turn-compact', + runId: 'run-1', runtimeContext: [ runtimeTextEvent({ id: 'old-1', @@ -7485,7 +7502,6 @@ describe('AiSdkBackend usage telemetry', () => { test('records cumulative usage checkpoints across tool-loop steps and turns', async () => { const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const usageCheckpoints: Array<{ inputTokens: number; outputTokens: number }> = []; const firstTurn = durableTurnHarness('turn-1', 'hi'); const secondTurn = durableTurnHarness('turn-2', 'continue'); @@ -7570,9 +7586,6 @@ describe('AiSdkBackend usage telemetry', () => { : secondTurn.loadTurnRuntimeEvents(turnId), newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record: LlmCallRecord) => { - llmRecords.push(record); - }, recordUsageCheckpoint: async (usage: { inputTokens: number; outputTokens: number }) => { usageCheckpoints.push(usage); }, @@ -7727,7 +7740,6 @@ describe('AiSdkBackend usage telemetry', () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const largeBody = 'SECRET_PAYLOAD_SHOULD_BE_ARCHIVED'.repeat(200); let streamCalls = 0; const prompts: unknown[] = []; @@ -7819,9 +7831,6 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, }); for await (const event of backend.send(durable.input())) { @@ -7856,7 +7865,6 @@ describe('AiSdkBackend usage telemetry', () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const recordedBlocks: ActiveFullCompactBlock[] = []; const largeBody = 'ACTIVE_FULL_COMPACT_RAW_TOOL_OUTPUT'.repeat(200); let streamCalls = 0; @@ -7936,9 +7944,6 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, recordActiveFullCompactBlock: (block) => { recordedBlocks.push(block); }, @@ -8043,60 +8048,18 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(recordedBlocks[0]?.blockId, 'afcompact-sync-test'); }); - test('does not record semantic compact usage when provider usage is unavailable', () => { - const llmRecords: LlmCallRecord[] = []; - const backend = createTestAiSdkBackend({ - sessionId: 'session-1', - header: header(), - appendMessage: async () => {}, - connection: connection(), - apiKey: 'sk-test', - modelId: 'mock-model-id', - modelFactory: () => completionModel(), - tools: [], - newId: idGenerator(), - now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, - }); - - ( - backend as unknown as { - compaction: { - recordSemanticCompactSummaryCall(input: { - callId: string; - turnId: string; - modelId: string; - startedAt: number; - latencyMs: number; - status: LlmCallRecord['status']; - }): void; - }; - } - ).compaction.recordSemanticCompactSummaryCall({ - callId: 'semantic-1', - turnId: 'turn-1', - modelId: 'mock-model-id', - startedAt: 1, - latencyMs: 2, - status: 'error', - }); - - assert.deepEqual(llmRecords, []); - }); - - test('semantic compact records a separate no-tools summarizer LLM call', async () => { - const durable = durableTurnHarness('turn-1', 'hi'); - const messages: unknown[] = []; - const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; - const recordedBlocks: SemanticCompactBlock[] = []; - const recordedActiveFullBlocks: ActiveFullCompactBlock[] = []; - const largeBody = 'SEMANTIC_COMPACT_RAW_TOOL_OUTPUT'.repeat(180); + /** + * One model whose stream drives a turn into semantic compaction: two tool + * steps, then a plain finish. Shared by the accept and dry-run cases so both + * exercise the same summarization, and only the policy differs. + */ + function semanticCompactFixtureModel(modelId?: string): { + model: MockLanguageModelV4; + streamCalls: () => number; + } { let streamCalls = 0; - let archiveCalls = 0; const model = new MockLanguageModelV4({ + ...(modelId ? { modelId } : {}), doGenerate: { content: [ { @@ -8114,6 +8077,14 @@ describe('AiSdkBackend usage telemetry', () => { usage: { inputTokens: { total: 21, noCache: 19, cacheRead: 2, cacheWrite: 0 }, outputTokens: { total: 13, text: 13, reasoning: 0 }, + // The provider's own payload, which is the only thing the canonical + // record will attribute cache tokens from. + raw: { + input_tokens: 19, + output_tokens: 13, + cache_read_input_tokens: 2, + cache_creation_input_tokens: 0, + }, }, warnings: [], }, @@ -8172,6 +8143,49 @@ describe('AiSdkBackend usage telemetry', () => { }; }, }); + return { model, streamCalls: () => streamCalls }; + } + + function semanticCompactContextBudget(mode: 'replace' | 'validate_only') { + return { + charsPerToken: 1, + activeToolResultPrune: { enabled: true, maxCurrentResultEstimatedTokens: 1 }, + semanticCompact: { + enabled: true, + mode, + minStepNumber: 1, + minRecentMessages: 0, + maxActiveEstimatedTokens: 1, + highWaterRatio: 0.1, + minSafePrefixEstimatedTokens: 1, + minNewPrefixEstimatedTokens: 1, + maxSummaryEstimatedTokens: 1024, + minSavingsTokens: 1, + minSavingsRatio: 0, + }, + activeFullCompact: { + enabled: true, + minStepNumber: 1, + maxActiveEstimatedTokens: 1_000_000, + highWaterRatio: 0.1, + minRecentMessages: 0, + maxSummaryEstimatedTokens: 1024, + }, + } as const; + } + + const SEMANTIC_COMPACT_LARGE_BODY = 'SEMANTIC_COMPACT_RAW_TOOL_OUTPUT'.repeat(180); + + test('semantic compact records a separate no-tools summarizer model call', async () => { + const durable = durableTurnHarness('turn-1', 'hi'); + const messages: unknown[] = []; + const events: SessionEvent[] = []; + const modelCalls: ModelCallAttempt[] = []; + const recordedBlocks: SemanticCompactBlock[] = []; + const recordedActiveFullBlocks: ActiveFullCompactBlock[] = []; + const largeBody = SEMANTIC_COMPACT_LARGE_BODY; + let archiveCalls = 0; + const { model, streamCalls } = semanticCompactFixtureModel(); const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), @@ -8192,31 +8206,7 @@ describe('AiSdkBackend usage telemetry', () => { }), }, ], - contextBudget: { - charsPerToken: 1, - activeToolResultPrune: { enabled: true, maxCurrentResultEstimatedTokens: 1 }, - semanticCompact: { - enabled: true, - mode: 'replace', - minStepNumber: 1, - minRecentMessages: 0, - maxActiveEstimatedTokens: 1, - highWaterRatio: 0.1, - minSafePrefixEstimatedTokens: 1, - minNewPrefixEstimatedTokens: 1, - maxSummaryEstimatedTokens: 1024, - minSavingsTokens: 1, - minSavingsRatio: 0, - }, - activeFullCompact: { - enabled: true, - minStepNumber: 1, - maxActiveEstimatedTokens: 1_000_000, - highWaterRatio: 0.1, - minRecentMessages: 0, - maxSummaryEstimatedTokens: 1024, - }, - }, + contextBudget: semanticCompactContextBudget('replace'), archiveToolResult: async () => { archiveCalls += 1; return { artifactId: 'archived-covered-semantic-result' }; @@ -8224,8 +8214,9 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-semantic-capture' }), + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); }, recordSemanticCompactBlock: (block) => { recordedBlocks.push(block); @@ -8235,12 +8226,14 @@ describe('AiSdkBackend usage telemetry', () => { }, }); - for await (const event of backend.send(durable.input())) { + // A canonical record belongs to a run; the send carries the one the + // harness's runtime events already claim. + for await (const event of backend.send(durable.input({ runId: 'run-1' }))) { durable.record(event); events.push(event); } - assert.equal(streamCalls, 3); + assert.equal(streamCalls(), 3); assert.equal(model.doGenerateCalls.length, 1); assert.match( JSON.stringify(model.doGenerateCalls[0]?.prompt), @@ -8304,13 +8297,33 @@ describe('AiSdkBackend usage telemetry', () => { 'projection replay after active pruning must retain the exact single user anchor', ); - const semanticRecord = llmRecords.find((record) => record.callKind === 'semantic_compact'); - assert.ok(semanticRecord, 'expected semantic compact LLM record'); - assert.match(semanticRecord.callId ?? '', /^semantic_compact_turn-1_2_/); - assert.equal(semanticRecord.inputTokens, 21); - assert.equal(semanticRecord.outputTokens, 13); - assert.equal(semanticRecord.cacheHitInputTokens, 2); - assert.equal(semanticRecord.totalTokens, 34); + // The summarization is one physical provider request, metered through the + // same canonical seam as the send it interrupts (#1679) — not a hand-built + // row in the frozen usage table. + const semanticAttempt = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'semantic_compact'); + assert.ok(semanticAttempt, 'expected a canonical semantic compact record'); + assert.equal(semanticAttempt.sessionId, 'session-1'); + assert.equal(semanticAttempt.runId, 'run-1'); + assert.equal(semanticAttempt.turnId, 'turn-1'); + assert.equal( + semanticAttempt.step, + 2, + 'the record carries the send step whose projection triggered the summarization', + ); + assert.equal(semanticAttempt.status, 'completed'); + assert.equal(semanticAttempt.inputTokens, 21); + assert.equal(semanticAttempt.outputTokens, 13); + assert.equal(semanticAttempt.usageBasis, 'reported'); + // Cache tokens are attributed from the provider's own payload, never from + // the SDK's normalized view — which is what the old row copied through. + assert.equal(semanticAttempt.cacheReadInputTokens, 2); + assert.equal( + modelCalls.filter((attempt) => attempt.callKind === 'semantic_compact').length, + 1, + 'one summarization is one record', + ); const usageEvent = events.find((event) => event.type === 'token_usage') as | (Extract & { @@ -8331,6 +8344,157 @@ describe('AiSdkBackend usage telemetry', () => { ); }); + test('a dry-run semantic compaction is still a real, billed model call', async () => { + // `validate_only` declines the *projection*, not the summarization: the + // summarizer runs to completion and only then is its block refused. That + // call reaches a provider and is charged for, so it settles a canonical + // record like any other. Pinned because a mode named "dry run" that bills + // is exactly the kind of thing a later reader would assume otherwise. + const durable = durableTurnHarness('turn-1', 'hi'); + const modelCalls: ModelCallAttempt[] = []; + const recordedBlocks: SemanticCompactBlock[] = []; + const { model } = semanticCompactFixtureModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [ + { + name: 'Read', + description: 'Read description', + parameters: z.object({ path: z.string() }), + impl: async ({ path }) => ({ + body: path === 'large.log' ? SEMANTIC_COMPACT_LARGE_BODY : 'FRESH_SEMANTIC_TAIL_RESULT', + }), + }, + ], + contextBudget: semanticCompactContextBudget('validate_only'), + archiveToolResult: async () => ({ artifactId: 'archived-covered-semantic-result' }), + loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, + newId: idGenerator(), + now: monotonicClock(), + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-semantic-capture' }), + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); + }, + recordSemanticCompactBlock: (block) => { + recordedBlocks.push(block); + }, + }); + + for await (const event of backend.send(durable.input({ runId: 'run-1' }))) { + durable.record(event); + } + + assert.equal(model.doGenerateCalls.length, 1, 'the dry run still calls the summarizer'); + assert.equal(recordedBlocks.length, 0, 'and still refuses to accept the block it produced'); + const dryRunAttempt = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'semantic_compact'); + assert.ok(dryRunAttempt, 'a declined projection does not make the call free'); + assert.equal(dryRunAttempt.status, 'completed'); + assert.equal(dryRunAttempt.inputTokens, 21); + assert.equal(dryRunAttempt.outputTokens, 13); + }); + + test('a separate summarizer model is priced as itself, not as the session model', async () => { + // The record already named the model that served the request. Pricing it + // against a different model's rates would store an id and a `pricingRates` + // that contradict each other — the exact thing recording the rates is for. + const summarizerPricing = { + modelKey: 'anthropic:summarizer-model-id', + inputUsdPer1M: 1, + outputUsdPer1M: 2, + cacheReadUsdPer1M: 0.1, + cacheWriteUsdPer1M: 1, + }; + const mainPricing = { + modelKey: 'anthropic:mock-model-id', + inputUsdPer1M: 1_000, + outputUsdPer1M: 2_000, + cacheReadUsdPer1M: 100, + cacheWriteUsdPer1M: 1_000, + }; + const pricingKeysAsked: string[] = []; + const durable = durableTurnHarness('turn-1', 'hi'); + const modelCalls: ModelCallAttempt[] = []; + const { model } = semanticCompactFixtureModel(); + // A real `modelFactory` builds the summarizer against the configured id, so + // the record names it; the fixture has to do the same or it would prove + // nothing about which model the rates belong to. + const { model: summarizerOnlyModel } = semanticCompactFixtureModel('summarizer-model-id'); + const budget = semanticCompactContextBudget('replace'); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: ({ modelId }) => + modelId === 'summarizer-model-id' ? summarizerOnlyModel : model, + tools: [ + { + name: 'Read', + description: 'Read description', + parameters: z.object({ path: z.string() }), + impl: async ({ path }) => ({ + body: path === 'large.log' ? SEMANTIC_COMPACT_LARGE_BODY : 'FRESH_SEMANTIC_TAIL_RESULT', + }), + }, + ], + contextBudget: { + ...budget, + semanticCompact: { ...budget.semanticCompact, summarizerModel: 'summarizer-model-id' }, + }, + archiveToolResult: async () => ({ artifactId: 'archived-covered-semantic-result' }), + loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, + newId: idGenerator(), + now: monotonicClock(), + lookupPricing: (modelKey) => { + pricingKeysAsked.push(modelKey); + if (modelKey === summarizerPricing.modelKey) return summarizerPricing; + if (modelKey === mainPricing.modelKey) return mainPricing; + return null; + }, + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-semantic-capture' }), + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); + }, + recordSemanticCompactBlock: () => {}, + }); + + for await (const event of backend.send(durable.input({ runId: 'run-1' }))) { + durable.record(event); + } + + const summarization = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'semantic_compact'); + assert.ok(summarization, 'expected a canonical semantic compact record'); + assert.equal(summarization.modelId, 'summarizer-model-id'); + assert.equal(summarization.costBasis, 'priced'); + assert.deepEqual( + summarization.pricingRates, + summarizerPricing, + 'the rates stored are the ones the summarizer model actually bills at', + ); + assert.equal( + pricingKeysAsked.includes(summarizerPricing.modelKey), + true, + 'the cost lookup asks for the model that served the request', + ); + // The session model is still priced as itself for its own steps. + const mainCall = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .find((attempt) => attempt.callKind === 'main'); + assert.equal(mainCall?.pricingRates?.modelKey, mainPricing.modelKey); + }); + test('active full compact keeps the accepted boundary projection across later AI SDK steps', async () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; @@ -8456,7 +8620,6 @@ describe('AiSdkBackend usage telemetry', () => { const durable = durableTurnHarness('turn-1', 'hi'); const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const largeBody = 'VALIDATE_ONLY_RAW_TOOL_OUTPUT'.repeat(80); let streamCalls = 0; const model = new MockLanguageModelV4({ @@ -8530,9 +8693,6 @@ describe('AiSdkBackend usage telemetry', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordLlmCall: (record) => { - llmRecords.push(record); - }, }); for await (const event of backend.send(durable.input())) { @@ -8578,7 +8738,6 @@ describe('AiSdkBackend usage telemetry', () => { test('normalizes cache and reasoning tokens to messages, events, and telemetry', async () => { const messages: unknown[] = []; const events: SessionEvent[] = []; - const llmRecords: LlmCallRecord[] = []; const runTraceEvents: Array<{ type: string; data?: Record }> = []; let pricingLookupCalls = 0; const pricing = { @@ -8638,9 +8797,6 @@ describe('AiSdkBackend usage telemetry', () => { pricingLookupCalls += 1; return modelKey === pricing.modelKey ? pricing : null; }, - recordLlmCall: (record) => { - llmRecords.push(record); - }, recordRunTrace: (event) => { runTraceEvents.push(event); }, diff --git a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts index 04d15f7e0c..c98e1fbd12 100644 --- a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts +++ b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts @@ -4,11 +4,13 @@ * * Run: `npm --workspace @maka/runtime run test` */ +import { MockLanguageModelV4 } from 'ai/test'; import { describe, test } from 'node:test'; import assert from 'node:assert/strict'; import { expect } from '../test-helpers.js'; import type { RuntimeEvent, RuntimeEventContent } from '@maka/core/runtime-event'; -import type { LlmCallRecord } from '@maka/core/usage-stats/types'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; +import { ProviderRequestTracker } from '../provider-request-telemetry.js'; import type { HistoryCompactSummaryInput } from '../ai-sdk-compaction-contract.js'; import { buildLlmHistorySummarizer, @@ -96,63 +98,67 @@ describe('buildLlmHistorySummarizer', () => { expect(seen?.maxOutputTokens).toBe(undefined); }); - test('attributes provider-reported usage to one history-compaction call', async () => { - const records: LlmCallRecord[] = []; + test('attributes provider-reported usage to one canonical history-compaction record', async () => { + // history_compact used to write a per-call row into the frozen table. It + // now settles through the same seam as a main send, so the record carries + // the run it belongs to and its cost basis (#1679). + const recorded: ModelCallAttempt[] = []; let now = 100; const summarize = buildLlmHistorySummarizer({ - resolveModel: () => 'fake-model', - generateText: async () => ({ - text: '## Goal\nX', - finishReason: 'stop', - usage: { - inputTokens: 7, - outputTokens: 3, - totalTokens: 10, - }, - }), - telemetry: { - connectionSlug: 'connection', - providerId: 'provider', - modelId: 'model', - newId: () => 'call-id', + resolveModel: () => + new MockLanguageModelV4({ + doGenerate: { + content: [{ type: 'text', text: '## Goal\nX' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 7, noCache: 7, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 3, text: 3, reasoning: 0 }, + }, + warnings: [], + }, + }), + }); + + // The backend hands over a built tracker; the summarizer assembles nothing. + await summarize({ + ...inputWith([ev({ role: 'user', author: 'user', content: { kind: 'text', text: 'hi' } })]), + providerRequestTracker: new ProviderRequestTracker({ + traceId: 'trace-id', + turnId: 'turn-1', now: () => { now += 10; return now; }, - recordLlmCall: (record) => { - records.push(record); + newId: () => 'trace-id', + persistCapture: async () => ({ artifactId: 'artifact-1' }), + recordAttempt: () => {}, + accounting: { + sessionId: 'sess-1', + resolveRunId: () => 'run-1', + connectionSlug: 'connection', + providerId: 'provider', + callKind: 'history_compact', + record: (attempt: ModelCallAttempt) => { + recorded.push(attempt); + }, }, - }, + }), }); - await summarize( - inputWith([ev({ role: 'user', author: 'user', content: { kind: 'text', text: 'hi' } })]), - ); - - assert.deepEqual(records, [ - { - sessionId: 'sess-1', - turnId: 'turn-1', - callKind: 'history_compact', - callId: 'history_compact_turn-1_call-id', - connectionSlug: 'connection', - providerId: 'provider', - modelId: 'model', - inputTokens: 7, - outputTokens: 3, - cacheHitInputTokens: 0, - cacheMissInputTokens: 7, - cacheMissInputSource: 'derived', - cachedInputTokens: 0, - cacheWriteInputTokens: 0, - reasoningTokens: 0, - totalTokens: 10, - rawFinishReason: 'stop', - latencyMs: 10, - status: 'success', - startedAt: 110, - }, - ]); + const attempt = decodeModelCallAttempt(recorded[0]); + assert.equal(attempt.callKind, 'history_compact'); + assert.equal(attempt.sessionId, 'sess-1'); + assert.equal(attempt.runId, 'run-1'); + assert.equal(attempt.turnId, 'turn-1'); + assert.equal(attempt.connectionSlug, 'connection'); + assert.equal(attempt.providerId, 'provider'); + assert.equal(attempt.inputTokens, 7); + assert.equal(attempt.outputTokens, 3); + assert.equal(attempt.usageBasis, 'reported'); + // No pricing was wired, so the record says the price is unknown rather + // than claiming the summarization was free. + assert.equal(attempt.costBasis, 'unpriced'); + assert.equal(attempt.costUsd, undefined); }); test('produces schema-valid tool-result messages (toolName + wrapped output) and does not fall back', async () => { diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index b43fef64c1..bcd6ae8f2f 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -21,6 +21,8 @@ import { evaluateHistoryCompactCheckpointReplay } from '../history-compact.js'; import type { HistoryCompactCheckpoint } from '../history-compact-checkpoint.js'; import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; import { HistoryCompactSummarizerError } from '../history-compact-error.js'; +import { buildLlmHistorySummarizer } from '../history-compact-summarizer.js'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; const RAW_SPAN_ONE = 'RAW_SPAN_ONE_'.repeat(24); @@ -41,6 +43,8 @@ interface MidTurnFixture { anchor: RuntimeEvent; /** The fixture's durable RuntimeEvent ledger for the current turn/run. */ ledger: RuntimeEvent[]; + /** Canonical accounting records settled during the turn (#1679). */ + modelCalls: ModelCallAttempt[]; ledgerReads: number; events: SessionEvent[]; messages: unknown[]; @@ -75,6 +79,12 @@ interface MidTurnFixtureOptions { activeToolResultPrune?: boolean; /** Enable semantic compaction so it competes with the capacity hook. */ semanticCompact?: boolean; + /** + * Summarize through the real `buildLlmHistorySummarizer` against a mock + * provider, so the compaction settles a canonical record instead of the + * stubbed string the other cases return. + */ + meteredSummarizer?: boolean; /** Override the checkpoint recorder (e.g. to simulate a write failure). */ record?: (checkpoint: HistoryCompactCheckpoint) => void; /** Make the prior turns large so folding them rescues an over-window turn. */ @@ -242,6 +252,23 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { ledger.push(mapped); }; + const modelCalls: ModelCallAttempt[] = []; + const summarizerModel = new MockLanguageModelV4({ + doGenerate: { + content: [{ type: 'text', text: 'MID_TURN_SUMMARY_SENTINEL' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 31, noCache: 31, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 7, text: 7, reasoning: 0 }, + raw: { input_tokens: 31, output_tokens: 7 }, + }, + warnings: [], + }, + }); + const meteredSummarize = buildLlmHistorySummarizer({ + resolveModel: () => summarizerModel, + }); + const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), @@ -333,9 +360,20 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { summarizeHistoryCompact: async (input) => { fixture.summarizerCalls += 1; summarizedSources.push(JSON.stringify(input.source.foldedRuntimeEvents)); + // The real summarizer is what carries the accounting the backend hands + // it, so the metered case must go through it rather than a stub. + if (options.meteredSummarizer) return await meteredSummarize(input); const summary = options.summarize ? await options.summarize() : 'MID_TURN_SUMMARY_SENTINEL'; return summary; }, + ...(options.meteredSummarizer + ? { + recordProviderRequestCapture: async () => ({ artifactId: 'artifact-mid-turn-capture' }), + recordModelCallAttempt: (attempt: ModelCallAttempt) => { + modelCalls.push(attempt); + }, + } + : {}), recordHistoryCompactCheckpoint: (checkpoint) => { if (options.record) return options.record(checkpoint); recorded.push(checkpoint); @@ -378,6 +416,7 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { priorEvents, anchor, ledger, + modelCalls, events, messages, llmCalls, @@ -487,6 +526,39 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { assert.equal(fit.fits, true); }); + test('a mid-turn compaction settles a canonical record for the run it interrupts', async () => { + // End-to-end over the backend glue, not the summarizer in isolation: the + // accounting identity is built inside `AiSdkCompaction` from the backend's + // live `runId`, which only exists while a send is in flight. A stubbed + // resolver in a unit test cannot show that the real one resolves. + const fixture = buildFixture({ meteredSummarizer: true }); + await runFixtureTurn(fixture, consumer); + + assert.equal(fixture.summarizerCalls, 1); + const attempts = fixture.modelCalls.map((attempt) => decodeModelCallAttempt(attempt)); + const compaction = attempts.find((attempt) => attempt.callKind === 'history_compact'); + assert.ok(compaction, 'the compaction call must settle its own canonical record'); + assert.equal(compaction.sessionId, 'session-1'); + assert.equal(compaction.runId, 'run-1', 'attributed to the run the send is executing'); + assert.equal(compaction.turnId, 'turn-1'); + assert.equal(compaction.status, 'completed'); + assert.equal(compaction.usageBasis, 'reported'); + assert.equal(compaction.inputTokens, 31); + assert.equal(compaction.outputTokens, 7); + // The send's own steps meter separately: one auxiliary call is not folded + // into the turn it interrupts. + assert.equal( + attempts.filter((attempt) => attempt.callKind === 'history_compact').length, + 1, + 'one summarization is one record', + ); + assert.equal( + attempts.some((attempt) => attempt.callKind === 'main'), + true, + 'and the send it interrupts still records its own', + ); + }); + test('recovery re-projection with ctx.branch replays the checkpoint without the raw span', async () => { const fixture = buildFixture({ branch: 'lane-7' }); await runFixtureTurn(fixture, consumer); diff --git a/packages/runtime/src/__tests__/provider-request-telemetry.test.ts b/packages/runtime/src/__tests__/provider-request-telemetry.test.ts index 44ade3b870..9f2d42892d 100644 --- a/packages/runtime/src/__tests__/provider-request-telemetry.test.ts +++ b/packages/runtime/src/__tests__/provider-request-telemetry.test.ts @@ -719,6 +719,9 @@ describe('canonical model-call accounting', () => { resolveCost?: telemetry.ModelCallAccountingInput['resolveCost']; assertReady?: () => void; resolveRunId?: () => string | undefined; + /** Models a deployment with request capture switched off. */ + withoutCapture?: boolean; + recordAttempt?: (attempt: telemetry.ProviderRequestAttemptRecord) => void; }): telemetry.ProviderRequestTracker { let n = 0; return new telemetry.ProviderRequestTracker({ @@ -726,8 +729,10 @@ describe('canonical model-call accounting', () => { turnId: 'turn-1', now: () => 1_000 + n, newId: () => `id-${++n}`, - persistCapture: async () => ({ artifactId: 'artifact-1' }), - recordAttempt: () => {}, + ...(overrides.withoutCapture + ? {} + : { persistCapture: async () => ({ artifactId: 'artifact-1' }) }), + recordAttempt: overrides.recordAttempt ?? (() => {}), accounting: { sessionId: 'session-1', resolveRunId: overrides.resolveRunId ?? (() => 'run-1'), @@ -770,6 +775,69 @@ describe('canonical model-call accounting', () => { assert.equal(attempt.attempt, 0); }); + test('a call the provider reported no usage for records usageBasis missing', async () => { + // The alternative is a record claiming zero tokens, which is a measurement + // nobody made. `missing` says the call happened and the meter did not read. + const recorded: ModelCallAttempt[] = []; + const tracker = accountingTracker({ + record: (a) => { + recorded.push(a); + }, + resolveCost: () => ({ costUsd: 0.002, pricingRevision: 4 }), + }); + + await tracker.trackGenerate({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('hello'), + doGenerate: async () => ({ finishReason: 'stop' }), + }); + + const attempt = decodeModelCallAttempt(recorded[0]); + assert.equal(attempt.status, 'completed'); + assert.equal(attempt.usageBasis, 'missing'); + assert.equal(attempt.inputTokens, undefined); + assert.equal(attempt.outputTokens, undefined); + // No usage means nothing to price, whatever the resolver would have said. + assert.equal(attempt.costBasis, 'unpriced'); + assert.equal(attempt.costUsd, undefined); + }); + + test('metering survives a deployment with request capture switched off', async () => { + // Capture is a diagnostic. A record that cannot be joined to a stored + // request body is still a record of a call that really was billed, so the + // canonical seam must not be gated on the capture sink being configured. + const recorded: ModelCallAttempt[] = []; + const attempts: telemetry.ProviderRequestAttemptRecord[] = []; + const tracker = accountingTracker({ + withoutCapture: true, + record: (a) => { + recorded.push(a); + }, + recordAttempt: (a) => { + attempts.push(a); + }, + }); + + const result = await tracker.trackStream({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('hello'), + doStream: async () => ({ stream: streamOf([finishPart()]) }), + }); + await drain(result.stream); + + const attempt = decodeModelCallAttempt(recorded[0]); + assert.equal(attempt.usageBasis, 'reported'); + assert.equal(attempt.captureArtifactId, undefined, 'there is no artifact to point at'); + // The request shape is computed locally, so it does not need the sink. + assert.equal(attempts.length, 1); + assert.equal(attempts[0]?.captureId, undefined); + assert.equal(attempts[0]?.captureArtifactId, undefined); + assert.ok((attempts[0]?.requestHash?.length ?? 0) > 0); + assert.ok((attempts[0]?.requestBytes ?? 0) > 0); + }); + test('an unresolvable price records unpriced rather than zero', async () => { const recorded: ModelCallAttempt[] = []; const tracker = accountingTracker({ diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index cbcceb1178..ecb24c6f8e 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -77,6 +77,8 @@ import { buildHistoryCompactCheckpoint, type HistoryCompactCheckpoint, } from '../history-compact-checkpoint.js'; +import { buildLlmHistorySummarizer } from '../history-compact-summarizer.js'; +import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { AGENT_WORKSPACE_WORKTREE, IMPLEMENTATION_AGENT_DEFINITION, @@ -3758,6 +3760,115 @@ describe('SessionManager manual compaction', () => { await manager.stopSession(session.id, { source: 'stop_button' }); }); + test('manual compaction settles one canonical record for the run the kernel opened', async () => { + // `sessions:compact` and CLI `/compact` both land here. The call has no + // send to inherit a run from, so the kernel states the run it opened and + // the record must carry it — otherwise a real, billed summarization is + // silently unmetered (#1679). + const store = new MemorySessionStore(); + const runStore = new MemoryAgentRunStore(); + const backends = new BackendRegistry(); + const modelCalls: ModelCallAttempt[] = []; + const summarizerModel = new MockLanguageModelV4({ + doGenerate: { + content: [{ type: 'text', text: 'MANUAL_COMPACT_SUMMARY' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 41, noCache: 41, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 9, text: 9, reasoning: 0 }, + raw: { input_tokens: 41, output_tokens: 9 }, + }, + warnings: [], + }, + }); + backends.register('fake', (ctx) => + createTestAiSdkBackend({ + sessionId: ctx.sessionId, + header: ctx.header, + appendMessage: async () => {}, + connection: { + slug: 'mock-main', + providerType: 'anthropic', + defaultModel: 'mock-model-id', + }, + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => + new MockLanguageModelV4({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: 'ok '.repeat(80) }, + { type: 'text-end', id: 'text-1' }, + { + type: 'finish', + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 100, noCache: 100, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 10, text: 10, reasoning: 0 }, + }, + }, + ] as LanguageModelV4StreamPart[], + initialDelayInMs: null, + chunkDelayInMs: null, + }), + }), + }), + tools: [], + newId: nextId(), + now: nextNow(1), + contextBudget: { + name: 'manual-compact-accounting', + maxHistoryEstimatedTokens: 10_000, + minRecentTurns: 1, + charsPerToken: 1, + }, + summarizeHistoryCompact: buildLlmHistorySummarizer({ + resolveModel: () => summarizerModel, + }), + recordHistoryCompactCheckpoint: () => {}, + recordModelCallAttempt: (attempt) => { + modelCalls.push(attempt); + }, + }), + ); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: runStore, + backends, + newId: nextId(), + now: nextNow(13_000), + }); + const session = await manager.createSession( + makeInput({ backend: 'fake', permissionMode: 'bypass' }), + ); + + await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'first '.repeat(400) })); + await drain(manager.sendMessage(session.id, { turnId: 'turn-2', text: 'second' })); + await drain(manager.compactSession(session.id, { turnId: 'turn-compact' })); + + const compactRun = (await runStore.listSessionRuns(session.id)).find( + (run) => run.turnId === 'turn-compact', + ); + assert.ok(compactRun, 'the kernel opens a run for a manual compaction'); + const compactions = modelCalls + .map((attempt) => decodeModelCallAttempt(attempt)) + .filter((attempt) => attempt.callKind === 'history_compact'); + assert.equal(compactions.length, 1, 'one manual compaction is one record'); + assert.equal( + compactions[0]?.runId, + compactRun.runId, + 'attributed to the run the kernel opened, not to whatever ran last', + ); + assert.equal(compactions[0]?.turnId, 'turn-compact'); + assert.equal(compactions[0]?.inputTokens, 41); + assert.equal(compactions[0]?.usageBasis, 'reported'); + await manager.stopSession(session.id, { source: 'stop_button' }); + }); + test('persists one visible warning when manual compaction fails open', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 6bc853290b..a9760b3fd9 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -92,7 +92,6 @@ import type { UserContent, } from './model-protocol.js'; import { z } from 'zod'; -import { llmCallUsageFields } from './telemetry/llm-call-usage.js'; import { AsyncEventQueue } from './async-queue.js'; import { StreamWatchdog, formatStreamWatchdogError } from './stream-watchdog.js'; @@ -160,9 +159,10 @@ import { toolSchemaCharsForDiagnostics, type RequestShapeDiagnostic, } from './request-shape.js'; -import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; +import type { ModelCallAttempt, ModelCallKind } from '@maka/core/model-call-attempt'; import { ProviderRequestTracker, + type ModelCallAccountingInput, type ProviderRequestAttemptRecord, type ProviderRequestCaptureRecord, type ProviderRequestUsage, @@ -266,7 +266,6 @@ export type { HistoryCompactWriter, HistoryCompactWriteInput, HistoryCompactWriteResult, - LlmTelemetryRecorder, SemanticCompactBlockRecorder, SynthesisCacheLoader, SynthesisCacheLoadInput, @@ -573,7 +572,8 @@ export class AiSdkBackend implements AgentBackend { sessionId: this.sessionId, now: this.now, modelAdapter: this.modelAdapter, - computeCostUsd: (usage) => this.computeTokenUsageCostUsd(usage), + createProviderRequestTracker: (trackerInput) => + this.createProviderRequestTracker(trackerInput), materializeRuntimeReplayPlan: (plan) => this.materializeRuntimeReplayPlan(plan), canReplayProviderNative: (plan) => this.canReplayProviderNative(plan), appendTurnTailPrompt: (content, turnTailPrompt) => @@ -797,38 +797,12 @@ export class AiSdkBackend implements AgentBackend { toSandboxRunTraceProjection(this.input.sandboxDiagnosticsSnapshot), ); } - const recordProviderRequestCapture = this.input.recordProviderRequestCapture; - const providerRequestTraceId = recordProviderRequestCapture ? this.newId() : undefined; - const providerRequestTracker = providerRequestTraceId - ? new ProviderRequestTracker({ - traceId: providerRequestTraceId, - turnId, - contextWindow: resolveSelectedModelContextWindow( - this.input.connection, - this.input.modelId, - ), - now: this.now, - newId: this.newId, - persistCapture: recordProviderRequestCapture!, - recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), - ...(this.input.recordModelCallAttempt - ? { - accounting: { - sessionId: this.sessionId, - resolveRunId: () => this.currentRunId ?? undefined, - connectionSlug: this.input.connection.slug, - providerId: this.input.connection.providerType, - callKind: 'main' as const, - record: this.input.recordModelCallAttempt, - resolveCost: (usage: ProviderRequestUsage) => this.resolveModelCallCost(usage), - ...(this.input.assertModelCallAccountingReady - ? { assertReady: this.input.assertModelCallAccountingReady } - : {}), - }, - } - : {}), - }) - : undefined; + const providerRequestTracker = this.createProviderRequestTracker({ + turnId, + callKind: 'main', + modelId: this.input.modelId, + }); + const providerRequestTraceId = providerRequestTracker?.traceId; // --- Resolve model (API key already attached at construct time) --- let model: unknown; @@ -2063,6 +2037,83 @@ export class AiSdkBackend implements AgentBackend { } } + /** + * One tracker for one physical provider call kind (#1679). + * + * Auxiliary calls get the same capture, attempt, and accounting plumbing the + * main send uses, built here because the sinks and the current run live on + * this backend. Callers receive a ready tracker rather than the ingredients: + * a half-wired tracker is what produces records nothing can attribute. + * + * Absent only when there is nothing to feed: no capture sink *and* no + * canonical sink. Metering deliberately does not depend on capture — capture + * is a diagnostic, and a deployment that turns it off must still be billed. + */ + private createProviderRequestTracker(input: { + turnId: string; + callKind: ModelCallKind; + modelId: string; + runId?: string; + }): ProviderRequestTracker | undefined { + const persistCapture = this.input.recordProviderRequestCapture; + const accounting = this.modelCallAccounting(input.callKind, { + modelId: input.modelId, + ...(input.runId ? { runId: input.runId } : {}), + }); + if (!persistCapture && !accounting) return undefined; + return new ProviderRequestTracker({ + traceId: this.newId(), + turnId: input.turnId, + contextWindow: resolveSelectedModelContextWindow(this.input.connection, input.modelId), + now: this.now, + newId: this.newId, + ...(persistCapture ? { persistCapture } : {}), + recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), + ...(accounting ? { accounting } : {}), + }); + } + + /** + * Accounting identity for one call kind (#1679). + * + * Owned here rather than by the host that wires the summarizers, because the + * only field a host cannot supply is the one that changes per turn: `runId` + * lives on this backend. A host configures the capture and attempt plumbing + * once; the run a record belongs to is resolved at the moment the call is + * actually made. + * + * Absent when there is no canonical sink, which leaves the corresponding + * tracker purely diagnostic. + */ + modelCallAccounting( + callKind: ModelCallKind, + identity?: { + /** + * States the run explicitly for a call made outside `send()`, where there + * is no live turn to resolve it from — a manual history compaction is one. + */ + runId?: string; + /** The model this call actually runs against; priced as that model. */ + modelId?: string; + }, + ): ModelCallAccountingInput | undefined { + const record = this.input.recordModelCallAttempt; + if (!record) return undefined; + const modelId = identity?.modelId ?? this.input.modelId; + return { + sessionId: this.sessionId, + resolveRunId: () => identity?.runId ?? this.currentRunId ?? undefined, + connectionSlug: this.input.connection.slug, + providerId: this.input.connection.providerType, + callKind, + record, + resolveCost: (usage: ProviderRequestUsage) => this.resolveModelCallCost(usage, modelId), + ...(this.input.assertModelCallAccountingReady + ? { assertReady: this.input.assertModelCallAccountingReady } + : {}), + }; + } + /** * Resolves cost for a canonical accounting record at settlement time, together * with the rates it was computed against. @@ -2072,11 +2123,20 @@ export class AiSdkBackend implements AgentBackend { * cost. An unresolvable price returns `undefined` rather than zero — the * record then carries `costBasis: 'unpriced'`, which is not the same claim as * a call that was free. + * + * Priced against the model that actually served the request, which is not + * always the session's model: a configured semantic-compact summarizer runs + * on its own. Recording one model's id beside another model's rates would + * make the stored amount unauditable in exactly the way `pricingRates` exists + * to prevent. */ - private resolveModelCallCost(usage: ProviderRequestUsage): ResolvedModelCallCost | undefined { + private resolveModelCallCost( + usage: ProviderRequestUsage, + modelId: string, + ): ResolvedModelCallCost | undefined { try { const pricing = (this.input.lookupPricing ?? getBuiltinPricing)( - `${this.input.connection.providerType}:${this.input.modelId}`, + `${this.input.connection.providerType}:${modelId}`, ); if (pricing === null) return undefined; const costUsd = computeCost( diff --git a/packages/runtime/src/ai-sdk-compaction-contract.ts b/packages/runtime/src/ai-sdk-compaction-contract.ts index 3670c9b6bf..eafed7f31b 100644 --- a/packages/runtime/src/ai-sdk-compaction-contract.ts +++ b/packages/runtime/src/ai-sdk-compaction-contract.ts @@ -1,7 +1,7 @@ import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { LlmCallRecord } from '@maka/core/usage-stats/types'; +import type { ProviderRequestTracker } from './provider-request-telemetry.js'; import type { ActiveFullCompactBlock } from './active-full-compact.js'; import type { ActiveToolResultArchiveCandidate } from './active-tool-result-prune.js'; import type { @@ -16,8 +16,6 @@ import type { HistoryCompactCheckpoint } from './history-compact-checkpoint.js'; import type { ModelFactory } from './model-adapter.js'; import type { SemanticCompactBlock } from './semantic-compact.js'; -export type LlmTelemetryRecorder = (record: LlmCallRecord) => void; - export type ToolResultArchiveRecorderInput = ( | StaleToolResultArchiveCandidate | (ActiveToolResultArchiveCandidate & { runtimeEventId: string }) @@ -120,6 +118,15 @@ export interface HistoryCompactSummaryInput { newlyFoldedRuntimeEvents?: RuntimeEvent[]; requestShapeHashBefore?: string; abortSignal?: AbortSignal; + /** + * Physical-call tracking for this summarization, built by the backend (#1679). + * + * A *ready* tracker, not the parts to assemble one: the products that wire a + * summarizer cannot know the run a call belongs to, and every root that had to + * assemble it independently eventually forgot a piece. Absent when the product + * supplied no canonical sink, which leaves the call untracked. + */ + providerRequestTracker?: ProviderRequestTracker; } export type HistoryCompactSummarizer = ( input: HistoryCompactSummaryInput, @@ -145,8 +152,6 @@ export interface AiSdkCompactionCapabilities { modelFactory: ModelFactory; /** Optional prior-history budget. Keeps whole turns to preserve tool-call/result pairs. */ contextBudget?: ContextBudgetPolicy; - /** Optional fire-and-forget LLM telemetry hook. */ - recordLlmCall?: LlmTelemetryRecorder; /** * Optional archive writer for replay-only stale tool-result pruning. The * runtime rewrites only candidates whose original body has been durably diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 47e5ee8ce3..4337293ca5 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -15,7 +15,7 @@ import type { BackendCompactHistoryResult, BackendSendInput, } from '@maka/core/backend-types'; -import type { ContextBudgetDiagnostic, LlmCallRecord } from '@maka/core/usage-stats/types'; +import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; import type { AiSdkCompactionCapabilities } from './ai-sdk-compaction-contract.js'; import { @@ -49,12 +49,7 @@ import { import { createHash } from 'node:crypto'; import type { ModelMessage } from './model-protocol.js'; -import { - normalizeAiSdkUsage, - type ModelAdapter, - type NormalizedAiSdkUsage, -} from './model-adapter.js'; -import { llmCallUsageFields } from './telemetry/llm-call-usage.js'; +import type { ModelAdapter } from './model-adapter.js'; import type { RequestProjection, RequestProjectionContext, @@ -87,6 +82,8 @@ import { type RuntimeEventModelReplayPlan, } from './model-history.js'; import { toolSchemaCharsForDiagnostics } from './request-shape.js'; +import type { ModelCallKind } from '@maka/core/model-call-attempt'; +import type { ProviderRequestTracker } from './provider-request-telemetry.js'; import { estimateNextRequestTokens, exceedsHighWater, @@ -100,7 +97,17 @@ export interface AiSdkCompactionDeps { sessionId: string; now: () => number; modelAdapter: ModelAdapter; - computeCostUsd: (usage: NormalizedAiSdkUsage) => number | undefined; + /** + * A ready tracker for a compaction call that has none of its own. The backend + * hands over the built tracker rather than the capture, attempt, and id sinks + * it is made of: compaction has no business assembling metering identity. + */ + createProviderRequestTracker: (input: { + turnId: string; + callKind: ModelCallKind; + modelId: string; + runId?: string; + }) => ProviderRequestTracker | undefined; materializeRuntimeReplayPlan: (plan: RuntimeEventModelReplayPlan) => Promise; canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; appendTurnTailPrompt: ( @@ -114,7 +121,12 @@ export class AiSdkCompaction { private readonly sessionId: string; private readonly now: () => number; private readonly modelAdapter: ModelAdapter; - private readonly computeCostUsd: (usage: NormalizedAiSdkUsage) => number | undefined; + private readonly createProviderRequestTracker: (input: { + turnId: string; + callKind: ModelCallKind; + modelId: string; + runId?: string; + }) => ProviderRequestTracker | undefined; private readonly materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, ) => Promise; @@ -130,7 +142,7 @@ export class AiSdkCompaction { this.sessionId = deps.sessionId; this.now = deps.now; this.modelAdapter = deps.modelAdapter; - this.computeCostUsd = deps.computeCostUsd; + this.createProviderRequestTracker = deps.createProviderRequestTracker; this.materializeRuntimeReplayPlan = deps.materializeRuntimeReplayPlan; this.canReplayProviderNative = deps.canReplayProviderNative; this.appendTurnTailPrompt = deps.appendTurnTailPrompt; @@ -374,6 +386,11 @@ export class AiSdkCompaction { public async writeHistoryCompactCheckpoint(input: { requestShapeHashBefore?: string; turnId: string; + /** + * Present for a manual compaction, which runs outside `send()` and so has + * no live run for the backend to resolve. Absent mid-send, where it does. + */ + runId?: string; contextBudget: ContextBudgetPolicy; priorRuntimeContext: readonly RuntimeEvent[]; draftBlock: HistoryCompactBlock; @@ -386,6 +403,13 @@ export class AiSdkCompaction { const summarizer = this.input.summarizeHistoryCompact; const recorder = this.input.recordHistoryCompactCheckpoint; if (!summarizer || !recorder) return { diagnosticPatch: {} }; + // One tracker for this summarization, built where every input lives. + const historyCompactTracker = this.createProviderRequestTracker({ + turnId: input.turnId, + callKind: 'history_compact', + modelId: this.input.modelId, + ...(input.runId ? { runId: input.runId } : {}), + }); const foldedIds = new Set(input.draftBlock.coverage.runtimeEventIds); const foldedRuntimeEvents = input.priorRuntimeContext.filter((event) => foldedIds.has(event.id), @@ -461,6 +485,7 @@ export class AiSdkCompaction { newlyFoldedRuntimeEvents, requestShapeHashBefore: input.requestShapeHashBefore, abortSignal: input.abortSignal, + ...(historyCompactTracker ? { providerRequestTracker: historyCompactTracker } : {}), }), ); if (!summary?.trim()) { @@ -758,6 +783,8 @@ export class AiSdkCompaction { } const writePatch = await this.writeHistoryCompactCheckpoint({ turnId: input.turnId, + // A manual compaction names its own run: nothing else can (#1679). + runId: input.runId, contextBudget: writeContextBudget, priorRuntimeContext: runtimeContext, draftBlock: draftBlocks[0]!, @@ -987,6 +1014,24 @@ export class AiSdkCompaction { compactCallTotalTokens: 0, acceptedEstimatedTokensSaved: 0, }; + // One auxiliary trace per turn rather than per call: a step that summarizes + // is a step of that trace, so a retried summarization is another attempt of + // the same logical call. Built on first use — most turns never summarize, + // and an unused trace id is a trace that never happened. + const summarizerModelId = policy.summarizerModel ?? this.input.modelId; + let summaryTracker: ProviderRequestTracker | undefined; + let summaryTrackerBuilt = false; + const resolveSummaryTracker = (): ProviderRequestTracker | undefined => { + if (!summaryTrackerBuilt) { + summaryTrackerBuilt = true; + summaryTracker = this.createProviderRequestTracker({ + turnId, + callKind: 'semantic_compact', + modelId: summarizerModelId, + }); + } + return summaryTracker; + }; return async (options) => { const activeToolsForStep = options.activeTools; const dryRun = policy.mode === 'validate_only' || policy.mode === 'prepare_step_dry_run'; @@ -1002,7 +1047,6 @@ export class AiSdkCompaction { modelId: policy.summarizerModel, }) : model; - const summarizerModelId = policy.summarizerModel ?? this.input.modelId; const rewritten = await rewriteSemanticCompactInMessages({ sessionId: this.sessionId, turnId, @@ -1021,38 +1065,19 @@ export class AiSdkCompaction { : {}), abortSignal: abortSignal, summarizer: async (request) => { - const startedAt = this.now(); - const callId = `semantic_compact_${turnId}_${options.stepNumber}_${startedAt}`; - try { - const result = await this.modelAdapter.generateCompactSummary({ - model: summarizerModel, - system: request.system, - messages: request.messages, - maxOutputTokens: request.maxOutputTokens, - abortSignal: request.abortSignal, - }); - this.recordSemanticCompactSummaryCall({ - callId, - turnId, - modelId: summarizerModelId, - startedAt, - latencyMs: Math.max(0, this.now() - startedAt), - usage: result.usage, - status: 'success', - }); - return result; - } catch (error) { - this.recordSemanticCompactSummaryCall({ - callId, - turnId, - modelId: summarizerModelId, - startedAt, - latencyMs: Math.max(0, this.now() - startedAt), - status: request.abortSignal?.aborted ? 'aborted' : 'error', - errorClass: this.modelAdapter.classifyError(error), - }); - throw error; - } + // The tracker settles this call itself, on the success, failure, and + // abort paths alike, so the summarization is metered as the physical + // provider request it is instead of a hand-built row (#1679). + const tracker = resolveSummaryTracker(); + tracker?.setStep(options.stepNumber); + return await this.modelAdapter.generateCompactSummary({ + model: summarizerModel, + system: request.system, + messages: request.messages, + maxOutputTokens: request.maxOutputTokens, + abortSignal: request.abortSignal, + ...(tracker ? { providerRequestTracker: tracker } : {}), + }); }, }); onDiagnosticPatch?.({ @@ -1134,35 +1159,6 @@ export class AiSdkCompaction { }; } - private recordSemanticCompactSummaryCall(input: { - callId: string; - turnId: string; - modelId: string; - startedAt: number; - latencyMs: number; - usage?: NormalizedAiSdkUsage; - status: LlmCallRecord['status']; - errorClass?: string; - }): void { - if (!input.usage) return; - const costUsd = this.computeCostUsd(input.usage); - this.input.recordLlmCall?.({ - sessionId: this.sessionId, - turnId: input.turnId, - callKind: 'semantic_compact', - callId: input.callId, - connectionSlug: this.input.connection.slug, - providerId: this.input.connection.providerType, - modelId: input.modelId, - ...llmCallUsageFields(input.usage), - latencyMs: input.latencyMs, - status: input.status, - ...(input.errorClass ? { errorClass: input.errorClass } : {}), - startedAt: input.startedAt, - ...(costUsd !== undefined ? { costUsd } : {}), - }); - } - private recordSemanticCompactBlock(block: SemanticCompactBlock): void { const recorder = this.input.recordSemanticCompactBlock; if (!recorder) return; @@ -1466,6 +1462,11 @@ export class AiSdkCompaction { abortSignal, } = input; const summarizer = this.input.summarizeHistoryCompact!; + const midTurnTracker = this.createProviderRequestTracker({ + turnId, + callKind: 'history_compact', + modelId: this.input.modelId, + }); const recorder = this.input.recordHistoryCompactCheckpoint!; const loadTurnRuntimeEvents = this.input.loadTurnRuntimeEvents!; const policy = this.input.contextBudget!; @@ -1563,6 +1564,7 @@ export class AiSdkCompaction { ...(previousCheckpoint ? { previousCheckpoint } : {}), newlyFoldedRuntimeEvents: [...newlyFoldedRuntimeEvents], ...(abortSignal ? { abortSignal } : {}), + ...(midTurnTracker ? { providerRequestTracker: midTurnTracker } : {}), }), ); }, diff --git a/packages/runtime/src/history-compact-summarizer.ts b/packages/runtime/src/history-compact-summarizer.ts index 07bda16f8c..6bafd48ed9 100644 --- a/packages/runtime/src/history-compact-summarizer.ts +++ b/packages/runtime/src/history-compact-summarizer.ts @@ -1,18 +1,10 @@ import { rawFinishReasonString, type ModelMessage } from './model-protocol.js'; import { buildRuntimeEventModelReplayPlan } from './model-history.js'; import { toolResultOutput } from './tool-result-output.js'; -import type { - HistoryCompactSummaryInput, - LlmTelemetryRecorder, -} from './ai-sdk-compaction-contract.js'; +import type { HistoryCompactSummaryInput } from './ai-sdk-compaction-contract.js'; import { HistoryCompactSummarizerError } from './history-compact-error.js'; -import { normalizeAiSdkUsage, type AiSdkUsageLike } from './model-adapter.js'; -import { - ProviderRequestTracker, - type ProviderGenerateResult, - type ProviderRequestTrackerInput, -} from './provider-request-telemetry.js'; -import { llmCallUsageFields } from './telemetry/llm-call-usage.js'; +import type { AiSdkUsageLike } from './model-adapter.js'; +import { withProviderGenerateTracking } from './provider-request-telemetry.js'; export { HistoryCompactSummarizerError } from './history-compact-error.js'; @@ -29,12 +21,6 @@ export type AiSdkGenerateTextLike = ( options: AiSdkGenerateTextOptions, ) => Promise<{ text: string; finishReason?: unknown; usage?: AiSdkUsageLike }>; -interface ProviderMiddlewareGenerateInput { - doGenerate: () => PromiseLike; - params: Record & { abortSignal?: AbortSignal }; - model: { provider: string; modelId: string }; -} - export interface BuildLlmHistorySummarizerOptions { /** Resolve the AI SDK model used for summarization. Reuses the session model. */ resolveModel: () => unknown; @@ -42,17 +28,6 @@ export interface BuildLlmHistorySummarizerOptions { providerOptions?: Record; /** Injectable `generateText` for tests; defaults to the real AI SDK export. */ generateText?: AiSdkGenerateTextLike; - /** Physical provider-call capture and attempt tracking for generated summaries. */ - providerRequestTracking?: Omit; - /** Usage attribution for the auxiliary history-compaction call. */ - telemetry?: { - connectionSlug: string; - providerId: string; - modelId: string; - newId: () => string; - now: () => number; - recordLlmCall: LlmTelemetryRecorder; - }; } // Conversation-summarization prompt (sectioned, modelled on pi/opencode): @@ -106,36 +81,20 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti ], }); } - const providerRequestTracker = options.providerRequestTracking - ? new ProviderRequestTracker({ - ...options.providerRequestTracking, - traceId: options.providerRequestTracking.newId(), - turnId: input.turnId, - }) - : undefined; + // Handed over whole by the backend, which owns every input a tracker + // needs — including the run, which no summarizer wiring can know (#1679). + const providerRequestTracker = input.providerRequestTracker; const ai = options.generateText && !providerRequestTracker ? undefined : await loadAiSdkTextModule(); const generateText = options.generateText ?? ai!.generateText; const model = providerRequestTracker - ? ai!.wrapLanguageModel({ + ? withProviderGenerateTracking({ model: options.resolveModel(), - middleware: { - wrapGenerate: async ({ - doGenerate, - params, - model: providerModel, - }: ProviderMiddlewareGenerateInput) => - await providerRequestTracker.trackGenerate({ - providerId: providerModel.provider, - modelId: providerModel.modelId, - params, - abortSignal: input.abortSignal, - doGenerate, - }), - }, + wrapLanguageModel: ai!.wrapLanguageModel, + tracker: providerRequestTracker, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), }) : options.resolveModel(); - const startedAt = options.telemetry?.now(); const result = await generateText({ model, instructions: SUMMARIZATION_SYSTEM_PROMPT, @@ -145,9 +104,6 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti : {}), ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), }); - if (options.telemetry && startedAt !== undefined) { - recordHistoryCompactCall(options.telemetry, input, startedAt, result); - } if (rawFinishReasonString(result.finishReason) === 'length') { throw new HistoryCompactSummarizerError('output_length'); } @@ -159,34 +115,6 @@ export function buildLlmHistorySummarizer(options: BuildLlmHistorySummarizerOpti }; } -function recordHistoryCompactCall( - telemetry: NonNullable, - input: HistoryCompactSummaryInput, - startedAt: number, - result: Awaited>, -): void { - const usage = normalizeAiSdkUsage(result.usage, { rawFinishReason: result.finishReason }); - if (!usage) return; - const completedAt = telemetry.now(); - try { - telemetry.recordLlmCall({ - sessionId: input.sessionId, - turnId: input.turnId, - callKind: 'history_compact', - callId: `history_compact_${input.turnId}_${telemetry.newId()}`, - connectionSlug: telemetry.connectionSlug, - providerId: telemetry.providerId, - modelId: telemetry.modelId, - ...llmCallUsageFields(usage), - latencyMs: Math.max(0, completedAt - startedAt), - status: 'success', - startedAt, - }); - } catch { - // Usage telemetry is diagnostic. The summary remains authoritative. - } -} - interface AiSdkTextModule { generateText: AiSdkGenerateTextLike; wrapLanguageModel(input: Record): unknown; diff --git a/packages/runtime/src/model-adapter.ts b/packages/runtime/src/model-adapter.ts index 3449c9b517..c1ba117607 100644 --- a/packages/runtime/src/model-adapter.ts +++ b/packages/runtime/src/model-adapter.ts @@ -38,7 +38,10 @@ import { errorPresentationFromClass, providerRetryMetadata, } from './provider-error-classification.js'; -import type { ProviderRequestTracker } from './provider-request-telemetry.js'; +import { + withProviderGenerateTracking, + type ProviderRequestTracker, +} from './provider-request-telemetry.js'; import { createKimiOpenAiTransportState, kimiReasoningFieldProviderOptions, @@ -86,6 +89,12 @@ export interface CompactSummaryRequest { messages: readonly ModelMessage[]; maxOutputTokens: number; abortSignal?: AbortSignal; + /** + * Physical provider-call tracking for this summarization. Attaching it here + * rather than at the call site keeps "wrap a model with a tracker" in the one + * place that already owns it for streams. + */ + providerRequestTracker?: ProviderRequestTracker; } export interface CompactSummaryResult { @@ -106,7 +115,7 @@ export interface ModelAdapterStreamInput { toolCall: RepairableAiSdkToolCall; error: unknown; }) => RepairableAiSdkToolCall | null | Promise; - /** Main-agent provider-call tracker. Auxiliary model calls intentionally omit it. */ + /** Main-agent provider-call tracker. Auxiliary calls track their own generates. */ providerRequestTracker?: ProviderRequestTracker; } @@ -257,7 +266,7 @@ export class ModelAdapter { `Failed to load 'ai' package. Run \`npm install ai\`. Inner: ${(err as Error).message}`, ); }); - const { generateText } = ai as unknown as { + const { generateText, wrapLanguageModel } = ai as unknown as { generateText: (opts: Record) => Promise<{ text?: string; usage?: AiSdkUsageLike; @@ -265,10 +274,20 @@ export class ModelAdapter { providerMetadata?: unknown; finalStep?: { response?: { id?: string } }; }>; + wrapLanguageModel: (input: Record) => unknown; }; + const trackedModel = input.providerRequestTracker + ? withProviderGenerateTracking({ + model: input.model, + wrapLanguageModel, + tracker: input.providerRequestTracker, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), + }) + : input.model; + const result = await generateText({ - model: input.model, + model: trackedModel, instructions: input.system, messages: input.messages, maxOutputTokens: input.maxOutputTokens, diff --git a/packages/runtime/src/provider-request-telemetry.ts b/packages/runtime/src/provider-request-telemetry.ts index b20f937914..2e4d0024ac 100644 --- a/packages/runtime/src/provider-request-telemetry.ts +++ b/packages/runtime/src/provider-request-telemetry.ts @@ -63,8 +63,13 @@ export interface ProviderRequestAttemptRecord extends ProviderRequestUsage { turnId: string; step: number; attempt: number; - captureId: string; - captureArtifactId: string; + /** + * Present only when a capture sink is wired. The request shape below is + * computed locally and always present; these two are the join keys to the + * persisted artifact, so they are absent when there is nothing to join to. + */ + captureId?: string; + captureArtifactId?: string; providerId: string; modelId: string; contextWindow?: number; @@ -96,7 +101,12 @@ export interface ProviderRequestTrackerInput { contextWindow?: number; now: () => number; newId: () => string; - persistCapture: ( + /** + * Request-body capture sink. Optional because capture is a diagnostic, and + * metering must not depend on one: a deployment with capture switched off + * still settles canonical records, it just has no artifact to join them to. + */ + persistCapture?: ( capture: ProviderRequestCaptureRecord, ) => Promise>; recordAttempt: (attempt: ProviderRequestAttemptRecord) => void | Promise; @@ -163,6 +173,42 @@ export interface TrackProviderGenerateInput { doGenerate: () => PromiseLike; } +interface ProviderMiddlewareGenerateInput { + doGenerate: () => PromiseLike; + params: Record & { abortSignal?: AbortSignal }; + model: { provider: string; modelId: string }; +} + +/** + * Wraps a language model so its single `generate` call is tracked. + * + * The AI SDK's `wrapLanguageModel` is passed in rather than imported: both + * callers load the `ai` package dynamically, and there is no reason for this + * module to load it a third time. Non-streaming counterpart of what + * `ModelAdapter.startStream` does with `wrapStream`, and the only place either + * auxiliary caller attaches a tracker to a model. + */ +export function withProviderGenerateTracking(input: { + model: unknown; + wrapLanguageModel: (input: Record) => unknown; + tracker: ProviderRequestTracker; + abortSignal?: AbortSignal; +}): unknown { + return input.wrapLanguageModel({ + model: input.model, + middleware: { + wrapGenerate: async ({ doGenerate, params, model }: ProviderMiddlewareGenerateInput) => + await input.tracker.trackGenerate({ + providerId: model.provider, + modelId: model.modelId, + params, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), + doGenerate, + }), + }, + }); +} + export function createProviderRequestCaptureRecorder( input: ProviderRequestCaptureRecorderInput, ): ( @@ -192,7 +238,8 @@ export interface ProviderGenerateResult { interface StoredCapture { capture: ProviderRequestCaptureRecord; - ref: ProviderRequestCaptureRef; + /** Absent when no capture sink is wired: there is no artifact to point at. */ + ref?: ProviderRequestCaptureRef; } const CANONICAL_USAGE_FIELDS = [ @@ -388,8 +435,9 @@ export class ProviderRequestTracker { turnId: this.input.turnId, step, attempt, - captureId: capture.ref.captureId, - captureArtifactId: capture.ref.artifactId, + ...(capture.ref + ? { captureId: capture.ref.captureId, captureArtifactId: capture.ref.artifactId } + : {}), providerId: input.providerId, modelId: input.modelId, ...(contextWindow !== undefined ? { contextWindow } : {}), @@ -518,6 +566,7 @@ export class ProviderRequestTracker { const existing = this.captures.get(key); if (existing) return await existing; + const persistCapture = this.input.persistCapture; const pending = (async (): Promise => { const captureId = this.input.newId(); const capture: ProviderRequestCaptureRecord = { @@ -529,7 +578,11 @@ export class ProviderRequestTracker { providerId: input.providerId, modelId: input.modelId, }; - const persisted = await this.input.persistCapture(capture); + // The request shape on `capture` is computed here and needs no sink. Only + // the artifact join keys depend on one, so without it the attempt still + // carries hash, bytes, and segments — it just points at nothing. + if (!persistCapture) return { capture }; + const persisted = await persistCapture(capture); return { capture, ref: { captureId, artifactId: persisted.artifactId } }; })(); this.captures.set(key, pending); diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 89db4e48b7..4e5dbe6d7d 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -978,6 +978,7 @@ export class RuntimeKernel implements RuntimeKernelLike { this.assertRunCanDispatch(run, begin.backend); const result = await begin.backend.compactHistory({ turnId: run.turnId, + runId: run.runId, runtimeContext: begin.runtimeContext, ...(input.minRecentTurns !== undefined ? { minRecentTurns: input.minRecentTurns } : {}), });