From 0cd4e3309eaa532dc88f5f4d8e62b731671b3cb6 Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Fri, 31 Jul 2026 15:55:39 +0800 Subject: [PATCH 1/2] feat(core): add canonical ModelCallAttempt accounting contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the accounting record that PR 2 will write from the provider request seam, per the design settled in #1679. Contract only: no behavior change, no writer, no read path. `ModelCallAttempt` is one record per physical provider request attempt. It extends the shape `ProviderRequestAttemptRecord` already carries with the three things it lacks — call kind, resolved cost, and a logical-call link — and adds a strict subtype codec, because the generic AgentRun event decoder only checks that `data` is a record. For an accounting record an unvalidated field silently becomes a wrong number in a cost report. Two invariants are enforced by the codec rather than left to convention: - `costUsd` is absent unless `costBasis` is `'priced'`. Zero means the call was genuinely free, never that the price was unknown. - `usageBasis: 'missing'` cannot carry token fields. "The provider never reported usage" and "we have usage but no price" are separate failures and must not collapse into one counter. `logicalCallId` is explicit rather than reconstructed from `(traceId, step)`: `traceId` is per-tracker-instance, so it does not group a turn's calls, and a compound key every consumer has to rebuild is the implicit contract this record exists to remove. Retries are attempts of one logical call; terminality is derived in projection, not stored. `ModelCallCoverage` classifies the records present and documents that it is not a completeness proof — nothing records an expected dispatch count, so an attempt lost between dispatch and settlement is undetectable. `sumModelCallCostUsd` returns coverage alongside the total for the same reason: a bare number cannot express "plus an unknown amount from unpriced calls". Registers `model_call_attempt_recorded` in `AGENT_RUN_EVENT_TYPES`, so the record rides the existing AgentRun event seam — no new table, store, or writer, and nothing for the #1649 persistence cutover to coordinate with. Co-Authored-By: Claude Opus 5 --- packages/core/package.json | 1 + .../src/__tests__/model-call-attempt.test.ts | 205 ++++++++++ packages/core/src/agent-run.ts | 1 + packages/core/src/index.ts | 26 ++ packages/core/src/model-call-attempt.ts | 361 ++++++++++++++++++ 5 files changed, 594 insertions(+) create mode 100644 packages/core/src/__tests__/model-call-attempt.test.ts create mode 100644 packages/core/src/model-call-attempt.ts diff --git a/packages/core/package.json b/packages/core/package.json index d6bff19c401..02b480689d3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -24,6 +24,7 @@ "./orchestration": "./dist/orchestration.js", "./plan": "./dist/plan.js", "./agent-run": "./dist/agent-run.js", + "./model-call-attempt": "./dist/model-call-attempt.js", "./shell-run": "./dist/shell-run.js", "./browser": "./dist/browser.js", "./session-event-health": "./dist/session-event-health.js", diff --git a/packages/core/src/__tests__/model-call-attempt.test.ts b/packages/core/src/__tests__/model-call-attempt.test.ts new file mode 100644 index 00000000000..0a7fe123cd9 --- /dev/null +++ b/packages/core/src/__tests__/model-call-attempt.test.ts @@ -0,0 +1,205 @@ +import { describe, test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { decodeAgentRunEvent } from '../agent-run.js'; +import { + MODEL_CALL_ATTEMPT_EVENT_TYPE, + MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + decodeModelCallAttempt, + dedupeModelCallAttempts, + groupModelCallAttempts, + isModelCallAttempt, + settledAttempt, + sumModelCallCostUsd, + summarizeModelCallCoverage, + type ModelCallAttempt, +} from '../model-call-attempt.js'; + +function attempt(overrides: Partial = {}): ModelCallAttempt { + return { + schemaVersion: MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + logicalCallId: 'call-1', + attemptId: 'attempt-1', + traceId: 'trace-1', + sessionId: 'session-1', + runId: 'run-1', + turnId: 'turn-1', + step: 0, + attempt: 0, + callKind: 'main', + providerId: 'anthropic', + modelId: 'claude-opus-5', + startedAt: 1_000, + completedAt: 1_250, + latencyMs: 250, + status: 'completed', + usageBasis: 'reported', + inputTokens: 100, + outputTokens: 20, + costBasis: 'priced', + costUsd: 0.004, + ...overrides, + }; +} + +describe('ModelCallAttempt codec', () => { + test('accepts a complete priced attempt', () => { + const decoded = decodeModelCallAttempt(attempt()); + assert.equal(decoded.logicalCallId, 'call-1'); + assert.equal(decoded.costUsd, 0.004); + }); + + test('accepts an unpriced attempt that carries usage but no cost', () => { + const decoded = decodeModelCallAttempt(attempt({ costBasis: 'unpriced', costUsd: undefined })); + assert.equal(decoded.costBasis, 'unpriced'); + assert.equal(decoded.costUsd, undefined); + }); + + test('rejects an unpriced attempt that carries a cost', () => { + assert.throws( + () => decodeModelCallAttempt(attempt({ costBasis: 'unpriced', costUsd: 0 })), + /unpriced record carries a cost/, + ); + }); + + test('rejects missing usage that still carries tokens', () => { + assert.throws( + () => decodeModelCallAttempt(attempt({ usageBasis: 'missing' })), + /missing usage but carries tokens/, + ); + }); + + test('accepts missing usage with no token fields', () => { + const decoded = decodeModelCallAttempt( + attempt({ + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + status: 'failed', + costBasis: 'unpriced', + costUsd: undefined, + }), + ); + assert.equal(decoded.usageBasis, 'missing'); + }); + + test('rejects completedAt before startedAt', () => { + assert.throws( + () => decodeModelCallAttempt(attempt({ startedAt: 2_000, completedAt: 1_000 })), + /completedAt precedes startedAt/, + ); + }); + + test('rejects unknown fields, wrong schema version, and bad enums', () => { + assert.throws(() => decodeModelCallAttempt({ ...attempt(), unexpected: true } as unknown)); + assert.throws(() => decodeModelCallAttempt(attempt({ schemaVersion: 2 as never }))); + assert.throws(() => decodeModelCallAttempt(attempt({ callKind: 'tool' as never }))); + assert.throws(() => decodeModelCallAttempt(attempt({ usageBasis: 'guessed' as never }))); + assert.throws(() => decodeModelCallAttempt(attempt({ costBasis: 'free' as never }))); + }); + + test('rejects empty identity and non-integer ordinals', () => { + assert.throws(() => decodeModelCallAttempt(attempt({ logicalCallId: '' }))); + assert.throws(() => decodeModelCallAttempt(attempt({ attemptId: '' }))); + assert.throws(() => decodeModelCallAttempt(attempt({ step: 1.5 }))); + assert.throws(() => decodeModelCallAttempt(attempt({ attempt: -1 }))); + }); + + test('validates optional pricing rates when present', () => { + const decoded = decodeModelCallAttempt( + attempt({ + pricingRevision: 7, + pricingRates: { modelKey: 'claude-opus-5', inputUsdPer1M: 15, outputUsdPer1M: 75 }, + }), + ); + assert.equal(decoded.pricingRevision, 7); + assert.throws(() => + decodeModelCallAttempt(attempt({ pricingRates: { modelKey: 'x' } as never })), + ); + }); + + test('isModelCallAttempt narrows without throwing', () => { + assert.equal(isModelCallAttempt(attempt()), true); + assert.equal(isModelCallAttempt({ nope: true }), false); + }); + + test('the AgentRun ledger accepts the new event type', () => { + const decoded = decodeAgentRunEvent({ + type: MODEL_CALL_ATTEMPT_EVENT_TYPE, + id: 'attempt-1', + runId: 'run-1', + sessionId: 'session-1', + turnId: 'turn-1', + ts: 1_250, + data: attempt() as unknown as Record, + }); + assert.equal(decoded.type, MODEL_CALL_ATTEMPT_EVENT_TYPE); + assert.equal(decodeModelCallAttempt(decoded.data).attemptId, 'attempt-1'); + }); +}); + +describe('ModelCallAttempt projections', () => { + test('dedupes re-appended records by attemptId, keeping the last', () => { + const first = attempt({ costUsd: 0.004 }); + const replayed = attempt({ costUsd: 0.005 }); + const unique = dedupeModelCallAttempts([first, replayed]); + assert.equal(unique.length, 1); + assert.equal(unique[0]?.costUsd, 0.005); + }); + + test('groups retries under one logical call and derives the settled attempt', () => { + const groups = groupModelCallAttempts([ + attempt({ attemptId: 'a-0', attempt: 0, status: 'failed', costUsd: 0.001 }), + attempt({ attemptId: 'a-1', attempt: 1, status: 'completed', costUsd: 0.004 }), + attempt({ attemptId: 'b-0', logicalCallId: 'call-2' }), + ]); + assert.equal(groups.length, 2); + const retried = groups.find((g) => g.logicalCallId === 'call-1'); + assert.equal(retried?.attempts.length, 2); + assert.equal(settledAttempt(retried!)?.attemptId, 'a-1'); + }); + + test('coverage counts priced, unpriced, and usage bases separately', () => { + const coverage = summarizeModelCallCoverage([ + attempt({ attemptId: 'a' }), + attempt({ attemptId: 'b', costBasis: 'unpriced', costUsd: undefined }), + attempt({ + attemptId: 'c', + costBasis: 'unpriced', + costUsd: undefined, + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + }), + attempt({ attemptId: 'd', usageBasis: 'partial', outputTokens: undefined }), + ]); + assert.deepEqual(coverage, { + attempts: 4, + pricedAttempts: 2, + unpricedAttempts: 2, + usageReportedAttempts: 2, + usagePartialAttempts: 1, + usageMissingAttempts: 1, + }); + }); + + test('cost sum reports the qualifying coverage alongside the total', () => { + const { costUsd, coverage } = sumModelCallCostUsd([ + attempt({ attemptId: 'a', costUsd: 0.004 }), + attempt({ attemptId: 'b', costUsd: 0.006 }), + attempt({ attemptId: 'c', costBasis: 'unpriced', costUsd: undefined }), + ]); + assert.equal(Math.round(costUsd * 1000) / 1000, 0.01); + // The unpriced call is real spend the total cannot express. + assert.equal(coverage.unpricedAttempts, 1); + }); + + test('a genuinely free priced call is distinguishable from an unpriced one', () => { + const free = attempt({ attemptId: 'free', costBasis: 'priced', costUsd: 0 }); + const unknown = attempt({ attemptId: 'unknown', costBasis: 'unpriced', costUsd: undefined }); + const { costUsd, coverage } = sumModelCallCostUsd([free, unknown]); + assert.equal(costUsd, 0); + assert.equal(coverage.pricedAttempts, 1); + assert.equal(coverage.unpricedAttempts, 1); + }); +}); diff --git a/packages/core/src/agent-run.ts b/packages/core/src/agent-run.ts index edcda988857..18d4f2a032a 100644 --- a/packages/core/src/agent-run.ts +++ b/packages/core/src/agent-run.ts @@ -166,6 +166,7 @@ export const AGENT_RUN_EVENT_TYPES = [ 'usage_recorded', 'provider_request_captured', 'provider_request_attempt_recorded', + 'model_call_attempt_recorded', 'history_compact_checkpoint_recorded', 'active_full_compact_block_recorded', 'semantic_compact_block_recorded', diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 20718b1be15..788c4b495f9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -331,6 +331,32 @@ export { isSessionInlineRun, } from './agent-run.js'; +// model-call-attempt.ts +export type { + ModelCallAttempt, + ModelCallAttemptStatus, + ModelCallCostBasis, + ModelCallCoverage, + ModelCallGroup, + ModelCallKind, + ModelCallUsageBasis, +} from './model-call-attempt.js'; +export { + MODEL_CALL_ATTEMPT_EVENT_TYPE, + MODEL_CALL_ATTEMPT_SCHEMA_VERSION, + MODEL_CALL_ATTEMPT_STATUSES, + MODEL_CALL_COST_BASES, + MODEL_CALL_KINDS, + MODEL_CALL_USAGE_BASES, + decodeModelCallAttempt, + dedupeModelCallAttempts, + groupModelCallAttempts, + isModelCallAttempt, + settledAttempt, + sumModelCallCostUsd, + summarizeModelCallCoverage, +} from './model-call-attempt.js'; + // shell-run.ts export type { PipeShellOutput, diff --git a/packages/core/src/model-call-attempt.ts b/packages/core/src/model-call-attempt.ts new file mode 100644 index 00000000000..0b6897f3d51 --- /dev/null +++ b/packages/core/src/model-call-attempt.ts @@ -0,0 +1,361 @@ +import { + defineObjectShape, + hasExactShape, + isFiniteNumber, + isOptionalFiniteNumber, + isOptionalString, + isRecord, +} from './record-schema.js'; +import type { PricingConfig } from './usage-stats/types.js'; + +/** + * Canonical accounting record for one physical provider request attempt. + * + * This is the metering source of truth: every real model call that is dispatched + * to a provider settles into exactly one `ModelCallAttempt`. Aggregate token + * usage on `RuntimeEvent` remains a per-turn projection for replay and recovery; + * it is not an accounting authority and is not derived from these records. + * + * Two properties are deliberate and must survive future edits: + * + * - `costUsd` is absent unless `costBasis` is `'priced'`. A zero cost means the + * call was genuinely free, never that the price was unknown. + * - These records are authoritative only for the calls they contain. A crash + * between provider dispatch and settlement leaves no record and no way to + * detect the omission, so no consumer may treat a set of attempts as proof of + * completeness. See {@link ModelCallCoverage}. + */ +export const MODEL_CALL_ATTEMPT_SCHEMA_VERSION = 1 as const; + +/** AgentRun event type carrying a {@link ModelCallAttempt} in `data`. */ +export const MODEL_CALL_ATTEMPT_EVENT_TYPE = 'model_call_attempt_recorded' as const; + +export const MODEL_CALL_KINDS = ['main', 'semantic_compact', 'history_compact'] as const; +export type ModelCallKind = (typeof MODEL_CALL_KINDS)[number]; + +export const MODEL_CALL_ATTEMPT_STATUSES = [ + 'completed', + 'failed', + 'interrupted', + 'aborted', +] as const; +export type ModelCallAttemptStatus = (typeof MODEL_CALL_ATTEMPT_STATUSES)[number]; + +/** + * Whether the provider reported usage for this attempt. Distinct from + * {@link ModelCallCostBasis}: "the provider never reported tokens" and "we have + * tokens but no price for this model" are different failures and must not + * collapse into one counter. + */ +export const MODEL_CALL_USAGE_BASES = ['reported', 'partial', 'missing'] as const; +export type ModelCallUsageBasis = (typeof MODEL_CALL_USAGE_BASES)[number]; + +/** Whether a price could be resolved for this attempt at record time. */ +export const MODEL_CALL_COST_BASES = ['priced', 'unpriced'] as const; +export type ModelCallCostBasis = (typeof MODEL_CALL_COST_BASES)[number]; + +export interface ModelCallAttempt { + schemaVersion: typeof MODEL_CALL_ATTEMPT_SCHEMA_VERSION; + + /** + * One logical model call. Every attempt of the same call — first try and each + * retry — shares this id. Explicit rather than reconstructed from + * `(traceId, step)`, because a compound key that each consumer has to rebuild + * is the kind of implicit contract this record exists to remove. + */ + logicalCallId: string; + /** Idempotency key: appending the same `attemptId` twice records once. */ + attemptId: string; + /** Tracker instance id, retained to join request-shape capture artifacts. */ + traceId: string; + + sessionId: string; + runId: string; + turnId: string; + + /** Runtime tool-loop step index within the turn. */ + step: number; + /** Retry ordinal within the logical call; 0 is the first dispatch. */ + attempt: number; + + callKind: ModelCallKind; + connectionSlug?: string; + providerId: string; + modelId: string; + contextWindow?: number; + /** Join key for request-shape diagnostics; absent when capture is disabled. */ + captureArtifactId?: string; + + startedAt: number; + completedAt: number; + latencyMs: number; + timeToFirstTokenMs?: number; + + status: ModelCallAttemptStatus; + finishReason?: string; + errorClass?: string; + + usageBasis: ModelCallUsageBasis; + inputTokens?: number; + outputTokens?: number; + cacheReadInputTokens?: number; + cacheMissInputTokens?: number; + cacheWriteInputTokens?: number; + reasoningTokens?: number; + + costBasis: ModelCallCostBasis; + /** Present only when `costBasis` is `'priced'`. Frozen at record time. */ + costUsd?: number; + /** Pricing authority revision the cost was computed against. */ + pricingRevision?: number; + /** Rates actually applied, so a recorded amount stays auditable. */ + pricingRates?: PricingConfig; +} + +const MODEL_CALL_ATTEMPT_SHAPE = defineObjectShape()( + [ + 'schemaVersion', + 'logicalCallId', + 'attemptId', + 'traceId', + 'sessionId', + 'runId', + 'turnId', + 'step', + 'attempt', + 'callKind', + 'providerId', + 'modelId', + 'startedAt', + 'completedAt', + 'latencyMs', + 'status', + 'usageBasis', + 'costBasis', + ], + [ + 'connectionSlug', + 'contextWindow', + 'captureArtifactId', + 'timeToFirstTokenMs', + 'finishReason', + 'errorClass', + 'inputTokens', + 'outputTokens', + 'cacheReadInputTokens', + 'cacheMissInputTokens', + 'cacheWriteInputTokens', + 'reasoningTokens', + 'costUsd', + 'pricingRevision', + 'pricingRates', + ], +); + +const TOKEN_FIELDS = [ + 'inputTokens', + 'outputTokens', + 'cacheReadInputTokens', + 'cacheMissInputTokens', + 'cacheWriteInputTokens', + 'reasoningTokens', +] as const satisfies readonly (keyof ModelCallAttempt)[]; + +function isNonNegativeInteger(value: unknown): boolean { + return isFiniteNumber(value) && Number.isInteger(value) && value >= 0; +} + +function isNonNegativeNumber(value: unknown): value is number { + return isFiniteNumber(value) && value >= 0; +} + +function isOptionalNonNegativeNumber(value: unknown): boolean { + return value === undefined || isNonNegativeNumber(value); +} + +function isPricingRates(value: unknown): value is PricingConfig { + if (value === undefined) return true; + if (!isRecord(value)) return false; + return ( + typeof value.modelKey === 'string' && + isFiniteNumber(value.inputUsdPer1M) && + isFiniteNumber(value.outputUsdPer1M) && + isOptionalFiniteNumber(value.cacheReadUsdPer1M) && + isOptionalFiniteNumber(value.cacheWriteUsdPer1M) + ); +} + +/** + * Strict subtype codec. The generic AgentRun event decoder only checks that + * `data` is a record, which is not enough for an accounting record — an + * unvalidated field silently becomes a wrong number in a cost report. + */ +export function decodeModelCallAttempt(value: unknown): ModelCallAttempt { + if (!isRecord(value) || !hasExactShape(value, MODEL_CALL_ATTEMPT_SHAPE)) { + throw new Error('Invalid ModelCallAttempt schema'); + } + const valid = + value.schemaVersion === MODEL_CALL_ATTEMPT_SCHEMA_VERSION && + typeof value.logicalCallId === 'string' && + value.logicalCallId.length > 0 && + typeof value.attemptId === 'string' && + value.attemptId.length > 0 && + typeof value.traceId === 'string' && + typeof value.sessionId === 'string' && + typeof value.runId === 'string' && + typeof value.turnId === 'string' && + isNonNegativeInteger(value.step) && + isNonNegativeInteger(value.attempt) && + (MODEL_CALL_KINDS as readonly unknown[]).includes(value.callKind) && + isOptionalString(value.connectionSlug) && + typeof value.providerId === 'string' && + typeof value.modelId === 'string' && + isOptionalNonNegativeNumber(value.contextWindow) && + isOptionalString(value.captureArtifactId) && + isFiniteNumber(value.startedAt) && + isFiniteNumber(value.completedAt) && + isNonNegativeNumber(value.latencyMs) && + isOptionalNonNegativeNumber(value.timeToFirstTokenMs) && + (MODEL_CALL_ATTEMPT_STATUSES as readonly unknown[]).includes(value.status) && + isOptionalString(value.finishReason) && + isOptionalString(value.errorClass) && + (MODEL_CALL_USAGE_BASES as readonly unknown[]).includes(value.usageBasis) && + TOKEN_FIELDS.every((field) => isOptionalNonNegativeNumber(value[field])) && + (MODEL_CALL_COST_BASES as readonly unknown[]).includes(value.costBasis) && + isOptionalNonNegativeNumber(value.costUsd) && + isOptionalNonNegativeNumber(value.pricingRevision) && + isPricingRates(value.pricingRates); + if (!valid) throw new Error('Invalid ModelCallAttempt schema'); + + const startedAt = value.startedAt as number; + const completedAt = value.completedAt as number; + if (completedAt < startedAt) { + throw new Error('ModelCallAttempt completedAt precedes startedAt'); + } + // A price we could not resolve must never be published as an amount. Zero is + // reserved for calls that genuinely cost nothing. + if (value.costBasis === 'unpriced' && value.costUsd !== undefined) { + throw new Error('ModelCallAttempt unpriced record carries a cost'); + } + if (value.usageBasis === 'missing' && TOKEN_FIELDS.some((f) => value[f] !== undefined)) { + throw new Error('ModelCallAttempt reports missing usage but carries tokens'); + } + return value as unknown as ModelCallAttempt; +} + +export function isModelCallAttempt(value: unknown): value is ModelCallAttempt { + try { + decodeModelCallAttempt(value); + return true; + } catch { + return false; + } +} + +/** + * Collapses re-appended records by `attemptId`, keeping the last occurrence. + * + * Order is the caller's append order, never `ts`: attempt events are appended + * asynchronously and carry the provider settlement time, so timestamp order and + * append order disagree. + */ +export function dedupeModelCallAttempts(attempts: readonly ModelCallAttempt[]): ModelCallAttempt[] { + const byId = new Map(); + for (const attempt of attempts) byId.set(attempt.attemptId, attempt); + return [...byId.values()]; +} + +/** Attempts of one logical model call, in append order. */ +export interface ModelCallGroup { + logicalCallId: string; + attempts: ModelCallAttempt[]; +} + +/** + * Groups attempts into logical calls. Retries are not separate calls: they are + * additional attempts of the same `logicalCallId`. + */ +export function groupModelCallAttempts(attempts: readonly ModelCallAttempt[]): ModelCallGroup[] { + const groups = new Map(); + for (const attempt of dedupeModelCallAttempts(attempts)) { + const existing = groups.get(attempt.logicalCallId); + if (existing) existing.attempts.push(attempt); + else + groups.set(attempt.logicalCallId, { + logicalCallId: attempt.logicalCallId, + attempts: [attempt], + }); + } + return [...groups.values()]; +} + +/** + * The attempt that settled a logical call: the highest `attempt` ordinal that + * reached a provider outcome. Terminality is a projection concern, not a stored + * field, so it is derived rather than recorded. + */ +export function settledAttempt(group: ModelCallGroup): ModelCallAttempt | undefined { + let settled: ModelCallAttempt | undefined; + for (const attempt of group.attempts) { + if (!settled || attempt.attempt > settled.attempt) settled = attempt; + } + return settled; +} + +/** + * Classification of the records present in a set. + * + * This is not a completeness proof and must never be presented as one. Nothing + * records an expected dispatch count, so an attempt lost between dispatch and + * settlement is invisible here. A total cost shown without this breakdown + * overstates what the ledger knows. + */ +export interface ModelCallCoverage { + attempts: number; + pricedAttempts: number; + /** Real spend whose price could not be resolved. */ + unpricedAttempts: number; + usageReportedAttempts: number; + usagePartialAttempts: number; + /** Dispatched calls the provider never reported usage for. */ + usageMissingAttempts: number; +} + +export function summarizeModelCallCoverage( + attempts: readonly ModelCallAttempt[], +): ModelCallCoverage { + const unique = dedupeModelCallAttempts(attempts); + const coverage: ModelCallCoverage = { + attempts: unique.length, + pricedAttempts: 0, + unpricedAttempts: 0, + usageReportedAttempts: 0, + usagePartialAttempts: 0, + usageMissingAttempts: 0, + }; + for (const attempt of unique) { + if (attempt.costBasis === 'priced') coverage.pricedAttempts += 1; + else coverage.unpricedAttempts += 1; + if (attempt.usageBasis === 'reported') coverage.usageReportedAttempts += 1; + else if (attempt.usageBasis === 'partial') coverage.usagePartialAttempts += 1; + else coverage.usageMissingAttempts += 1; + } + return coverage; +} + +/** + * Sums cost across attempts. Returns the total alongside the coverage that + * qualifies it, because a bare number cannot express "plus an unknown amount + * from unpriced calls". + */ +export function sumModelCallCostUsd(attempts: readonly ModelCallAttempt[]): { + costUsd: number; + coverage: ModelCallCoverage; +} { + const unique = dedupeModelCallAttempts(attempts); + let costUsd = 0; + for (const attempt of unique) { + if (attempt.costBasis === 'priced' && attempt.costUsd !== undefined) costUsd += attempt.costUsd; + } + return { costUsd, coverage: summarizeModelCallCoverage(unique) }; +} From 7000efa8be268bbc13bb694ae7a9b1efd0a727bf Mon Sep 17 00:00:00 2001 From: 404ARE <936233544@qq.com> Date: Fri, 31 Jul 2026 16:57:27 +0800 Subject: [PATCH 2/2] fix(core): pair costBasis with costUsd in both directions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the P2 and P3 notes on #1687. The codec forbade an unpriced record carrying an amount but still accepted a priced record with no amount. Coverage counted that row as priced while `sumModelCallCostUsd` skipped it, so a set of such rows read as "every call priced, total $0" — indistinguishable from genuinely free, which is the exact confusion the basis fields exist to prevent. `costBasis` and `costUsd` now travel together in both directions; zero stays legal and remains the only way to record a call that cost nothing. Also tightens the surrounding validation, all audit and attribution quality: - `pricingRates` is held to an exact object shape with non-negative rates. It previously accepted negative rates and unrecognized nested keys, which makes a recorded amount unexplainable and defeats storing the basis. - `traceId`, `sessionId`, `runId`, `turnId`, `providerId`, and `modelId` must be non-empty, matching the existing treatment of `logicalCallId` and `attemptId`. - Documents that the payload identity is the portable source of truth and must agree with the AgentRun envelope when the record is written as an event. Tests: symmetric priced-without-cost rejection and a priced-zero acceptance, negative and non-finite gates on costs and tokens, empty attribution keys, the strengthened rate shapes, and a replayed `attemptId` carried through `sumModelCallCostUsd` and `summarizeModelCallCoverage` rather than only the bare dedupe helper. Co-Authored-By: Claude Opus 5 --- .../src/__tests__/model-call-attempt.test.ts | 71 +++++++++++++++++++ packages/core/src/model-call-attempt.ts | 60 +++++++++++----- 2 files changed, 114 insertions(+), 17 deletions(-) diff --git a/packages/core/src/__tests__/model-call-attempt.test.ts b/packages/core/src/__tests__/model-call-attempt.test.ts index 0a7fe123cd9..a59fe5fdffc 100644 --- a/packages/core/src/__tests__/model-call-attempt.test.ts +++ b/packages/core/src/__tests__/model-call-attempt.test.ts @@ -62,6 +62,20 @@ describe('ModelCallAttempt codec', () => { ); }); + test('rejects a priced attempt that carries no cost', () => { + // Otherwise coverage counts it as priced while the sum skips it, and + // "every call priced, total $0" reads as genuinely free. + assert.throws( + () => decodeModelCallAttempt(attempt({ costBasis: 'priced', costUsd: undefined })), + /priced record carries no cost/, + ); + }); + + test('accepts a priced attempt costing exactly zero', () => { + const decoded = decodeModelCallAttempt(attempt({ costBasis: 'priced', costUsd: 0 })); + assert.equal(decoded.costUsd, 0); + }); + test('rejects missing usage that still carries tokens', () => { assert.throws( () => decodeModelCallAttempt(attempt({ usageBasis: 'missing' })), @@ -105,6 +119,26 @@ describe('ModelCallAttempt codec', () => { assert.throws(() => decodeModelCallAttempt(attempt({ attempt: -1 }))); }); + test('rejects empty attribution keys', () => { + for (const field of ['traceId', 'sessionId', 'runId', 'turnId', 'providerId', 'modelId']) { + assert.throws( + () => decodeModelCallAttempt(attempt({ [field]: '' } as Partial)), + `expected empty ${field} to be rejected`, + ); + } + }); + + test('rejects negative and non-finite amounts', () => { + for (const costUsd of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + assert.throws(() => decodeModelCallAttempt(attempt({ costUsd }))); + } + for (const inputTokens of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + assert.throws(() => decodeModelCallAttempt(attempt({ inputTokens }))); + } + assert.throws(() => decodeModelCallAttempt(attempt({ latencyMs: -1 }))); + assert.throws(() => decodeModelCallAttempt(attempt({ pricingRevision: -1 }))); + }); + test('validates optional pricing rates when present', () => { const decoded = decodeModelCallAttempt( attempt({ @@ -113,9 +147,30 @@ describe('ModelCallAttempt codec', () => { }), ); assert.equal(decoded.pricingRevision, 7); + // Incomplete, negative, or unrecognized rate shapes make a recorded amount + // unexplainable, which defeats storing the basis at all. assert.throws(() => decodeModelCallAttempt(attempt({ pricingRates: { modelKey: 'x' } as never })), ); + assert.throws(() => + decodeModelCallAttempt( + attempt({ + pricingRates: { modelKey: 'x', inputUsdPer1M: -1, outputUsdPer1M: 75 } as never, + }), + ), + ); + assert.throws(() => + decodeModelCallAttempt( + attempt({ + pricingRates: { + modelKey: 'x', + inputUsdPer1M: 15, + outputUsdPer1M: 75, + surcharge: 3, + } as never, + }), + ), + ); }); test('isModelCallAttempt narrows without throwing', () => { @@ -194,6 +249,22 @@ describe('ModelCallAttempt projections', () => { assert.equal(coverage.unpricedAttempts, 1); }); + test('a replayed attemptId is counted once through sum and coverage', () => { + // The bare dedupe helper is covered above; this locks the same guarantee on + // the paths a consumer actually calls, across a multi-id stream. + const stream = [ + attempt({ attemptId: 'a', logicalCallId: 'call-1', costUsd: 0.004 }), + attempt({ attemptId: 'b', logicalCallId: 'call-2', costUsd: 0.006 }), + attempt({ attemptId: 'a', logicalCallId: 'call-1', costUsd: 0.004 }), + ]; + const { costUsd, coverage } = sumModelCallCostUsd(stream); + assert.equal(Math.round(costUsd * 1000) / 1000, 0.01); + assert.equal(coverage.attempts, 2); + assert.equal(coverage.pricedAttempts, 2); + assert.equal(summarizeModelCallCoverage(stream).attempts, 2); + assert.equal(groupModelCallAttempts(stream).length, 2); + }); + test('a genuinely free priced call is distinguishable from an unpriced one', () => { const free = attempt({ attemptId: 'free', costBasis: 'priced', costUsd: 0 }); const unknown = attempt({ attemptId: 'unknown', costBasis: 'unpriced', costUsd: undefined }); diff --git a/packages/core/src/model-call-attempt.ts b/packages/core/src/model-call-attempt.ts index 0b6897f3d51..e2901fc4eba 100644 --- a/packages/core/src/model-call-attempt.ts +++ b/packages/core/src/model-call-attempt.ts @@ -69,6 +69,12 @@ export interface ModelCallAttempt { /** Tracker instance id, retained to join request-shape capture artifacts. */ traceId: string; + /** + * Session, run, and turn the call belongs to. This payload identity is the + * portable source of truth: when the record is written as an AgentRun event it + * must agree with the envelope, so a record stays attributable on its own once + * it leaves the event stream. + */ sessionId: string; runId: string; turnId: string; @@ -161,6 +167,10 @@ const TOKEN_FIELDS = [ 'reasoningTokens', ] as const satisfies readonly (keyof ModelCallAttempt)[]; +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} + function isNonNegativeInteger(value: unknown): boolean { return isFiniteNumber(value) && Number.isInteger(value) && value >= 0; } @@ -173,15 +183,27 @@ function isOptionalNonNegativeNumber(value: unknown): boolean { return value === undefined || isNonNegativeNumber(value); } +const PRICING_RATES_SHAPE = defineObjectShape()( + ['modelKey', 'inputUsdPer1M', 'outputUsdPer1M'], + ['cacheReadUsdPer1M', 'cacheWriteUsdPer1M'], +); + +/** + * Audit-quality gate on the rates a cost was computed against. Held to the same + * exact shape as the top-level record: a negative rate or an unrecognized nested + * key makes a recorded amount unexplainable, which defeats the point of storing + * the basis at all. + */ function isPricingRates(value: unknown): value is PricingConfig { if (value === undefined) return true; - if (!isRecord(value)) return false; + if (!isRecord(value) || !hasExactShape(value, PRICING_RATES_SHAPE)) return false; return ( typeof value.modelKey === 'string' && - isFiniteNumber(value.inputUsdPer1M) && - isFiniteNumber(value.outputUsdPer1M) && - isOptionalFiniteNumber(value.cacheReadUsdPer1M) && - isOptionalFiniteNumber(value.cacheWriteUsdPer1M) + value.modelKey.length > 0 && + isNonNegativeNumber(value.inputUsdPer1M) && + isNonNegativeNumber(value.outputUsdPer1M) && + isOptionalNonNegativeNumber(value.cacheReadUsdPer1M) && + isOptionalNonNegativeNumber(value.cacheWriteUsdPer1M) ); } @@ -196,20 +218,18 @@ export function decodeModelCallAttempt(value: unknown): ModelCallAttempt { } const valid = value.schemaVersion === MODEL_CALL_ATTEMPT_SCHEMA_VERSION && - typeof value.logicalCallId === 'string' && - value.logicalCallId.length > 0 && - typeof value.attemptId === 'string' && - value.attemptId.length > 0 && - typeof value.traceId === 'string' && - typeof value.sessionId === 'string' && - typeof value.runId === 'string' && - typeof value.turnId === 'string' && + isNonEmptyString(value.logicalCallId) && + isNonEmptyString(value.attemptId) && + isNonEmptyString(value.traceId) && + isNonEmptyString(value.sessionId) && + isNonEmptyString(value.runId) && + isNonEmptyString(value.turnId) && isNonNegativeInteger(value.step) && isNonNegativeInteger(value.attempt) && (MODEL_CALL_KINDS as readonly unknown[]).includes(value.callKind) && isOptionalString(value.connectionSlug) && - typeof value.providerId === 'string' && - typeof value.modelId === 'string' && + isNonEmptyString(value.providerId) && + isNonEmptyString(value.modelId) && isOptionalNonNegativeNumber(value.contextWindow) && isOptionalString(value.captureArtifactId) && isFiniteNumber(value.startedAt) && @@ -232,11 +252,17 @@ export function decodeModelCallAttempt(value: unknown): ModelCallAttempt { if (completedAt < startedAt) { throw new Error('ModelCallAttempt completedAt precedes startedAt'); } - // A price we could not resolve must never be published as an amount. Zero is - // reserved for calls that genuinely cost nothing. + // `costBasis` and `costUsd` travel together in both directions. A price we + // could not resolve must never be published as an amount, and a priced record + // must carry one — otherwise coverage counts it as priced while the sum skips + // it, and "every call priced, total $0" reads as genuinely free. Zero stays + // legal, and is the only way to say a call cost nothing. if (value.costBasis === 'unpriced' && value.costUsd !== undefined) { throw new Error('ModelCallAttempt unpriced record carries a cost'); } + if (value.costBasis === 'priced' && value.costUsd === undefined) { + throw new Error('ModelCallAttempt priced record carries no cost'); + } if (value.usageBasis === 'missing' && TOKEN_FIELDS.some((f) => value[f] !== undefined)) { throw new Error('ModelCallAttempt reports missing usage but carries tokens'); }