Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,8 @@
* 3. A provider with no catalogued entries falls back to the env-configured
* defaults AI_PRICING_FALLBACK_INPUT_USD_PER_M /
* AI_PRICING_FALLBACK_OUTPUT_USD_PER_M.
* 4. With no catalog and no env default, the request stays servable at a
* non-zero hardcoded frontier-max last resort — never $0 (#11635; also
* covered in lookup-missing-pricing.test.ts).
* 4. With no catalog and no env default, a non-zero token side rejects
* instead of guessing a price. A `..._USD_PER_M=0` env is treated as unset.
* 5. Reserve (estimated tokens) and settle (actual tokens) resolve the same
* fallback rate, so billing stays consistent across the request.
*/
Expand Down Expand Up @@ -179,33 +178,56 @@ test("env-configured default applies when the provider has no catalogued entries
expect(result.totalCost).toBe(12.5 * 1.2);
});

test("invalid env default is ignored (falls through to the non-zero last resort, #11635)", async () => {
test("invalid env default is ignored, then missing pricing fails closed (#11635)", async () => {
process.env.AI_PRICING_FALLBACK_INPUT_USD_PER_M = "not-a-number";
process.env.AI_PRICING_FALLBACK_OUTPUT_USD_PER_M = "-4";

const result = await calculateTextCostFromCatalog({
model: "mystery-model-1",
provider: "someprovider",
inputTokens: 1_000,
outputTokens: 1_000,
});
await expect(
calculateTextCostFromCatalog({
model: "mystery-model-1",
provider: "someprovider",
inputTokens: 1_000,
outputTokens: 1_000,
}),
).rejects.toThrow("refusing to bill unknown-priced inference");
});

// Invalid env is still ignored, but the last resort is now the hardcoded
// frontier-max rate, never $0 (#11635).
expect(result.totalCost).toBeGreaterThan(0);
expect(result.totalCost).toBeLessThan(0.1);
test("AI_PRICING_FALLBACK_*=0 is treated as unset, not as a $0 price (#11635)", async () => {
process.env.AI_PRICING_FALLBACK_INPUT_USD_PER_M = "0";
process.env.AI_PRICING_FALLBACK_OUTPUT_USD_PER_M = "0";

await expect(
calculateTextCostFromCatalog({
model: "mystery-model-1",
provider: "someprovider",
inputTokens: 1_000,
outputTokens: 1_000,
}),
).rejects.toThrow("refusing to bill unknown-priced inference");
});

test("no catalog and no env default rejects instead of selling unknown-priced inference (#11635)", async () => {
await expect(
calculateTextCostFromCatalog({
model: "mystery-model-1",
provider: "someprovider",
inputTokens: 1_000,
outputTokens: 1_000,
}),
).rejects.toThrow("refusing to bill unknown-priced inference");
});

test("no catalog and no env default keeps the request servable at a non-zero last resort (#11635)", async () => {
test("a side with zero tokens does not require pricing (#11635)", async () => {
process.env.AI_PRICING_FALLBACK_INPUT_USD_PER_M = "2.5";

const result = await calculateTextCostFromCatalog({
model: "mystery-model-1",
provider: "someprovider",
inputTokens: 1_000,
outputTokens: 1_000,
outputTokens: 0,
});

expect(result.inputCost).toBeGreaterThan(0);
expect(result.outputCost).toBeGreaterThan(0);
expect(result.totalCost).toBeGreaterThan(0);
expect(result.totalCost).toBeLessThan(0.1);
expect(result.baseOutputCost).toBe(0);
expect(result.baseInputCost).toBeCloseTo(0.0025, 9);
expect(result.totalCost).toBeCloseTo(0.003, 9);
});
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Regression: a missing price must (1) NOT throw a 500 and (2) NEVER bill $0.
* Regression: a missing price must never bill $0 or a guessed hardcoded floor.
*
* `calculateTextCostFromCatalog` once left the input-price lookup unguarded, so
* a catalog miss threw `Pricing unavailable for language:input <model>` → a 500
Expand All @@ -9,14 +9,15 @@
* inference / uncollected revenue (#11635).
*
* Both seams (persisted repo + live gateway) are mocked empty so EVERY lookup
* misses — the worst case — and no env fallback is set, so the last-resort tier
* is exercised. Post-#11635 it bills a conservative non-zero frontier-max rate
* (`lastResortTokenUnitPrice`), keyed by product family, and still never throws.
* (Provider-max / env-default tiers are covered in lookup-fallback-pricing.test.ts.)
* misses — the worst case — and no env fallback is set. Post-#11635, a non-zero
* token side rejects so we do not sell inference we do not know how to price.
* Provider-max and env-default fallback tiers are covered in
* lookup-fallback-pricing.test.ts.
*/
import { beforeEach, expect, mock, test } from "bun:test";

const warnSpy = mock(() => {});
const errorSpy = mock(() => {});

mock.module("../../../db/repositories/ai-pricing", () => ({
aiPricingRepository: {
Expand All @@ -27,6 +28,7 @@ mock.module("../../../db/repositories/ai-pricing", () => ({
mock.module("../../utils/logger", () => ({
logger: {
warn: warnSpy,
error: errorSpy,
},
}));
mock.module("./providers/gateway", () => ({
Expand All @@ -37,56 +39,53 @@ const { calculateTextCostFromCatalog } = await import("./lookup");

beforeEach(() => {
warnSpy.mockClear();
errorSpy.mockClear();
});

test("missing language pricing bills a non-zero last-resort rate, never $0 (#11635)", async () => {
const result = await calculateTextCostFromCatalog({
model: "totally-uncatalogued-model",
provider: "someprovider",
inputTokens: 1000,
outputTokens: 500,
});
test("missing language pricing rejects instead of billing an unknown price (#11635)", async () => {
await expect(
calculateTextCostFromCatalog({
model: "totally-uncatalogued-model",
provider: "someprovider",
inputTokens: 1000,
outputTokens: 500,
}),
).rejects.toThrow("refusing to bill unknown-priced inference");

// Never throws (the original 500-degradation) AND never $0 (the #11635 fix):
// 1000in@$5/M + 500out@$25/M ≈ $0.0175 base + markup — a small, sane amount.
expect(result.inputCost).toBeGreaterThan(0);
expect(result.outputCost).toBeGreaterThan(0);
expect(result.totalCost).toBeGreaterThan(0);
expect(result.totalCost).toBeLessThan(0.1); // sane: not an absurd rate
expect(warnSpy.mock.calls).toContainEqual([
"ai-pricing: input pricing unavailable; billing at fallback rate",
expect(errorSpy.mock.calls).toContainEqual([
"ai-pricing: missing token price with no fallback; refusing request",
{
canonicalModel: "someprovider/totally-uncatalogued-model",
provider: "someprovider",
billingSource: undefined,
fallbackSource: "last_resort",
fallbackUnitPrice: 0.000005,
productFamily: "language",
chargeType: "input",
tokens: 1000,
},
]);
expect(warnSpy.mock.calls).toContainEqual([
"ai-pricing: output pricing unavailable; billing at fallback rate",
expect(warnSpy.mock.calls).toHaveLength(0);
});

test("missing input-only embedding pricing rejects instead of billing an unknown price (#11635)", async () => {
await expect(
calculateTextCostFromCatalog({
model: "uncatalogued-embedding-model",
provider: "someprovider",
inputTokens: 800,
outputTokens: 0,
}),
).rejects.toThrow("refusing to bill unknown-priced inference");

expect(errorSpy.mock.calls).toContainEqual([
"ai-pricing: missing token price with no fallback; refusing request",
{
canonicalModel: "someprovider/totally-uncatalogued-model",
canonicalModel: "someprovider/uncatalogued-embedding-model",
provider: "someprovider",
billingSource: undefined,
fallbackSource: "last_resort",
fallbackUnitPrice: 0.000025,
productFamily: "embedding",
chargeType: "input",
tokens: 800,
},
]);
});

test("missing embedding pricing bills the cheaper embedding last-resort rate, non-zero (#11635)", async () => {
const result = await calculateTextCostFromCatalog({
model: "uncatalogued-embedding-model",
provider: "someprovider",
inputTokens: 800,
outputTokens: 0,
});

// Non-zero (never free) but keyed to the embedding family ($0.2/M), so 800
// tokens is a tiny amount — proving the family keying picked the cheap rate,
// not the $5/M language default.
expect(result.inputCost).toBeGreaterThan(0);
expect(result.totalCost).toBeGreaterThan(0);
expect(result.totalCost).toBeLessThan(0.001);
expect(warnSpy.mock.calls).toHaveLength(0);
});
71 changes: 28 additions & 43 deletions packages/cloud/shared/src/lib/services/ai-pricing/lookup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,10 @@ function envFallbackTokenUnitPrice(chargeType: "input" | "output"): number | nul
return null;
}
const usdPerMillion = Number(raw);
if (!Number.isFinite(usdPerMillion) || usdPerMillion < 0) {
// #11635: reject 0 too (not just negative/non-finite) — a `..._USD_PER_M=0`
// env value would otherwise masquerade as a configured floor while still
// billing $0. Treat it as unset so the missing-price path fails closed.
if (!Number.isFinite(usdPerMillion) || usdPerMillion <= 0) {
logger.warn("ai-pricing: ignoring invalid fallback-rate env value", {
envName,
value: raw,
Expand All @@ -179,33 +182,9 @@ function envFallbackTokenUnitPrice(chargeType: "input" | "output"): number | nul
return usdPerMillion / 1_000_000;
}

/**
* Fail-closed last-resort per-token rate (USD/token) for a servable but
* uncatalogued model with no provider-max catalog entry AND no env fallback
* (#11635). The request still serves — it just can never bill $0. Deliberately
* conservative (frontier-max) so an unpriced model over-bills rather than gives
* away free inference; a real catalog entry (cheaper) supersedes this on the
* next lookup, and the loud log points the catalog gap.
*/
const LAST_RESORT_USD_PER_MILLION: Partial<
Record<PricingProductFamily, { input: number; output: number }>
> = {
language: { input: 5, output: 25 },
embedding: { input: 0.2, output: 0.2 },
};
const DEFAULT_LAST_RESORT_USD_PER_MILLION = { input: 5, output: 25 };

function lastResortTokenUnitPrice(
productFamily: PricingProductFamily,
chargeType: "input" | "output",
): number {
const family = LAST_RESORT_USD_PER_MILLION[productFamily] ?? DEFAULT_LAST_RESORT_USD_PER_MILLION;
return family[chargeType] / 1_000_000;
}

type FallbackTokenRate = {
unitPrice: number;
source: "provider_max_catalog" | "env_default" | "last_resort";
source: "provider_max_catalog" | "env_default";
referenceModel?: string;
};

Expand All @@ -219,10 +198,8 @@ type FallbackTokenRate = {
* catalogued token rate for the same product family/charge type (an upper
* bound over any plausible real price from that provider), or an
* env-configured default (AI_PRICING_FALLBACK_{INPUT,OUTPUT}_USD_PER_M) when
* the provider has no catalogued entries at all, or a hardcoded last-resort
* rate when neither source exists. Both reserve (pre-flight estimate) and
* settle (actual usage) resolve through this same path, so both sides of a
* request bill at the same rate.
* the provider has no catalogued entries at all. If neither source exists, the
* caller must fail closed rather than inventing a price.
*/
async function resolveFallbackTokenRate(params: {
billingSource?: PricingBillingSource;
Expand Down Expand Up @@ -294,10 +271,7 @@ async function resolveFallbackTokenRate(params: {
return { unitPrice: envUnitPrice, source: "env_default" };
}

return {
unitPrice: lastResortTokenUnitPrice(params.productFamily, params.chargeType),
source: "last_resort",
};
return null;
}

function computeCostFromEntry(entry: PreparedPricingEntry, quantity: number): FlatOperationCost {
Expand Down Expand Up @@ -373,10 +347,9 @@ export async function calculateTextCostFromCatalog(params: {
// the catalog (notably embedding models, which are input-only and run every
// turn). A servable request must never fail purely on a missing price — but
// it must not be under-billed at $0 either. On a miss, bill the missing side
// at a conservative fallback rate (provider max → env default → a non-zero
// hardcoded frontier-max last resort, NEVER $0; see resolveFallbackTokenRate
// + lastResortTokenUnitPrice, #11635) and log loudly so the catalog gap gets
// priced.
// only when a real fallback exists (provider max → env default). If neither
// source exists, fail closed because we should not sell inference we do not
// know how to price (#11635).
const inputEntry = await resolvePreparedPricingEntry({
billingSource: params.billingSource,
provider: params.provider,
Expand Down Expand Up @@ -409,13 +382,25 @@ export async function calculateTextCostFromCatalog(params: {
productFamily,
chargeType,
});
if (!fallback) {
const message = `Pricing unavailable for ${productFamily}:${chargeType} ${canonicalModel}; refusing to bill unknown-priced inference`;
logger.error("ai-pricing: missing token price with no fallback; refusing request", {
canonicalModel,
provider: params.provider,
billingSource: params.billingSource,
productFamily,
chargeType,
tokens,
});
throw new Error(message);
}
logger.warn(`ai-pricing: ${chargeType} pricing unavailable; billing at fallback rate`, {
canonicalModel,
provider: params.provider,
billingSource: params.billingSource,
fallbackSource: fallback?.source ?? "none",
fallbackUnitPrice: fallback?.unitPrice ?? 0,
...(fallback?.referenceModel ? { fallbackReferenceModel: fallback.referenceModel } : {}),
fallbackSource: fallback.source,
fallbackUnitPrice: fallback.unitPrice,
...(fallback.referenceModel ? { fallbackReferenceModel: fallback.referenceModel } : {}),
});
return fallback;
};
Expand All @@ -427,10 +412,10 @@ export async function calculateTextCostFromCatalog(params: {

const inputUnitPrice = inputEntry
? asDecimal(inputEntry.unitPrice)
: asDecimal(inputFallback?.unitPrice ?? lastResortTokenUnitPrice(productFamily, "input"));
: asDecimal(inputFallback?.unitPrice ?? 0);
const outputUnitPrice = outputEntry
? asDecimal(outputEntry.unitPrice)
: asDecimal(outputFallback?.unitPrice ?? lastResortTokenUnitPrice(productFamily, "output"));
: asDecimal(outputFallback?.unitPrice ?? 0);

const baseInputCost = inputUnitPrice.mul(params.inputTokens);
const baseOutputCost = outputUnitPrice.mul(params.outputTokens);
Expand Down
Loading