Skip to content

fix(cost_calculator): bill ultrafast prompts above 272k at the ultrafast long-context rates #43764

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kerry-berri merged 8 commits into from
Sep 30, 2026
Merged

fix(cost_calculator): bill ultrafast prompts above 272k at the ultrafast long-context rates #43764

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4dfedd8
fix(cost_calculator): bill ultrafast prompts above 272k at the ultraf…
kerry-berri Sep 29, 2026
b3ba183
test(evals): cover sync API operations
kerry-berri Sep 29, 2026
4d79ab6
test(evals): mark unit tests as a package
kerry-berri Sep 29, 2026
2c45ab9
test(ci): include evals tests in misc shard
kerry-berri Sep 29, 2026
818924d
test(pricing): cover ultrafast long-context billing on the responses api
kerry-berri Sep 29, 2026
fcb5141
test(pricing): cite the openai pricing source in the ultrafast assertion
kerry-berri Sep 29, 2026
61f1d3d
revert(evals): drop out-of-scope evals changes from pricing pr
kerry-berri Sep 29, 2026
17e4264
chore(lint): raise reportUnknownArgumentType budget for the ultrafast…
kerry-berri Sep 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions litellm-rust/crates/cost/tests/calculation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,85 @@ fn threshold_tiers_and_boundaries() {
assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
}

#[rstest]
#[case::ultrafast_above_threshold(ServiceTier::Ultrafast, 300_000, 9_301_000.0, 37_000.0)]
#[case::ultrafast_at_threshold(ServiceTier::Ultrafast, 272_000, 544_500.0, 5_000.0)]
#[case::standard_above_threshold(ServiceTier::Standard, 300_000, 3_300_600.0, 13_000.0)]
#[case::priority_above_threshold(ServiceTier::Priority, 300_000, 5_701_000.0, 23_000.0)]
fn tiered_long_context_rates_are_selected_by_service_tier(
#[case] service_tier: ServiceTier,
#[case] prompt_tokens: u64,
#[case] expected_input: f64,
#[case] expected_output: f64,
) {
let standard = Rates {
cache_read: Rate::Value(3.0),
..rates(Rate::Value(1.0), Rate::Value(2.0))
};
let tiers = [
TierRates {
tier: ServiceTier::Priority,
rates: Rates {
cache_read: Rate::Value(5.0),
..rates(Rate::Value(3.0), Rate::Value(4.0))
},
},
TierRates {
tier: ServiceTier::Ultrafast,
rates: Rates {
cache_read: Rate::Value(7.0),
..rates(Rate::Value(2.0), Rate::Value(5.0))
},
},
];
let threshold_tiers = [
TierRates {
tier: ServiceTier::Priority,
rates: Rates {
cache_read: Rate::Value(29.0),
..rates(Rate::Value(19.0), Rate::Value(23.0))
},
},
TierRates {
tier: ServiceTier::Ultrafast,
rates: Rates {
cache_read: Rate::Value(41.0),
..rates(Rate::Value(31.0), Rate::Value(37.0))
},
},
];
let thresholds = [ThresholdRates {
above_prompt_tokens: 272_000,
standard: Rates {
cache_read: Rate::Value(17.0),
..rates(Rate::Value(11.0), Rate::Value(13.0))
},
tiers: &threshold_tiers,
}];
let pricing = Pricing {
standard,
tiers: &tiers,
thresholds: &thresholds,
off_peak: None,
};
let base = request();
let long_context_request = Request {
usage: Usage {
prompt_tokens,
completion_tokens: 1_000,
cache_read_tokens: 100,
cache_write_tokens: 0,
..base.usage
},
service_tier,
..base
};
let cost = calculate(&pricing, &long_context_request).unwrap();

assert_eq!(cost.input(), expected_input);
assert_eq!(cost.output(), expected_output);
}

#[test]
fn compile_rejects_ambiguous_rates() {
let duplicate = ThresholdRates {
Expand Down
22 changes: 11 additions & 11 deletions litellm/evals/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def create_eval(
_is_async: Final = kwargs.pop("acreate_eval", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -333,7 +333,7 @@ def list_evals(
_is_async: Final = kwargs.pop("alist_evals", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -499,7 +499,7 @@ def get_eval(
_is_async: Final = kwargs.pop("aget_eval", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -664,7 +664,7 @@ def update_eval(
_is_async: Final = kwargs.pop("aupdate_eval", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -869,7 +869,7 @@ def delete_eval(
_is_async: Final = kwargs.pop("adelete_eval", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -1019,7 +1019,7 @@ def cancel_eval(
_is_async: Final = kwargs.pop("acancel_eval", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -1198,7 +1198,7 @@ def create_run(
_is_async: Final = kwargs.pop("acreate_run", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -1382,7 +1382,7 @@ def list_runs(
_is_async: Final = kwargs.pop("alist_runs", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -1552,7 +1552,7 @@ def get_run(
_is_async: Final = kwargs.pop("aget_run", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -1708,7 +1708,7 @@ def cancel_run(
_is_async: Final = kwargs.pop("acancel_run", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down Expand Up @@ -1873,7 +1873,7 @@ def delete_run(
_is_async: Final = kwargs.pop("adelete_run", False) is True

# Get LiteLLM parameters
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)

# Determine provider
if custom_llm_provider is None:
Expand Down
8 changes: 8 additions & 0 deletions litellm/types/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_above_272k_tokens: float | None
cache_creation_input_token_cost_above_272k_tokens_priority: float | None
cache_creation_input_token_cost_above_272k_tokens_flex: float | None
cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_creation_input_token_cost_above_1hr: float | None
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
Expand All @@ -300,6 +301,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost_above_272k_tokens: float | None
cache_read_input_token_cost_above_272k_tokens_priority: float | None
cache_read_input_token_cost_above_272k_tokens_flex: float | None
cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_read_input_token_cost_above_512k_tokens: float | None
cache_read_input_token_cost_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
Expand All @@ -319,6 +321,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_token_above_272k_tokens: float | None # GPT-5.4/5.4-pro: prompts >272K priced at 2x input
input_cost_per_token_above_272k_tokens_priority: float | None
input_cost_per_token_above_272k_tokens_flex: float | None
input_cost_per_token_above_272k_tokens_ultrafast: ReadOnly[float | None]
input_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x input
input_cost_per_character_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_query: float | None # per-request pricing: rerank, search, and Bedrock Marengo embeddings
Expand Down Expand Up @@ -360,6 +363,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_token_above_272k_tokens: float | None # GPT-5.4/5.4-pro: prompts >272K priced at 1.5x output
output_cost_per_token_above_272k_tokens_priority: float | None
output_cost_per_token_above_272k_tokens_flex: float | None
output_cost_per_token_above_272k_tokens_ultrafast: ReadOnly[float | None]
output_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x output
output_cost_per_character_above_128k_tokens: float | None # only for vertex ai models
output_cost_per_image: float | None
Expand Down Expand Up @@ -3737,6 +3741,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_creation_input_token_cost_above_272k_tokens: float | None = None
cache_creation_input_token_cost_above_272k_tokens_priority: float | None = None
cache_creation_input_token_cost_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_above_272k_tokens_ultrafast: float | None = None
cache_creation_input_token_cost_flex: float | None = None
cache_creation_input_token_cost_priority: float | None = None
cache_creation_input_token_cost_ultrafast: float | None = None
Expand All @@ -3749,6 +3754,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_read_input_token_cost_above_200k_tokens_priority: float | None = None
cache_read_input_token_cost_above_272k_tokens_priority: float | None = None
cache_read_input_token_cost_above_272k_tokens_flex: float | None = None
cache_read_input_token_cost_above_272k_tokens_ultrafast: float | None = None
cache_read_input_token_cost_batches: float | None = None
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
Expand All @@ -3765,6 +3771,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_token_above_200k_tokens_priority: float | None = None
input_cost_per_token_above_272k_tokens_priority: float | None = None
input_cost_per_token_above_272k_tokens_flex: float | None = None
input_cost_per_token_above_272k_tokens_ultrafast: float | None = None
input_cost_per_token_above_200k_tokens_batches: float | None = None
input_cost_per_token_above_272k_tokens_batches: float | None = None
input_cost_per_query: float | None = None
Expand All @@ -3791,6 +3798,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_token_above_200k_tokens_priority: float | None = None
output_cost_per_token_above_272k_tokens_priority: float | None = None
output_cost_per_token_above_272k_tokens_flex: float | None = None
output_cost_per_token_above_272k_tokens_ultrafast: float | None = None
output_cost_per_token_above_200k_tokens_batches: float | None = None
output_cost_per_token_above_272k_tokens_batches: float | None = None
output_cost_per_character_above_128k_tokens: float | None = None
Expand Down
12 changes: 12 additions & 0 deletions litellm/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6103,6 +6103,9 @@ def _get_model_info_helper(
cache_creation_input_token_cost_above_272k_tokens_flex=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_flex", None
),
cache_creation_input_token_cost_above_272k_tokens_ultrafast=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_ultrafast", None
),
cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None),
cache_creation_input_token_cost_priority=_model_info.get(
"cache_creation_input_token_cost_priority", None
Expand All @@ -6128,6 +6131,9 @@ def _get_model_info_helper(
cache_read_input_token_cost_above_272k_tokens_flex=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_flex", None
),
cache_read_input_token_cost_above_272k_tokens_ultrafast=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_ultrafast", None
),
cache_read_input_token_cost_above_512k_tokens=_model_info.get(
"cache_read_input_token_cost_above_512k_tokens", None
),
Expand Down Expand Up @@ -6166,6 +6172,9 @@ def _get_model_info_helper(
input_cost_per_token_above_272k_tokens_flex=_model_info.get(
"input_cost_per_token_above_272k_tokens_flex", None
),
input_cost_per_token_above_272k_tokens_ultrafast=_model_info.get(
"input_cost_per_token_above_272k_tokens_ultrafast", None
),
input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None),
input_cost_per_query=_model_info.get("input_cost_per_query", None),
cost_per_second=_model_info.get("cost_per_second", None),
Expand Down Expand Up @@ -6233,6 +6242,9 @@ def _get_model_info_helper(
output_cost_per_token_above_272k_tokens_flex=_model_info.get(
"output_cost_per_token_above_272k_tokens_flex", None
),
output_cost_per_token_above_272k_tokens_ultrafast=_model_info.get(
"output_cost_per_token_above_272k_tokens_ultrafast", None
),
output_cost_per_token_above_512k_tokens=_model_info.get(
"output_cost_per_token_above_512k_tokens", None
),
Expand Down
Loading
Loading