From 5bfa4e1a7abad75d52bffd11feeae9ab263bb936 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Fri, 10 Jul 2026 13:57:06 +0300 Subject: [PATCH 1/6] feat(canary): report Reborn inference cost --- .github/workflows/live-canary.yml | 8 +- crates/ironclaw_llm/src/nearai_chat.rs | 44 +++- crates/ironclaw_runner/src/model_gateway.rs | 70 ++++- scripts/live-canary/README.md | 7 + scripts/live-canary/notify_slack.py | 95 +++++++ scripts/live-canary/test_notify_slack.py | 42 +++ .../reborn_webui_v2_live_qa/run_live_qa.py | 239 +++++++++++++++++- .../reborn_webui_v2_live_qa/semantic_judge.py | 49 +++- .../test_run_live_qa.py | 144 +++++++++++ 9 files changed, 680 insertions(+), 18 deletions(-) diff --git a/.github/workflows/live-canary.yml b/.github/workflows/live-canary.yml index 320caf038af..e96098555cb 100644 --- a/.github/workflows/live-canary.yml +++ b/.github/workflows/live-canary.yml @@ -795,8 +795,12 @@ jobs: name: Reborn WebUI v2 Live QA (${{ matrix.shard_name }}) needs: prepare-reborn-webui-v2-live-qa if: > - (github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *') || - (github.event_name == 'workflow_dispatch' && (inputs.lane == 'all' || inputs.lane == 'reborn-webui-v2-live-qa')) + always() && + needs.prepare-reborn-webui-v2-live-qa.result == 'success' && + ( + (github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *') || + (github.event_name == 'workflow_dispatch' && (inputs.lane == 'all' || inputs.lane == 'reborn-webui-v2-live-qa')) + ) runs-on: ubuntu-latest timeout-minutes: 120 permissions: diff --git a/crates/ironclaw_llm/src/nearai_chat.rs b/crates/ironclaw_llm/src/nearai_chat.rs index 089517fce3f..c944465ae45 100644 --- a/crates/ironclaw_llm/src/nearai_chat.rs +++ b/crates/ironclaw_llm/src/nearai_chat.rs @@ -1071,7 +1071,7 @@ impl LlmProvider for NearAiChatProvider { { return rates; } - costs::model_cost(&model).unwrap_or_else(costs::default_cost) + remote_model_fallback_cost(&model) } async fn list_models(&self) -> Result, LlmError> { @@ -1204,6 +1204,18 @@ struct ChatCompletionMessage { tool_calls: Option>, } +fn remote_model_fallback_cost(model: &str) -> (Decimal, Decimal) { + let rates = costs::model_cost(model).unwrap_or_else(costs::default_cost); + if rates == (Decimal::ZERO, Decimal::ZERO) && !is_explicit_free_model(model) { + return costs::default_cost(); + } + rates +} + +fn is_explicit_free_model(model: &str) -> bool { + model.ends_with(":free") || model == "openrouter/free" || model == "free" +} + // -- Pricing fetch types and logic ----------------------------------------- /// Cost amount from the NEAR AI `/v1/model/list` response. @@ -2909,6 +2921,36 @@ data: [DONE] assert_eq!(output, default_out); } + #[test] + fn test_cost_per_token_does_not_treat_remote_nearai_models_as_local_free() { + let mut cfg = test_nearai_config("http://127.0.0.1:8318"); + cfg.model = DEFAULT_MODEL.to_string(); + let provider = NearAiChatProvider::new(cfg, test_session()).expect("provider"); + + let (input, output) = provider.cost_per_token(); + let (default_in, default_out) = costs::default_cost(); + assert_eq!(input, default_in); + assert_eq!(output, default_out); + + provider + .set_model("Qwen/Qwen3-32B") + .expect("set active model"); + let (input, output) = provider.cost_per_token(); + assert_eq!(input, default_in); + assert_eq!(output, default_out); + } + + #[test] + fn test_cost_per_token_preserves_explicit_free_model_fallback() { + let mut cfg = test_nearai_config("http://127.0.0.1:8318"); + cfg.model = "stepfun/step-3.5-flash:free".to_string(); + let provider = NearAiChatProvider::new(cfg, test_session()).expect("provider"); + + let (input, output) = provider.cost_per_token(); + assert_eq!(input, Decimal::ZERO); + assert_eq!(output, Decimal::ZERO); + } + /// Regression: reasoning fallbacks must NOT leak into tool-call responses. #[test] fn test_reasoning_content_not_leaked_into_tool_call_response() { diff --git a/crates/ironclaw_runner/src/model_gateway.rs b/crates/ironclaw_runner/src/model_gateway.rs index 29d6b35d911..b465f3c0cb4 100644 --- a/crates/ironclaw_runner/src/model_gateway.rs +++ b/crates/ironclaw_runner/src/model_gateway.rs @@ -48,7 +48,7 @@ use ironclaw_turns::{ ProviderToolDefinition, RegisterProviderToolCallRequest, sanitize_model_visible_text, }, }; -use tracing::debug; +use tracing::{debug, info}; use crate::{ failure_categories::MODEL_CREDITS_EXHAUSTED_REASON_KIND, @@ -67,8 +67,46 @@ const PROVIDER_TOOL_ARGUMENTS_INVALID_SUMMARY: &str = "model returned invalid to const PROVIDER_TOOL_ARGUMENTS_INVALID_MARKER: &str = "arguments omitted because the provider emitted malformed tool-call JSON"; const CONTEXT_SHADOW_TARGET: &str = "ironclaw::reborn::context_shadow"; +const MODEL_USAGE_TARGET: &str = "ironclaw_runner::model_usage"; const UNAVAILABLE_CAPABILITY_REPLY: &str = "That capability is unavailable or disabled for this request, so I will not route it through another tool."; +fn trace_model_usage( + provider: &P, + operation: &'static str, + input_tokens: u32, + output_tokens: u32, + cache_read_input_tokens: u32, + cache_creation_input_tokens: u32, +) { + if !tracing::enabled!(target: MODEL_USAGE_TARGET, tracing::Level::INFO) { + return; + } + let (input_rate, output_rate) = provider.cost_per_token(); + let estimated_usd = provider.calculate_cost(input_tokens, output_tokens); + info!( + target: MODEL_USAGE_TARGET, + "REBORN_INFERENCE_USAGE operation={} model={} usage_available=true input_tokens={} output_tokens={} cache_read_input_tokens={} cache_creation_input_tokens={} input_usd_per_token={} output_usd_per_token={} estimated_usd={}", + operation, + provider.active_model_name(), + input_tokens, + output_tokens, + cache_read_input_tokens, + cache_creation_input_tokens, + input_rate, + output_rate, + estimated_usd, + ); +} + +fn trace_unpriced_model_call(provider: &P, operation: &'static str) { + info!( + target: MODEL_USAGE_TARGET, + "REBORN_INFERENCE_USAGE operation={} model={} usage_available=false", + operation, + provider.active_model_name(), + ); +} + fn trace_model_latency_ok( operation: &'static str, replay_identity: &ProviderReplayIdentity, @@ -1159,6 +1197,7 @@ where response } Err(error) => { + trace_unpriced_model_call(provider, "provider_complete_with_tools"); trace_model_latency_error( "provider_complete_with_tools", &replay_identity, @@ -1169,6 +1208,14 @@ where return Err(map_provider_error(error)); } }; + trace_model_usage( + provider, + "provider_complete_with_tools", + response.input_tokens, + response.output_tokens, + response.cache_read_input_tokens, + response.cache_creation_input_tokens, + ); let response = recover_textual_tool_calls_from_tool_response(response, &recovery_tool_names)?; let host_response_started_at = live_latency_started_at(); @@ -1224,6 +1271,10 @@ where response } Err(error) => { + trace_unpriced_model_call( + provider, + "provider_complete_with_tools_repair", + ); trace_model_latency_error( "provider_complete_with_tools_repair", &replay_identity, @@ -1234,6 +1285,14 @@ where return Err(map_provider_error(error)); } }; + trace_model_usage( + provider, + "provider_complete_with_tools_repair", + response.input_tokens, + response.output_tokens, + response.cache_read_input_tokens, + response.cache_creation_input_tokens, + ); let mut response = recover_textual_tool_calls_from_tool_response( response, &recovery_tool_names, @@ -1309,6 +1368,7 @@ where response } Err(error) => { + trace_unpriced_model_call(provider, "provider_complete"); trace_model_latency_error( "provider_complete", &replay_identity, @@ -1319,6 +1379,14 @@ where return Err(map_provider_error(error)); } }; + trace_model_usage( + provider, + "provider_complete", + response.input_tokens, + response.output_tokens, + 0, + 0, + ); debug!( finish_reason = ?response.finish_reason, content_bytes = response.content.len(), diff --git a/scripts/live-canary/README.md b/scripts/live-canary/README.md index b7e6dfab51d..b0c7c6527b0 100644 --- a/scripts/live-canary/README.md +++ b/scripts/live-canary/README.md @@ -58,6 +58,13 @@ They must pass the `reborn-live-canary-pr` GitHub environment gate and have an approving review for the exact PR head commit from a collaborator with write access. Scheduled and manual default-branch runs do not require this PR gate. +The lane records provider-reported input/output token counts and an estimated +USD cost for each Reborn and semantic-judge inference. `results.json` contains +per-case and lane totals, and the final Slack report aggregates the full run. +The estimate uses the provider's active per-token rates. Calls that fail before +the provider returns usage are reported as unpriced rather than counted as +zero cost. + ## Local Commands Run the public live smoke lane: diff --git a/scripts/live-canary/notify_slack.py b/scripts/live-canary/notify_slack.py index 5686c44dbd7..c330a69ff0b 100755 --- a/scripts/live-canary/notify_slack.py +++ b/scripts/live-canary/notify_slack.py @@ -22,6 +22,7 @@ import urllib.request import xml.etree.ElementTree as ET from dataclasses import dataclass, field +from decimal import Decimal, InvalidOperation from pathlib import Path MODEL = "claude-haiku-4-5-20251001" @@ -78,6 +79,11 @@ class RebornQaCaseReport: message: str = "" tool_calls: list["RebornQaToolCall"] = field(default_factory=list) debug_paths: list[str] = field(default_factory=list) + inference_call_count: int = 0 + inference_input_tokens: int = 0 + inference_output_tokens: int = 0 + inference_estimated_usd: Decimal = Decimal(0) + inference_unpriced_call_count: int = 0 @dataclass @@ -110,6 +116,11 @@ class LaneReport: root_cause: str = "" fix: str = "" reborn_qa_cases: list[RebornQaCaseReport] = field(default_factory=list) + inference_call_count: int = 0 + inference_input_tokens: int = 0 + inference_output_tokens: int = 0 + inference_estimated_usd: Decimal = Decimal(0) + inference_unpriced_call_count: int = 0 @dataclass(frozen=True) @@ -180,6 +191,17 @@ def parse_results_json(path: Path, report: LaneReport) -> None: results = data.get("results") or [] if not isinstance(results, list): return + usage = data.get("inference_usage") + if isinstance(usage, dict): + report.inference_call_count = _non_negative_int(usage.get("call_count")) + report.inference_input_tokens = _non_negative_int(usage.get("input_tokens")) + report.inference_output_tokens = _non_negative_int(usage.get("output_tokens")) + report.inference_estimated_usd = _non_negative_decimal( + usage.get("estimated_usd") + ) + report.inference_unpriced_call_count = _non_negative_int( + usage.get("unpriced_call_count") + ) for entry in results: if not isinstance(entry, dict): continue @@ -221,6 +243,22 @@ def _trim_slack_text(value: object, limit: int = 180) -> str: return text[: max(0, limit - 1)].rstrip() + "…" +def _non_negative_int(value: object) -> int: + try: + parsed = int(str(value)) + except (TypeError, ValueError): + return 0 + return max(parsed, 0) + + +def _non_negative_decimal(value: object) -> Decimal: + try: + parsed = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError): + return Decimal(0) + return parsed if parsed >= 0 else Decimal(0) + + def _trim_slack_block_text(value: object, limit: int = 2900) -> str: text = str(value or "").strip() text = "\n".join(re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()) @@ -391,6 +429,9 @@ def parse_reborn_qa_case_reports(lane_dir: Path, report: LaneReport) -> None: or case.replace("_", " ") ) latency = entry.get("latency_ms") + inference_usage = details.get("inference_usage") + if not isinstance(inference_usage, dict): + inference_usage = {} tool_calls = parse_reborn_trace_tool_calls(lane_dir / "traces" / f"{case}.json") debug_paths = [] if not entry.get("success"): @@ -410,6 +451,21 @@ def parse_reborn_qa_case_reports(lane_dir: Path, report: LaneReport) -> None: message=_reborn_failure_message(entry), tool_calls=tool_calls, debug_paths=debug_paths, + inference_call_count=_non_negative_int( + inference_usage.get("call_count") + ), + inference_input_tokens=_non_negative_int( + inference_usage.get("input_tokens") + ), + inference_output_tokens=_non_negative_int( + inference_usage.get("output_tokens") + ), + inference_estimated_usd=_non_negative_decimal( + inference_usage.get("estimated_usd") + ), + inference_unpriced_call_count=_non_negative_int( + inference_usage.get("unpriced_call_count") + ), ) ) report.reborn_qa_cases = cases @@ -671,6 +727,20 @@ def _format_reborn_qa_group( f"{status} *QA {group}* — {passed}/{len(cases)} passed{duration}", f"*Cases:* {_trim_slack_text('; '.join(case_summaries), 900)}", ] + inference_calls = sum(case.inference_call_count for case in cases) + if inference_calls: + inference_usd = sum( + (case.inference_estimated_usd for case in cases), + Decimal(0), + ) + input_tokens = sum(case.inference_input_tokens for case in cases) + output_tokens = sum(case.inference_output_tokens for case in cases) + unpriced = sum(case.inference_unpriced_call_count for case in cases) + unpriced_text = f"; {unpriced} unpriced" if unpriced else "" + lines.append( + f"*Inference:* {inference_calls} calls; {input_tokens:,} input + " + f"{output_tokens:,} output tokens; estimated `${inference_usd:.6f}`{unpriced_text}" + ) lines.extend(_format_reborn_failure_lines(cases, run_url)) lines.extend(_format_reborn_tool_summary(cases)) blocks: list[dict] = [] @@ -776,6 +846,31 @@ def slack_payload( blocks.append( {"type": "context", "elements": [{"type": "mrkdwn", "text": context_text}]} ) + inference_calls = sum(r.inference_call_count for r in reports) + if inference_calls: + inference_usd = sum( + (r.inference_estimated_usd for r in reports), + Decimal(0), + ) + input_tokens = sum(r.inference_input_tokens for r in reports) + output_tokens = sum(r.inference_output_tokens for r in reports) + unpriced = sum(r.inference_unpriced_call_count for r in reports) + unpriced_text = f" • {unpriced} calls unpriced" if unpriced else "" + blocks.append( + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": ( + f"*Reborn inference estimate:* `${inference_usd:.6f}` • " + f"{inference_calls} calls • {input_tokens:,} input + " + f"{output_tokens:,} output tokens{unpriced_text}" + ), + } + ], + } + ) for r in reports: renders_reborn_qa_groups = bool(r.reborn_qa_cases) header_line = ( diff --git a/scripts/live-canary/test_notify_slack.py b/scripts/live-canary/test_notify_slack.py index 439ed2c9d6c..0e0367ba89b 100644 --- a/scripts/live-canary/test_notify_slack.py +++ b/scripts/live-canary/test_notify_slack.py @@ -193,6 +193,13 @@ def test_collect_lane_populates_per_case_reports(self): (lane_dir / "results.json").write_text( json.dumps( { + "inference_usage": { + "call_count": 2, + "input_tokens": 150, + "output_tokens": 30, + "estimated_usd": "0.000210", + "unpriced_call_count": 0, + }, "results": [ { "provider": "reborn-webui-v2", @@ -202,6 +209,13 @@ def test_collect_lane_populates_per_case_reports(self): "details": { "case": "qa_2a_gmail_connect", "gate": "requires live Google browser consent state", + "inference_usage": { + "call_count": 2, + "input_tokens": 150, + "output_tokens": 30, + "estimated_usd": "0.000210", + "unpriced_call_count": 0, + }, }, }, { @@ -260,10 +274,19 @@ def test_collect_lane_populates_per_case_reports(self): self.assertEqual(report.tests, 2) self.assertEqual(report.passed, 1) self.assertEqual(report.failed, 1) + self.assertEqual(report.inference_call_count, 2) + self.assertEqual(report.inference_input_tokens, 150) + self.assertEqual(report.inference_output_tokens, 30) + self.assertEqual(report.inference_estimated_usd, notify.Decimal("0.000210")) self.assertEqual(len(report.reborn_qa_cases), 2) self.assertEqual(report.reborn_qa_cases[0].rows, ("2A",)) self.assertEqual(report.reborn_qa_cases[0].feature, "Gmail connection flow") self.assertEqual(report.reborn_qa_cases[0].message, "") + self.assertEqual(report.reborn_qa_cases[0].inference_call_count, 2) + self.assertEqual( + report.reborn_qa_cases[0].inference_estimated_usd, + notify.Decimal("0.000210"), + ) self.assertEqual(len(report.reborn_qa_cases[0].tool_calls), 1) self.assertEqual(report.reborn_qa_cases[0].tool_calls[0].name, "gmail.list_messages") self.assertEqual(report.reborn_qa_cases[0].tool_calls[0].args_hash, "1234567890123") @@ -293,6 +316,10 @@ def test_slack_payload_renders_each_reborn_qa_row(self): tests=3, duration_s=1.2, status="fail", + inference_call_count=2, + inference_input_tokens=150, + inference_output_tokens=30, + inference_estimated_usd=notify.Decimal("0.000210"), reborn_qa_cases=[ notify.RebornQaCaseReport( rows=("2A",), @@ -300,6 +327,10 @@ def test_slack_payload_renders_each_reborn_qa_row(self): feature="Gmail connection flow", success=True, latency_ms=1200, + inference_call_count=2, + inference_input_tokens=150, + inference_output_tokens=30, + inference_estimated_usd=notify.Decimal("0.000210"), tool_calls=[ notify.RebornQaToolCall( name="gmail.list_messages", @@ -356,6 +387,17 @@ def test_slack_payload_renders_each_reborn_qa_row(self): qa_sections = [text for text in section_texts if "*QA 2*" in text] self.assertEqual(len(qa_sections), 1) + self.assertIn("*Inference:* 2 calls", qa_sections[0]) + self.assertIn("estimated `$0.000210`", qa_sections[0]) + context_texts = [ + element["text"] + for block in payload["blocks"] + if block.get("type") == "context" + for element in block.get("elements", []) + ] + self.assertTrue( + any("*Reborn inference estimate:* `$0.000210`" in text for text in context_texts) + ) self.assertTrue( any( "*reborn-webui-v2-live-qa* (reborn-webui-v2) — 1/3 passed" diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index 2004e4d0121..a044ae69947 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -24,6 +24,7 @@ from contextlib import closing from dataclasses import dataclass, field from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, Awaitable, Callable @@ -739,6 +740,18 @@ def server_env( env = os.environ.copy() if extra_env: env.update(extra_env) + rust_log = _log_filter_with_model_usage( + os.environ.get( + "RUST_LOG", + "ironclaw=warn,ironclaw_reborn=warn,ironclaw_reborn_webui_ingress=info", + ) + ) + reborn_log = _log_filter_with_model_usage( + os.environ.get( + "IRONCLAW_REBORN_LOG", + "info,ironclaw_runner=info,ironclaw_reborn_composition=info", + ) + ) env.update( { "HOME": str(process_home), @@ -749,10 +762,8 @@ def server_env( "NO_PROXY": "127.0.0.1,localhost,::1", "no_proxy": "127.0.0.1,localhost,::1", "RUST_BACKTRACE": "1", - "RUST_LOG": os.environ.get( - "RUST_LOG", - "ironclaw=warn,ironclaw_runner=warn,ironclaw_reborn_webui_ingress=info", - ), + "RUST_LOG": rust_log, + "IRONCLAW_REBORN_LOG": reborn_log, } ) env.setdefault("IRONCLAW_TRIGGER_POLLER_ENABLED", "true") @@ -760,6 +771,12 @@ def server_env( return env +def _log_filter_with_model_usage(value: str) -> str: + if "ironclaw_runner::model_usage" in value: + return value + return f"{value},ironclaw_runner::model_usage=info" + + async def start_reborn_server( binary: Path, reborn_home: Path, @@ -1339,18 +1356,22 @@ async def action(page: object) -> None: }, ) except Exception as exc: + failure_details: dict[str, object] = { + "error": _exc_text(exc), + "prompt": prompt, + "marker": marker, + "required_text": required_text, + **(extra_details or {}), + **observed, + } + semantic_judge = getattr(exc, "semantic_judge", None) + if isinstance(semantic_judge, dict): + failure_details["semantic_judge"] = semantic_judge return _result( case_name, False, started, - { - "error": _exc_text(exc), - "prompt": prompt, - "marker": marker, - "required_text": required_text, - **(extra_details or {}), - **observed, - }, + failure_details, ) @@ -1568,13 +1589,16 @@ async def _wait_for_assistant_reply( ), semantic_judge=semantic_judge, ) - raise AssertionError( + error = AssertionError( "assistant reply did not contain required text before timeout. " f"marker={marker!r} required_text={required_text!r} " f"latest_final_reply_state={last_final_reply_state!r} " f"last_assistant={last_text[-500:]!r} main_excerpt={main_text[-1000:]!r} " f"semantic_judge={_compact_json(semantic_judge)}" ) + if semantic_judge is not None: + setattr(error, "semantic_judge", semantic_judge) + raise error async def _approve_visible_tool_gate(page: object) -> None: @@ -6850,6 +6874,187 @@ def write_trace_index(output_dir: Path, traces: list[dict[str, object]]) -> Path return path +INFERENCE_USAGE_MARKER = "REBORN_INFERENCE_USAGE " +SERVER_START_MARKER = "--- ironclaw-reborn serve start " + + +def _non_negative_int(value: object) -> int: + try: + parsed = int(str(value)) + except (TypeError, ValueError): + return 0 + return max(parsed, 0) + + +def _decimal(value: object) -> Decimal | None: + try: + parsed = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError): + return None + return parsed if parsed >= 0 else None + + +def _product_inference_usage(output_dir: Path) -> list[dict[str, object]]: + events: list[dict[str, object]] = [] + for name in ("ironclaw-reborn-serve.stdout.log", "ironclaw-reborn-serve.stderr.log"): + path = output_dir / name + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + segment = text.rsplit(SERVER_START_MARKER, 1)[-1] + for line in segment.splitlines(): + if INFERENCE_USAGE_MARKER not in line: + continue + fields = dict( + token.split("=", 1) + for token in line.split(INFERENCE_USAGE_MARKER, 1)[1].split() + if "=" in token + ) + if fields.get("usage_available") == "false": + events.append( + { + "source": "product", + "operation": str(fields.get("operation") or "unknown"), + "model": str(fields.get("model") or "unknown"), + "input_tokens": 0, + "output_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "pricing_source": "provider_usage_unavailable", + } + ) + continue + input_rate = _decimal(fields.get("input_usd_per_token")) + output_rate = _decimal(fields.get("output_usd_per_token")) + estimated_usd = _decimal(fields.get("estimated_usd")) + if input_rate is None or output_rate is None or estimated_usd is None: + continue + events.append( + { + "source": "product", + "operation": str(fields.get("operation") or "unknown"), + "model": str(fields.get("model") or "unknown"), + "input_tokens": _non_negative_int(fields.get("input_tokens")), + "output_tokens": _non_negative_int(fields.get("output_tokens")), + "cache_read_input_tokens": _non_negative_int( + fields.get("cache_read_input_tokens") + ), + "cache_creation_input_tokens": _non_negative_int( + fields.get("cache_creation_input_tokens") + ), + "input_usd_per_token": str(input_rate), + "output_usd_per_token": str(output_rate), + "estimated_usd": str(estimated_usd), + "pricing_source": "provider_active_rate", + } + ) + return events + + +def _semantic_judge_usage(value: object) -> list[dict[str, object]]: + found: list[dict[str, object]] = [] + if isinstance(value, dict): + if value.get("source") == "semantic_judge": + found.append(dict(value)) + else: + for child in value.values(): + found.extend(_semantic_judge_usage(child)) + elif isinstance(value, list): + for child in value: + found.extend(_semantic_judge_usage(child)) + return found + + +def _summarize_inference_usage(events: list[dict[str, object]]) -> dict[str, object]: + rates_by_model: dict[str, tuple[Decimal, Decimal]] = {} + for event in events: + input_rate = _decimal(event.get("input_usd_per_token")) + output_rate = _decimal(event.get("output_usd_per_token")) + if event.get("source") == "product" and input_rate is not None and output_rate is not None: + rates_by_model[str(event.get("model") or "unknown")] = (input_rate, output_rate) + + priced_events: list[dict[str, object]] = [] + unpriced_calls = 0 + total_usd = Decimal(0) + for event in events: + event = dict(event) + cost = _decimal(event.get("estimated_usd")) + if ( + cost is None + and event.get("source") == "semantic_judge" + and event.get("pricing_source") != "provider_usage_unavailable" + ): + rates = rates_by_model.get(str(event.get("model") or "unknown")) + if rates is not None: + cost = ( + Decimal(_non_negative_int(event.get("input_tokens"))) * rates[0] + + Decimal(_non_negative_int(event.get("output_tokens"))) * rates[1] + ) + event["input_usd_per_token"] = str(rates[0]) + event["output_usd_per_token"] = str(rates[1]) + event["estimated_usd"] = str(cost) + event["pricing_source"] = "matched_product_rate" + if cost is None: + unpriced_calls += 1 + else: + total_usd += cost + priced_events.append(event) + + return { + "call_count": len(priced_events), + "input_tokens": sum(_non_negative_int(e.get("input_tokens")) for e in priced_events), + "output_tokens": sum(_non_negative_int(e.get("output_tokens")) for e in priced_events), + "cache_read_input_tokens": sum( + _non_negative_int(e.get("cache_read_input_tokens")) for e in priced_events + ), + "cache_creation_input_tokens": sum( + _non_negative_int(e.get("cache_creation_input_tokens")) for e in priced_events + ), + "estimated_usd": str(total_usd), + "unpriced_call_count": unpriced_calls, + "cost_kind": "estimated_from_provider_reported_tokens_and_active_rates", + "events": priced_events, + } + + +def _case_inference_usage(output_dir: Path, result: ProbeResult) -> dict[str, object]: + events = _product_inference_usage(output_dir) + events.extend(_semantic_judge_usage(result.details)) + return _summarize_inference_usage(events) + + +def _write_results_inference_summary(path: Path, results: list[ProbeResult]) -> None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return + case_summaries = [ + result.details.get("inference_usage") + for result in results + if isinstance(result.details.get("inference_usage"), dict) + ] + payload["inference_usage"] = { + "call_count": sum(_non_negative_int(item.get("call_count")) for item in case_summaries), + "input_tokens": sum(_non_negative_int(item.get("input_tokens")) for item in case_summaries), + "output_tokens": sum(_non_negative_int(item.get("output_tokens")) for item in case_summaries), + "estimated_usd": str( + sum( + ( + _decimal(item.get("estimated_usd")) or Decimal(0) + for item in case_summaries + ), + Decimal(0), + ) + ), + "unpriced_call_count": sum( + _non_negative_int(item.get("unpriced_call_count")) for item in case_summaries + ), + "cost_kind": "estimated_from_provider_reported_tokens_and_active_rates", + } + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + def write_preflight(output_dir: Path, prepared_home: PreparedRebornHome) -> Path: payload = { "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), @@ -7163,6 +7368,7 @@ async def run_cases(args: argparse.Namespace) -> int: ) if not first_base_url: first_base_url = base_url + case_result: ProbeResult | None = None try: ctx = LiveQaContext( base_url=base_url, @@ -7207,6 +7413,7 @@ async def run_cases(args: argparse.Namespace) -> int: print(f"[reborn-webui-v2-live-qa] running case={name}", flush=True) result = await CASES[name].fn(ctx) result = _attach_browser_diagnostics(args.output_dir, result) + case_result = result results.append(result) print( f"[reborn-webui-v2-live-qa] case={name} success={result.success} " @@ -7215,6 +7422,11 @@ async def run_cases(args: argparse.Namespace) -> int: ) finally: stop_process(proc) + if case_result is not None: + case_result.details["inference_usage"] = _case_inference_usage( + args.output_dir, + case_result, + ) trace_export = export_case_trace(args.output_dir, name, prepared_home.path) trace_exports.append(trace_export) print( @@ -7223,6 +7435,7 @@ async def run_cases(args: argparse.Namespace) -> int: flush=True, ) results_path = write_results(args.output_dir, results, first_base_url) + _write_results_inference_summary(results_path, results) trace_index_path = write_trace_index(args.output_dir, trace_exports) green_explanation_path = write_green_run_explanation(args.output_dir, results) print(f"[reborn-webui-v2-live-qa] results={results_path}", flush=True) diff --git a/scripts/reborn_webui_v2_live_qa/semantic_judge.py b/scripts/reborn_webui_v2_live_qa/semantic_judge.py index e82d09825a6..b3ff5c18e8e 100644 --- a/scripts/reborn_webui_v2_live_qa/semantic_judge.py +++ b/scripts/reborn_webui_v2_live_qa/semantic_judge.py @@ -45,7 +45,11 @@ async def _judge_assistant_reply_completion( body = response.json() content = _completion_content(body) except Exception as exc: - return {"enabled": True, "error": str(exc)} + return { + "enabled": True, + "error": str(exc), + "inference_usage": _unpriced_completion_usage(), + } parsed = _parse_json_object(content) if not isinstance(parsed, dict): @@ -53,8 +57,11 @@ async def _judge_assistant_reply_completion( "enabled": True, "error": "judge response was not a JSON object", "response_excerpt": str(content)[-500:], + "inference_usage": _completion_usage(body) + or _unpriced_completion_usage(), } parsed["enabled"] = True + parsed["inference_usage"] = _completion_usage(body) or _unpriced_completion_usage() return parsed @@ -181,6 +188,46 @@ def _completion_content(body: object) -> object: return message.get("content") or "" +def _completion_usage(body: object) -> dict[str, object] | None: + if not isinstance(body, dict): + return None + usage = body.get("usage") + if not isinstance(usage, dict): + return None + + def token_count(name: str) -> int: + value = usage.get(name) + return value if isinstance(value, int) and value >= 0 else 0 + + prompt_details = usage.get("prompt_tokens_details") + cached_tokens = 0 + if isinstance(prompt_details, dict): + value = prompt_details.get("cached_tokens") + if isinstance(value, int) and value >= 0: + cached_tokens = value + return { + "source": "semantic_judge", + "model": str(body.get("model") or _judge_model()), + "input_tokens": token_count("prompt_tokens"), + "output_tokens": token_count("completion_tokens"), + "cache_read_input_tokens": cached_tokens, + "cache_creation_input_tokens": 0, + "pricing_source": "pending_product_rate_match", + } + + +def _unpriced_completion_usage() -> dict[str, object]: + return { + "source": "semantic_judge", + "model": _judge_model(), + "input_tokens": 0, + "output_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "pricing_source": "provider_usage_unavailable", + } + + def _parse_json_object(text: object) -> object: if not isinstance(text, str): return None diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index d3fdac259cd..35f869be260 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -1254,6 +1254,49 @@ async def fake_sleep(_seconds): ) ) + def test_wait_for_assistant_reply_attaches_failed_semantic_judge_to_error(self): + response_text = "I found something adjacent but did not complete the requested task." + judge_payload = { + "completed": False, + "confidence": 0.85, + "reason": "The response does not complete the task.", + "inference_usage": { + "source": "semantic_judge", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "input_tokens": 10, + "output_tokens": 5, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + }, + } + + async def fake_judge(**_kwargs): + return judge_payload + + async def fake_sleep(_seconds): + return None + + with ( + patch.object( + run_live_qa, + "_judge_assistant_reply_completion", + side_effect=fake_judge, + ), + patch.object(run_live_qa.asyncio, "sleep", side_effect=fake_sleep), + ): + with self.assertRaisesRegex(AssertionError, "semantic_judge=") as raised: + asyncio.run( + run_live_qa._wait_for_assistant_reply( + self._fake_assistant_reply_page(response_text), + marker=None, + required_text=["completed"], + timeout=0.001, + semantic_goal="Complete the requested task.", + ) + ) + + self.assertEqual(getattr(raised.exception, "semantic_judge"), judge_payload) + def test_semantic_judge_passed_respects_confidence_threshold(self): with patch.dict( os.environ, @@ -1290,6 +1333,57 @@ def test_semantic_judge_completion_content_handles_unexpected_shapes(self): '{"completed": true}', ) + def test_semantic_judge_completion_usage_extracts_provider_counts(self): + usage = semantic_judge._completion_usage( + { + "model": "deepseek-ai/DeepSeek-V4-Flash", + "usage": { + "prompt_tokens": 123, + "completion_tokens": 45, + "total_tokens": 168, + "prompt_tokens_details": {"cached_tokens": 12}, + }, + } + ) + + self.assertEqual( + usage, + { + "source": "semantic_judge", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "input_tokens": 123, + "output_tokens": 45, + "cache_read_input_tokens": 12, + "cache_creation_input_tokens": 0, + "pricing_source": "pending_product_rate_match", + }, + ) + + def test_semantic_judge_unpriced_usage_is_explicit(self): + usage = semantic_judge._unpriced_completion_usage() + + self.assertEqual(usage["source"], "semantic_judge") + self.assertEqual(usage["pricing_source"], "provider_usage_unavailable") + self.assertEqual(usage["input_tokens"], 0) + self.assertEqual(usage["output_tokens"], 0) + + def test_server_env_enables_usage_telemetry_with_custom_log_filters(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + with patch.dict( + os.environ, + { + "RUST_LOG": "custom_crate=debug", + "IRONCLAW_REBORN_LOG": "ironclaw_reborn=warn", + }, + ): + env = run_live_qa.server_env(root / "reborn", root / "process") + + self.assertIn("custom_crate=debug", env["RUST_LOG"]) + self.assertIn("ironclaw_runner::model_usage=info", env["RUST_LOG"]) + self.assertIn("ironclaw_reborn=warn", env["IRONCLAW_REBORN_LOG"]) + self.assertIn("ironclaw_runner::model_usage=info", env["IRONCLAW_REBORN_LOG"]) + def test_semantic_judge_json_parser_handles_non_string_inputs(self): self.assertIsNone(semantic_judge._parse_json_object(None)) self.assertIsNone(semantic_judge._parse_json_object(123)) @@ -4714,6 +4808,10 @@ def test_live_canary_workflow_shards_cover_non_telegram_qa_suite(self): workflow, ) self.assertIn("needs: prepare-reborn-webui-v2-live-qa", match.group("body")) + self.assertIn( + "always() &&\n needs.prepare-reborn-webui-v2-live-qa.result == 'success'", + match.group("body"), + ) self.assertIn( "ref: ${{ needs.prepare-reborn-webui-v2-live-qa.outputs.checkout_ref }}", match.group("body"), @@ -5239,6 +5337,52 @@ def test_export_case_trace_writes_runtime_entries_without_secret_store(self): self.assertNotIn(secret_path, paths) self.assertEqual(payload["entries"][1]["contents"]["content"], "hello live trace") + def test_case_inference_usage_aggregates_product_and_judge_calls(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + (output_dir / "ironclaw-reborn-serve.stderr.log").write_text( + "old log\n" + "--- ironclaw-reborn serve start 2026-07-10T12:00:00Z ---\n" + "INFO REBORN_INFERENCE_USAGE operation=provider_complete_with_tools " + "model=deepseek-ai/DeepSeek-V4-Flash input_tokens=100 " + "output_tokens=20 cache_read_input_tokens=0 " + "cache_creation_input_tokens=0 input_usd_per_token=0.000001 " + "output_usd_per_token=0.000002 estimated_usd=0.000140\n" + "INFO REBORN_INFERENCE_USAGE operation=provider_complete " + "model=deepseek-ai/DeepSeek-V4-Flash usage_available=false\n", + encoding="utf-8", + ) + result = run_live_qa.ProbeResult( + provider="test", + mode="live:test", + success=True, + latency_ms=1, + details={ + "semantic_judge": { + "inference_usage": { + "source": "semantic_judge", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "input_tokens": 50, + "output_tokens": 10, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + } + } + }, + ) + + summary = run_live_qa._case_inference_usage(output_dir, result) + + self.assertEqual(summary["call_count"], 3) + self.assertEqual(summary["input_tokens"], 150) + self.assertEqual(summary["output_tokens"], 30) + self.assertEqual(summary["estimated_usd"], "0.000210") + self.assertEqual(summary["unpriced_call_count"], 1) + self.assertEqual( + summary["events"][2]["pricing_source"], + "matched_product_rate", + ) + def test_run_cases_isolates_reborn_home_and_preflight_per_selected_case(self): async def fake_case(ctx: run_live_qa.LiveQaContext) -> run_live_qa.ProbeResult: return run_live_qa.ProbeResult( From ede7b9eab6c2ab70206702da60b22ce3a4df444e Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Fri, 10 Jul 2026 16:14:47 +0300 Subject: [PATCH 2/6] fix(canary): address review telemetry comments --- crates/ironclaw_runner/src/model_gateway.rs | 91 +++++++++++--- scripts/live-canary/notify_slack.py | 4 +- scripts/live-canary/test_notify_slack.py | 13 ++ .../reborn_webui_v2_live_qa/run_live_qa.py | 67 ++++++++-- .../reborn_webui_v2_live_qa/semantic_judge.py | 16 ++- .../test_run_live_qa.py | 114 ++++++++++++++++-- 6 files changed, 265 insertions(+), 40 deletions(-) diff --git a/crates/ironclaw_runner/src/model_gateway.rs b/crates/ironclaw_runner/src/model_gateway.rs index b465f3c0cb4..8c66654720b 100644 --- a/crates/ironclaw_runner/src/model_gateway.rs +++ b/crates/ironclaw_runner/src/model_gateway.rs @@ -48,7 +48,8 @@ use ironclaw_turns::{ ProviderToolDefinition, RegisterProviderToolCallRequest, sanitize_model_visible_text, }, }; -use tracing::{debug, info}; +use rust_decimal::Decimal; +use tracing::debug; use crate::{ failure_categories::MODEL_CREDITS_EXHAUSTED_REASON_KIND, @@ -70,40 +71,88 @@ const CONTEXT_SHADOW_TARGET: &str = "ironclaw::reborn::context_shadow"; const MODEL_USAGE_TARGET: &str = "ironclaw_runner::model_usage"; const UNAVAILABLE_CAPABILITY_REPLY: &str = "That capability is unavailable or disabled for this request, so I will not route it through another tool."; -fn trace_model_usage( +#[derive(Debug, Clone)] +struct ModelUsageContext { + model: String, + input_usd_per_token: Decimal, + output_usd_per_token: Decimal, +} + +fn normalized_request_model_override(model: Option<&str>) -> Option<&str> { + let trimmed = model?.trim(); + if trimmed.is_empty() || trimmed.eq_ignore_ascii_case("default") { + None + } else { + Some(trimmed) + } +} + +fn fallback_usage_rates_for_model(model: &str) -> (Decimal, Decimal) { + let rates = model_cost(model).unwrap_or_else(default_cost); + if rates == (Decimal::ZERO, Decimal::ZERO) && !is_explicit_free_model(model) { + return default_cost(); + } + rates +} + +fn is_explicit_free_model(model: &str) -> bool { + model.ends_with(":free") || model == "openrouter/free" || model == "free" +} + +fn model_usage_context( provider: &P, + model_override: Option<&str>, +) -> ModelUsageContext { + let active_model = provider.active_model_name(); + let effective_model = normalized_request_model_override(model_override) + .map(str::to_string) + .unwrap_or_else(|| active_model.clone()); + let (input_usd_per_token, output_usd_per_token) = if effective_model == active_model { + provider.cost_per_token() + } else { + fallback_usage_rates_for_model(&effective_model) + }; + ModelUsageContext { + model: effective_model, + input_usd_per_token, + output_usd_per_token, + } +} + +fn trace_model_usage( + usage: &ModelUsageContext, operation: &'static str, input_tokens: u32, output_tokens: u32, cache_read_input_tokens: u32, cache_creation_input_tokens: u32, ) { - if !tracing::enabled!(target: MODEL_USAGE_TARGET, tracing::Level::INFO) { + if !tracing::enabled!(target: MODEL_USAGE_TARGET, tracing::Level::DEBUG) { return; } - let (input_rate, output_rate) = provider.cost_per_token(); - let estimated_usd = provider.calculate_cost(input_tokens, output_tokens); - info!( + let estimated_usd = usage.input_usd_per_token * Decimal::from(input_tokens) + + usage.output_usd_per_token * Decimal::from(output_tokens); + debug!( target: MODEL_USAGE_TARGET, "REBORN_INFERENCE_USAGE operation={} model={} usage_available=true input_tokens={} output_tokens={} cache_read_input_tokens={} cache_creation_input_tokens={} input_usd_per_token={} output_usd_per_token={} estimated_usd={}", operation, - provider.active_model_name(), + usage.model, input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens, - input_rate, - output_rate, + usage.input_usd_per_token, + usage.output_usd_per_token, estimated_usd, ); } -fn trace_unpriced_model_call(provider: &P, operation: &'static str) { - info!( +fn trace_unpriced_model_call(usage: &ModelUsageContext, operation: &'static str) { + debug!( target: MODEL_USAGE_TARGET, "REBORN_INFERENCE_USAGE operation={} model={} usage_available=false", operation, - provider.active_model_name(), + usage.model, ); } @@ -1175,6 +1224,7 @@ where .collect::>(); let tool_request = ToolCompletionRequest::from_completion_request(completion, llm_tool_definitions); + let tool_model_usage = model_usage_context(provider, tool_request.model.as_deref()); debug!("reborn model gateway dispatching tool-capable provider request"); let provider_started_at = live_latency_started_at(); let response = match if let Some(stream_sink) = stream_sink.as_ref() { @@ -1197,7 +1247,7 @@ where response } Err(error) => { - trace_unpriced_model_call(provider, "provider_complete_with_tools"); + trace_unpriced_model_call(&tool_model_usage, "provider_complete_with_tools"); trace_model_latency_error( "provider_complete_with_tools", &replay_identity, @@ -1209,7 +1259,7 @@ where } }; trace_model_usage( - provider, + &tool_model_usage, "provider_complete_with_tools", response.input_tokens, response.output_tokens, @@ -1272,7 +1322,7 @@ where } Err(error) => { trace_unpriced_model_call( - provider, + &tool_model_usage, "provider_complete_with_tools_repair", ); trace_model_latency_error( @@ -1286,7 +1336,7 @@ where } }; trace_model_usage( - provider, + &tool_model_usage, "provider_complete_with_tools_repair", response.input_tokens, response.output_tokens, @@ -1348,6 +1398,7 @@ where } let provider_started_at = live_latency_started_at(); + let text_model_usage = model_usage_context(provider, completion.model.as_deref()); let response = match if let Some(stream_sink) = stream_sink.as_ref() { provider .complete_streaming( @@ -1368,7 +1419,7 @@ where response } Err(error) => { - trace_unpriced_model_call(provider, "provider_complete"); + trace_unpriced_model_call(&text_model_usage, "provider_complete"); trace_model_latency_error( "provider_complete", &replay_identity, @@ -1380,12 +1431,12 @@ where } }; trace_model_usage( - provider, + &text_model_usage, "provider_complete", response.input_tokens, response.output_tokens, - 0, - 0, + response.cache_read_input_tokens, + response.cache_creation_input_tokens, ); debug!( finish_reason = ?response.finish_reason, diff --git a/scripts/live-canary/notify_slack.py b/scripts/live-canary/notify_slack.py index c330a69ff0b..ecab3af7122 100755 --- a/scripts/live-canary/notify_slack.py +++ b/scripts/live-canary/notify_slack.py @@ -256,7 +256,9 @@ def _non_negative_decimal(value: object) -> Decimal: parsed = Decimal(str(value)) except (InvalidOperation, TypeError, ValueError): return Decimal(0) - return parsed if parsed >= 0 else Decimal(0) + if not parsed.is_finite() or parsed < 0: + return Decimal(0) + return parsed def _trim_slack_block_text(value: object, limit: int = 2900) -> str: diff --git a/scripts/live-canary/test_notify_slack.py b/scripts/live-canary/test_notify_slack.py index 0e0367ba89b..e5b9df5ca12 100644 --- a/scripts/live-canary/test_notify_slack.py +++ b/scripts/live-canary/test_notify_slack.py @@ -186,6 +186,15 @@ def test_status_row_with_extra_whitespace(self): class RebornQaSlackReportTests(unittest.TestCase): + def test_non_negative_decimal_rejects_non_finite_values(self): + self.assertEqual(notify._non_negative_decimal("NaN"), notify.Decimal(0)) + self.assertEqual(notify._non_negative_decimal("Infinity"), notify.Decimal(0)) + self.assertEqual(notify._non_negative_decimal("-Infinity"), notify.Decimal(0)) + self.assertEqual( + notify._non_negative_decimal("0.000210"), + notify.Decimal("0.000210"), + ) + def test_collect_lane_populates_per_case_reports(self): with tempfile.TemporaryDirectory() as tmpdir: lane_dir = Path(tmpdir) / "reborn-webui-v2-live-qa" / "reborn-webui-v2" / "20260628T000000Z" @@ -320,6 +329,7 @@ def test_slack_payload_renders_each_reborn_qa_row(self): inference_input_tokens=150, inference_output_tokens=30, inference_estimated_usd=notify.Decimal("0.000210"), + inference_unpriced_call_count=1, reborn_qa_cases=[ notify.RebornQaCaseReport( rows=("2A",), @@ -331,6 +341,7 @@ def test_slack_payload_renders_each_reborn_qa_row(self): inference_input_tokens=150, inference_output_tokens=30, inference_estimated_usd=notify.Decimal("0.000210"), + inference_unpriced_call_count=1, tool_calls=[ notify.RebornQaToolCall( name="gmail.list_messages", @@ -389,6 +400,7 @@ def test_slack_payload_renders_each_reborn_qa_row(self): self.assertEqual(len(qa_sections), 1) self.assertIn("*Inference:* 2 calls", qa_sections[0]) self.assertIn("estimated `$0.000210`", qa_sections[0]) + self.assertIn("1 unpriced", qa_sections[0]) context_texts = [ element["text"] for block in payload["blocks"] @@ -398,6 +410,7 @@ def test_slack_payload_renders_each_reborn_qa_row(self): self.assertTrue( any("*Reborn inference estimate:* `$0.000210`" in text for text in context_texts) ) + self.assertTrue(any("1 calls unpriced" in text for text in context_texts)) self.assertTrue( any( "*reborn-webui-v2-live-qa* (reborn-webui-v2) — 1/3 passed" diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index a044ae69947..acd8e316c3e 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -741,13 +741,13 @@ def server_env( if extra_env: env.update(extra_env) rust_log = _log_filter_with_model_usage( - os.environ.get( + env.get( "RUST_LOG", "ironclaw=warn,ironclaw_reborn=warn,ironclaw_reborn_webui_ingress=info", ) ) reborn_log = _log_filter_with_model_usage( - os.environ.get( + env.get( "IRONCLAW_REBORN_LOG", "info,ironclaw_runner=info,ironclaw_reborn_composition=info", ) @@ -774,7 +774,7 @@ def server_env( def _log_filter_with_model_usage(value: str) -> str: if "ironclaw_runner::model_usage" in value: return value - return f"{value},ironclaw_runner::model_usage=info" + return f"{value},ironclaw_runner::model_usage=debug" async def start_reborn_server( @@ -1208,7 +1208,55 @@ def _record_assistant_reply_wait_result( observed["assistant_reply_wait_ms"] = reply.final_reply_wait_ms observed["assistant_reply_wait_reason"] = reply.final_reply_reason if reply.semantic_judge is not None: - observed["semantic_judge"] = reply.semantic_judge + _append_semantic_judge(observed, reply.semantic_judge) + + +def _safe_semantic_judge_usage(value: object) -> dict[str, object] | None: + if not isinstance(value, dict): + return None + safe: dict[str, object] = {} + for key in ( + "source", + "model", + "pricing_source", + "estimated_usd", + "input_usd_per_token", + "output_usd_per_token", + ): + if key in value: + safe[key] = str(value[key]) + for key in ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ): + safe[key] = _non_negative_int(value.get(key)) + return safe + + +def _safe_semantic_judge_payload(value: dict[str, object]) -> dict[str, object]: + safe: dict[str, object] = {} + if isinstance(value.get("completed"), bool): + safe["completed"] = value["completed"] + if isinstance(value.get("confidence"), (int, float)): + safe["confidence"] = value["confidence"] + reason = value.get("reason") + if isinstance(reason, str): + safe["reason"] = reason[:300] + usage = _safe_semantic_judge_usage(value.get("inference_usage")) + if usage is not None: + safe["inference_usage"] = usage + return safe + + +def _append_semantic_judge( + details: dict[str, object], + semantic_judge: dict[str, object], +) -> None: + judges = details.setdefault("semantic_judges", []) + if isinstance(judges, list): + judges.append(_safe_semantic_judge_payload(semantic_judge)) def _routine_confirmation_follow_up_for_text( @@ -1366,7 +1414,7 @@ async def action(page: object) -> None: } semantic_judge = getattr(exc, "semantic_judge", None) if isinstance(semantic_judge, dict): - failure_details["semantic_judge"] = semantic_judge + _append_semantic_judge(failure_details, semantic_judge) return _result( case_name, False, @@ -1589,15 +1637,18 @@ async def _wait_for_assistant_reply( ), semantic_judge=semantic_judge, ) + safe_semantic_judge = ( + _safe_semantic_judge_payload(semantic_judge) if semantic_judge else None + ) error = AssertionError( "assistant reply did not contain required text before timeout. " f"marker={marker!r} required_text={required_text!r} " f"latest_final_reply_state={last_final_reply_state!r} " f"last_assistant={last_text[-500:]!r} main_excerpt={main_text[-1000:]!r} " - f"semantic_judge={_compact_json(semantic_judge)}" + f"semantic_judge={_compact_json(safe_semantic_judge)}" ) if semantic_judge is not None: - setattr(error, "semantic_judge", semantic_judge) + setattr(error, "semantic_judge", safe_semantic_judge) raise error @@ -6891,6 +6942,8 @@ def _decimal(value: object) -> Decimal | None: parsed = Decimal(str(value)) except (InvalidOperation, TypeError, ValueError): return None + if not parsed.is_finite(): + return None return parsed if parsed >= 0 else None diff --git a/scripts/reborn_webui_v2_live_qa/semantic_judge.py b/scripts/reborn_webui_v2_live_qa/semantic_judge.py index b3ff5c18e8e..a830d20573d 100644 --- a/scripts/reborn_webui_v2_live_qa/semantic_judge.py +++ b/scripts/reborn_webui_v2_live_qa/semantic_judge.py @@ -195,9 +195,15 @@ def _completion_usage(body: object) -> dict[str, object] | None: if not isinstance(usage, dict): return None - def token_count(name: str) -> int: - value = usage.get(name) - return value if isinstance(value, int) and value >= 0 else 0 + prompt_tokens = usage.get("prompt_tokens") + completion_tokens = usage.get("completion_tokens") + if ( + not isinstance(prompt_tokens, int) + or prompt_tokens < 0 + or not isinstance(completion_tokens, int) + or completion_tokens < 0 + ): + return None prompt_details = usage.get("prompt_tokens_details") cached_tokens = 0 @@ -208,8 +214,8 @@ def token_count(name: str) -> int: return { "source": "semantic_judge", "model": str(body.get("model") or _judge_model()), - "input_tokens": token_count("prompt_tokens"), - "output_tokens": token_count("completion_tokens"), + "input_tokens": prompt_tokens, + "output_tokens": completion_tokens, "cache_read_input_tokens": cached_tokens, "cache_creation_input_tokens": 0, "pricing_source": "pending_product_rate_match", diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index 35f869be260..9c6d97349f0 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -1260,6 +1260,7 @@ def test_wait_for_assistant_reply_attaches_failed_semantic_judge_to_error(self): "completed": False, "confidence": 0.85, "reason": "The response does not complete the task.", + "response_excerpt": "do not persist this response content", "inference_usage": { "source": "semantic_judge", "model": "deepseek-ai/DeepSeek-V4-Flash", @@ -1270,12 +1271,13 @@ def test_wait_for_assistant_reply_attaches_failed_semantic_judge_to_error(self): }, } - async def fake_judge(**_kwargs): + async def fake_judge(**_kwargs: object) -> dict[str, object]: return judge_payload - async def fake_sleep(_seconds): + async def fake_sleep(_seconds: float) -> None: return None + raised_error: AssertionError | None = None with ( patch.object( run_live_qa, @@ -1284,7 +1286,7 @@ async def fake_sleep(_seconds): ), patch.object(run_live_qa.asyncio, "sleep", side_effect=fake_sleep), ): - with self.assertRaisesRegex(AssertionError, "semantic_judge=") as raised: + try: asyncio.run( run_live_qa._wait_for_assistant_reply( self._fake_assistant_reply_page(response_text), @@ -1294,8 +1296,17 @@ async def fake_sleep(_seconds): semantic_goal="Complete the requested task.", ) ) - - self.assertEqual(getattr(raised.exception, "semantic_judge"), judge_payload) + except AssertionError as exc: + raised_error = exc + else: + self.fail("expected semantic judge failure") + + self.assertIsNotNone(raised_error) + assert raised_error is not None + self.assertIn("semantic_judge=", str(raised_error)) + self.assertNotIn("response_excerpt", str(raised_error)) + self.assertEqual(raised_error.semantic_judge["completed"], False) + self.assertNotIn("response_excerpt", raised_error.semantic_judge) def test_semantic_judge_passed_respects_confidence_threshold(self): with patch.dict( @@ -1359,6 +1370,19 @@ def test_semantic_judge_completion_usage_extracts_provider_counts(self): }, ) + def test_semantic_judge_completion_usage_requires_provider_counts(self): + self.assertIsNone(semantic_judge._completion_usage({"usage": {}})) + self.assertIsNone( + semantic_judge._completion_usage( + {"usage": {"prompt_tokens": 1, "completion_tokens": -1}} + ) + ) + self.assertIsNone( + semantic_judge._completion_usage( + {"usage": {"prompt_tokens": "1", "completion_tokens": 1}} + ) + ) + def test_semantic_judge_unpriced_usage_is_explicit(self): usage = semantic_judge._unpriced_completion_usage() @@ -1367,6 +1391,15 @@ def test_semantic_judge_unpriced_usage_is_explicit(self): self.assertEqual(usage["input_tokens"], 0) self.assertEqual(usage["output_tokens"], 0) + def test_decimal_rejects_non_finite_values(self): + self.assertIsNone(run_live_qa._decimal("NaN")) + self.assertIsNone(run_live_qa._decimal("Infinity")) + self.assertIsNone(run_live_qa._decimal("-Infinity")) + self.assertEqual( + run_live_qa._decimal("0.000210"), + run_live_qa.Decimal("0.000210"), + ) + def test_server_env_enables_usage_telemetry_with_custom_log_filters(self): with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -1380,9 +1413,26 @@ def test_server_env_enables_usage_telemetry_with_custom_log_filters(self): env = run_live_qa.server_env(root / "reborn", root / "process") self.assertIn("custom_crate=debug", env["RUST_LOG"]) - self.assertIn("ironclaw_runner::model_usage=info", env["RUST_LOG"]) + self.assertIn("ironclaw_runner::model_usage=debug", env["RUST_LOG"]) self.assertIn("ironclaw_reborn=warn", env["IRONCLAW_REBORN_LOG"]) - self.assertIn("ironclaw_runner::model_usage=info", env["IRONCLAW_REBORN_LOG"]) + self.assertIn("ironclaw_runner::model_usage=debug", env["IRONCLAW_REBORN_LOG"]) + + def test_server_env_honors_extra_env_log_filters(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + env = run_live_qa.server_env( + root / "reborn", + root / "process", + extra_env={ + "RUST_LOG": "extra_rust=trace", + "IRONCLAW_REBORN_LOG": "extra_reborn=warn", + }, + ) + + self.assertIn("extra_rust=trace", env["RUST_LOG"]) + self.assertIn("extra_reborn=warn", env["IRONCLAW_REBORN_LOG"]) + self.assertIn("ironclaw_runner::model_usage=debug", env["RUST_LOG"]) + self.assertIn("ironclaw_runner::model_usage=debug", env["IRONCLAW_REBORN_LOG"]) def test_semantic_judge_json_parser_handles_non_string_inputs(self): self.assertIsNone(semantic_judge._parse_json_object(None)) @@ -5383,6 +5433,56 @@ def test_case_inference_usage_aggregates_product_and_judge_calls(self): "matched_product_rate", ) + def test_case_inference_usage_preserves_multiple_semantic_judge_calls(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + (output_dir / "ironclaw-reborn-serve.stderr.log").write_text( + "--- ironclaw-reborn serve start 2026-07-10T12:00:00Z ---\n" + "INFO REBORN_INFERENCE_USAGE operation=provider_complete " + "model=deepseek-ai/DeepSeek-V4-Flash input_tokens=10 " + "output_tokens=5 cache_read_input_tokens=0 " + "cache_creation_input_tokens=0 input_usd_per_token=0.000001 " + "output_usd_per_token=0.000002 estimated_usd=0.000020\n", + encoding="utf-8", + ) + result = run_live_qa.ProbeResult( + provider="test", + mode="live:test", + success=True, + latency_ms=1, + details={ + "semantic_judges": [ + { + "inference_usage": { + "source": "semantic_judge", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "input_tokens": 20, + "output_tokens": 2, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + } + }, + { + "inference_usage": { + "source": "semantic_judge", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "input_tokens": 30, + "output_tokens": 3, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + } + }, + ] + }, + ) + + summary = run_live_qa._case_inference_usage(output_dir, result) + + self.assertEqual(summary["call_count"], 3) + self.assertEqual(summary["input_tokens"], 60) + self.assertEqual(summary["output_tokens"], 10) + self.assertEqual(summary["unpriced_call_count"], 0) + def test_run_cases_isolates_reborn_home_and_preflight_per_selected_case(self): async def fake_case(ctx: run_live_qa.LiveQaContext) -> run_live_qa.ProbeResult: return run_live_qa.ProbeResult( From 0630084d72a386b61bd5e1bd174edb8e49eb7bbb Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Fri, 10 Jul 2026 16:38:34 +0300 Subject: [PATCH 3/6] fix(canary): add runner decimal dependency --- crates/ironclaw_runner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ironclaw_runner/Cargo.toml b/crates/ironclaw_runner/Cargo.toml index c8fe7276f28..a51422579f8 100644 --- a/crates/ironclaw_runner/Cargo.toml +++ b/crates/ironclaw_runner/Cargo.toml @@ -76,6 +76,7 @@ parking_lot = "0.12" secrecy = { version = "0.10", optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" +rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] } thiserror = "2" tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } tokio-util = { version = "0.7", features = ["rt"] } @@ -94,7 +95,6 @@ ironclaw_resources = { path = "../ironclaw_resources", version = "0.1.0" } ironclaw_scripts = { path = "../ironclaw_scripts", version = "0.1.0" } ironclaw_skills = { path = "../ironclaw_skills", version = "0.3.0", default-features = false } ironclaw_trust = { path = "../ironclaw_trust", version = "0.1.0" } -rust_decimal = "1" serde = "1" serde_json = "1" tempfile = "3" From f8280ad90c9d0b225a92e05b97615da80e01bb4a Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Fri, 10 Jul 2026 17:48:01 +0300 Subject: [PATCH 4/6] fix(canary): price DeepSeek V4 Flash accurately --- crates/ironclaw_llm/src/costs.rs | 15 ++++ crates/ironclaw_llm/src/nearai_chat.rs | 6 +- crates/ironclaw_runner/src/model_gateway.rs | 83 +++++++++++++++++-- .../reborn_webui_v2_live_qa/run_live_qa.py | 61 +++++++++----- .../test_run_live_qa.py | 52 ++++++++++++ 5 files changed, 184 insertions(+), 33 deletions(-) diff --git a/crates/ironclaw_llm/src/costs.rs b/crates/ironclaw_llm/src/costs.rs index f501c1eae6d..0bc334b9903 100644 --- a/crates/ironclaw_llm/src/costs.rs +++ b/crates/ironclaw_llm/src/costs.rs @@ -74,6 +74,9 @@ pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> { | "claude-3-5-haiku-latest" => Some((dec!(0.0000008), dec!(0.000004))), "claude-3-haiku-20240307" => Some((dec!(0.00000025), dec!(0.00000125))), + // DeepSeek + "DeepSeek-V4-Flash" | "deepseek-v4-flash" => Some((dec!(0.00000014), dec!(0.00000028))), + // Ollama / local models -- free _ if is_local_model(id) => Some((Decimal::ZERO, Decimal::ZERO)), @@ -164,6 +167,18 @@ mod tests { assert_eq!(model_cost("openai/gpt-4o"), model_cost("gpt-4o")); } + #[test] + fn test_deepseek_v4_flash_uses_remote_pricing_before_local_heuristic() { + assert_eq!( + model_cost("deepseek-ai/DeepSeek-V4-Flash"), + Some((dec!(0.00000014), dec!(0.00000028))) + ); + assert_eq!( + model_cost("deepseek-ai/deepseek-v4-flash"), + Some((dec!(0.00000014), dec!(0.00000028))) + ); + } + #[test] fn test_openrouter_free_suffix_zero_cost() { // Models with `:free` suffix should report zero cost (#463) diff --git a/crates/ironclaw_llm/src/nearai_chat.rs b/crates/ironclaw_llm/src/nearai_chat.rs index c944465ae45..2ea6775def1 100644 --- a/crates/ironclaw_llm/src/nearai_chat.rs +++ b/crates/ironclaw_llm/src/nearai_chat.rs @@ -2928,14 +2928,14 @@ data: [DONE] let provider = NearAiChatProvider::new(cfg, test_session()).expect("provider"); let (input, output) = provider.cost_per_token(); - let (default_in, default_out) = costs::default_cost(); - assert_eq!(input, default_in); - assert_eq!(output, default_out); + assert_eq!(input, dec!(0.00000014)); + assert_eq!(output, dec!(0.00000028)); provider .set_model("Qwen/Qwen3-32B") .expect("set active model"); let (input, output) = provider.cost_per_token(); + let (default_in, default_out) = costs::default_cost(); assert_eq!(input, default_in); assert_eq!(output, default_out); } diff --git a/crates/ironclaw_runner/src/model_gateway.rs b/crates/ironclaw_runner/src/model_gateway.rs index 8c66654720b..8b6f65951a9 100644 --- a/crates/ironclaw_runner/src/model_gateway.rs +++ b/crates/ironclaw_runner/src/model_gateway.rs @@ -75,6 +75,7 @@ const UNAVAILABLE_CAPABILITY_REPLY: &str = "That capability is unavailable or di struct ModelUsageContext { model: String, input_usd_per_token: Decimal, + cache_read_input_usd_per_token: Option, output_usd_per_token: Decimal, } @@ -99,6 +100,19 @@ fn is_explicit_free_model(model: &str) -> bool { model.ends_with(":free") || model == "openrouter/free" || model == "free" } +fn cache_read_input_rate_for_model(model: &str) -> Option { + let id = model.rsplit('/').next().unwrap_or(model); + if id.eq_ignore_ascii_case("deepseek-v4-flash") { + Some(Decimal::new(28, 10)) + } else { + None + } +} + +fn locked_usage_rates_for_model(model: &str) -> Option<(Decimal, Decimal)> { + cache_read_input_rate_for_model(model).and_then(|_| model_cost(model)) +} + fn model_usage_context( provider: &P, model_override: Option<&str>, @@ -107,18 +121,39 @@ fn model_usage_context( let effective_model = normalized_request_model_override(model_override) .map(str::to_string) .unwrap_or_else(|| active_model.clone()); - let (input_usd_per_token, output_usd_per_token) = if effective_model == active_model { - provider.cost_per_token() - } else { - fallback_usage_rates_for_model(&effective_model) - }; + let (input_usd_per_token, output_usd_per_token) = + if let Some(rates) = locked_usage_rates_for_model(&effective_model) { + rates + } else if effective_model == active_model { + provider.cost_per_token() + } else { + fallback_usage_rates_for_model(&effective_model) + }; + let cache_read_input_usd_per_token = cache_read_input_rate_for_model(&effective_model); ModelUsageContext { model: effective_model, input_usd_per_token, + cache_read_input_usd_per_token, output_usd_per_token, } } +fn estimate_model_usage_usd( + usage: &ModelUsageContext, + input_tokens: u32, + output_tokens: u32, + cache_read_input_tokens: u32, +) -> Decimal { + let cached_input_tokens = cache_read_input_tokens.min(input_tokens); + let uncached_input_tokens = input_tokens.saturating_sub(cached_input_tokens); + let cache_read_input_rate = usage + .cache_read_input_usd_per_token + .unwrap_or(usage.input_usd_per_token); + usage.input_usd_per_token * Decimal::from(uncached_input_tokens) + + cache_read_input_rate * Decimal::from(cached_input_tokens) + + usage.output_usd_per_token * Decimal::from(output_tokens) +} + fn trace_model_usage( usage: &ModelUsageContext, operation: &'static str, @@ -130,11 +165,14 @@ fn trace_model_usage( if !tracing::enabled!(target: MODEL_USAGE_TARGET, tracing::Level::DEBUG) { return; } - let estimated_usd = usage.input_usd_per_token * Decimal::from(input_tokens) - + usage.output_usd_per_token * Decimal::from(output_tokens); + let cache_read_input_usd_per_token = usage + .cache_read_input_usd_per_token + .unwrap_or(usage.input_usd_per_token); + let estimated_usd = + estimate_model_usage_usd(usage, input_tokens, output_tokens, cache_read_input_tokens); debug!( target: MODEL_USAGE_TARGET, - "REBORN_INFERENCE_USAGE operation={} model={} usage_available=true input_tokens={} output_tokens={} cache_read_input_tokens={} cache_creation_input_tokens={} input_usd_per_token={} output_usd_per_token={} estimated_usd={}", + "REBORN_INFERENCE_USAGE operation={} model={} usage_available=true input_tokens={} output_tokens={} cache_read_input_tokens={} cache_creation_input_tokens={} input_usd_per_token={} cache_read_input_usd_per_token={} output_usd_per_token={} estimated_usd={}", operation, usage.model, input_tokens, @@ -142,6 +180,7 @@ fn trace_model_usage( cache_read_input_tokens, cache_creation_input_tokens, usage.input_usd_per_token, + cache_read_input_usd_per_token, usage.output_usd_per_token, estimated_usd, ); @@ -2549,6 +2588,34 @@ mod tests { } } + #[test] + fn estimate_model_usage_charges_cache_reads_at_cache_rate() { + let usage = ModelUsageContext { + model: "deepseek-ai/DeepSeek-V4-Flash".to_string(), + input_usd_per_token: Decimal::new(14, 8), + cache_read_input_usd_per_token: Some(Decimal::new(28, 10)), + output_usd_per_token: Decimal::new(28, 8), + }; + + let cost = estimate_model_usage_usd(&usage, 100, 20, 40); + + assert_eq!(cost, Decimal::new(14112, 9)); + } + + #[test] + fn estimate_model_usage_clamps_cache_reads_to_input_tokens() { + let usage = ModelUsageContext { + model: "deepseek-ai/DeepSeek-V4-Flash".to_string(), + input_usd_per_token: Decimal::new(14, 8), + cache_read_input_usd_per_token: Some(Decimal::new(28, 10)), + output_usd_per_token: Decimal::new(28, 8), + }; + + let cost = estimate_model_usage_usd(&usage, 10, 0, 40); + + assert_eq!(cost, Decimal::new(28, 9)); + } + #[test] fn guard_ignores_incidental_code_references() { // The playwright/browser tasks literally instruct: "use `playwright.sync_api`" diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index acd8e316c3e..a444d7be5cb 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -6979,29 +6979,31 @@ def _product_inference_usage(output_dir: Path) -> list[dict[str, object]]: ) continue input_rate = _decimal(fields.get("input_usd_per_token")) + cache_read_input_rate = _decimal(fields.get("cache_read_input_usd_per_token")) output_rate = _decimal(fields.get("output_usd_per_token")) estimated_usd = _decimal(fields.get("estimated_usd")) if input_rate is None or output_rate is None or estimated_usd is None: continue - events.append( - { - "source": "product", - "operation": str(fields.get("operation") or "unknown"), - "model": str(fields.get("model") or "unknown"), - "input_tokens": _non_negative_int(fields.get("input_tokens")), - "output_tokens": _non_negative_int(fields.get("output_tokens")), - "cache_read_input_tokens": _non_negative_int( - fields.get("cache_read_input_tokens") - ), - "cache_creation_input_tokens": _non_negative_int( - fields.get("cache_creation_input_tokens") - ), - "input_usd_per_token": str(input_rate), - "output_usd_per_token": str(output_rate), - "estimated_usd": str(estimated_usd), - "pricing_source": "provider_active_rate", - } - ) + event = { + "source": "product", + "operation": str(fields.get("operation") or "unknown"), + "model": str(fields.get("model") or "unknown"), + "input_tokens": _non_negative_int(fields.get("input_tokens")), + "output_tokens": _non_negative_int(fields.get("output_tokens")), + "cache_read_input_tokens": _non_negative_int( + fields.get("cache_read_input_tokens") + ), + "cache_creation_input_tokens": _non_negative_int( + fields.get("cache_creation_input_tokens") + ), + "input_usd_per_token": str(input_rate), + "output_usd_per_token": str(output_rate), + "estimated_usd": str(estimated_usd), + "pricing_source": "provider_active_rate", + } + if cache_read_input_rate is not None: + event["cache_read_input_usd_per_token"] = str(cache_read_input_rate) + events.append(event) return events @@ -7020,12 +7022,17 @@ def _semantic_judge_usage(value: object) -> list[dict[str, object]]: def _summarize_inference_usage(events: list[dict[str, object]]) -> dict[str, object]: - rates_by_model: dict[str, tuple[Decimal, Decimal]] = {} + rates_by_model: dict[str, tuple[Decimal, Decimal, Decimal | None]] = {} for event in events: input_rate = _decimal(event.get("input_usd_per_token")) + cache_read_input_rate = _decimal(event.get("cache_read_input_usd_per_token")) output_rate = _decimal(event.get("output_usd_per_token")) if event.get("source") == "product" and input_rate is not None and output_rate is not None: - rates_by_model[str(event.get("model") or "unknown")] = (input_rate, output_rate) + rates_by_model[str(event.get("model") or "unknown")] = ( + input_rate, + output_rate, + cache_read_input_rate, + ) priced_events: list[dict[str, object]] = [] unpriced_calls = 0 @@ -7040,11 +7047,21 @@ def _summarize_inference_usage(events: list[dict[str, object]]) -> dict[str, obj ): rates = rates_by_model.get(str(event.get("model") or "unknown")) if rates is not None: + input_tokens = _non_negative_int(event.get("input_tokens")) + cache_read_input_tokens = min( + _non_negative_int(event.get("cache_read_input_tokens")), + input_tokens, + ) + uncached_input_tokens = max(input_tokens - cache_read_input_tokens, 0) + cache_read_input_rate = rates[2] or rates[0] cost = ( - Decimal(_non_negative_int(event.get("input_tokens"))) * rates[0] + Decimal(uncached_input_tokens) * rates[0] + + Decimal(cache_read_input_tokens) * cache_read_input_rate + Decimal(_non_negative_int(event.get("output_tokens"))) * rates[1] ) event["input_usd_per_token"] = str(rates[0]) + if rates[2] is not None: + event["cache_read_input_usd_per_token"] = str(rates[2]) event["output_usd_per_token"] = str(rates[1]) event["estimated_usd"] = str(cost) event["pricing_source"] = "matched_product_rate" diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index 9c6d97349f0..0d22888def1 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -5433,6 +5433,58 @@ def test_case_inference_usage_aggregates_product_and_judge_calls(self): "matched_product_rate", ) + def test_case_inference_usage_charges_cache_reads_at_cache_rate(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + (output_dir / "ironclaw-reborn-serve.stderr.log").write_text( + "--- ironclaw-reborn serve start 2026-07-10T12:00:00Z ---\n" + "INFO REBORN_INFERENCE_USAGE operation=provider_complete_with_tools " + "model=deepseek-ai/DeepSeek-V4-Flash input_tokens=100 " + "output_tokens=20 cache_read_input_tokens=40 " + "cache_creation_input_tokens=0 input_usd_per_token=0.00000014 " + "cache_read_input_usd_per_token=0.0000000028 " + "output_usd_per_token=0.00000028 estimated_usd=0.000014112\n", + encoding="utf-8", + ) + result = run_live_qa.ProbeResult( + provider="test", + mode="live:test", + success=True, + latency_ms=1, + details={ + "semantic_judge": { + "inference_usage": { + "source": "semantic_judge", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "input_tokens": 50, + "output_tokens": 10, + "cache_read_input_tokens": 20, + "cache_creation_input_tokens": 0, + } + } + }, + ) + + summary = run_live_qa._case_inference_usage(output_dir, result) + + self.assertEqual(summary["call_count"], 2) + self.assertEqual(summary["input_tokens"], 150) + self.assertEqual(summary["cache_read_input_tokens"], 60) + self.assertEqual(summary["output_tokens"], 30) + self.assertEqual(summary["estimated_usd"], "0.0000211680") + self.assertEqual( + summary["events"][0]["cache_read_input_usd_per_token"], + "2.8E-9", + ) + self.assertEqual( + summary["events"][1]["cache_read_input_usd_per_token"], + "2.8E-9", + ) + self.assertEqual( + summary["events"][1]["pricing_source"], + "matched_product_rate", + ) + def test_case_inference_usage_preserves_multiple_semantic_judge_calls(self): with tempfile.TemporaryDirectory() as tmpdir: output_dir = Path(tmpdir) From 72be995e10fba5013632ecb0a63d2177c80d2110 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Fri, 10 Jul 2026 19:41:59 +0300 Subject: [PATCH 5/6] fix(canary): address coderabbit pricing comments (#5931) --- crates/ironclaw_llm/src/costs.rs | 188 +++++++++++++++--- crates/ironclaw_llm/src/nearai_chat.rs | 14 +- crates/ironclaw_runner/src/model_gateway.rs | 61 +++--- .../reborn_webui_v2_live_qa/run_live_qa.py | 44 ++-- .../test_run_live_qa.py | 37 ++++ 5 files changed, 248 insertions(+), 96 deletions(-) diff --git a/crates/ironclaw_llm/src/costs.rs b/crates/ironclaw_llm/src/costs.rs index 0bc334b9903..f8675549dda 100644 --- a/crates/ironclaw_llm/src/costs.rs +++ b/crates/ironclaw_llm/src/costs.rs @@ -6,14 +6,59 @@ use rust_decimal::Decimal; use rust_decimal_macros::dec; +/// Known per-token usage rates for a model. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ModelUsageRates { + pub input_per_token: Decimal, + pub output_per_token: Decimal, + pub cache_read_input_per_token: Option, +} + +impl ModelUsageRates { + const fn new(input_per_token: Decimal, output_per_token: Decimal) -> Self { + Self { + input_per_token, + output_per_token, + cache_read_input_per_token: None, + } + } + + const fn with_cache_read( + input_per_token: Decimal, + output_per_token: Decimal, + cache_read_input_per_token: Decimal, + ) -> Self { + Self { + input_per_token, + output_per_token, + cache_read_input_per_token: Some(cache_read_input_per_token), + } + } + + const fn zero() -> Self { + Self { + input_per_token: Decimal::ZERO, + output_per_token: Decimal::ZERO, + cache_read_input_per_token: None, + } + } +} + /// Look up known per-token costs for a model by its identifier. /// /// Returns `Some((input_cost, output_cost))` for known models, `None` otherwise. pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> { + model_usage_rates(model_id).map(|rates| (rates.input_per_token, rates.output_per_token)) +} + +/// Look up known per-token usage rates for a model by its identifier. +/// +/// Returns `Some(rates)` for known models, `None` otherwise. +pub fn model_usage_rates(model_id: &str) -> Option { // OpenRouter free-tier models: `:free` suffix or the `openrouter/free` router // should always report zero cost (see #463). - if model_id.ends_with(":free") || model_id == "openrouter/free" || model_id == "free" { - return Some((Decimal::ZERO, Decimal::ZERO)); + if is_explicit_free_model(model_id) { + return Some(ModelUsageRates::zero()); } // Normalize: strip provider prefixes (e.g., "openai/gpt-4o" -> "gpt-4o") @@ -21,33 +66,52 @@ pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> { .rsplit_once('/') .map(|(_, name)| name) .unwrap_or(model_id); + let id = id.to_ascii_lowercase(); - match id { + match id.as_str() { // OpenAI — GPT-5.x / Codex - "gpt-5.5" | "gpt-5.5-codex" => Some((dec!(0.000002), dec!(0.000008))), - "gpt-5.3-codex" | "gpt-5.3-codex-spark" => Some((dec!(0.000002), dec!(0.000008))), - "gpt-5.2-codex" | "gpt-5.2-pro" | "gpt-5.2" => Some((dec!(0.000002), dec!(0.000008))), - "gpt-5.1-codex" | "gpt-5.1-codex-max" | "gpt-5.1" => Some((dec!(0.000002), dec!(0.000008))), - "gpt-5.1-codex-mini" => Some((dec!(0.0000003), dec!(0.0000012))), - "gpt-5-codex" | "gpt-5-pro" | "gpt-5" => Some((dec!(0.000002), dec!(0.000008))), - "gpt-5-mini" | "gpt-5-nano" => Some((dec!(0.0000003), dec!(0.0000012))), + "gpt-5.5" | "gpt-5.5-codex" => Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))), + "gpt-5.3-codex" | "gpt-5.3-codex-spark" => { + Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))) + } + "gpt-5.2-codex" | "gpt-5.2-pro" | "gpt-5.2" => { + Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))) + } + "gpt-5.1-codex" | "gpt-5.1-codex-max" | "gpt-5.1" => { + Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))) + } + "gpt-5.1-codex-mini" => Some(ModelUsageRates::new(dec!(0.0000003), dec!(0.0000012))), + "gpt-5-codex" | "gpt-5-pro" | "gpt-5" => { + Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))) + } + "gpt-5-mini" | "gpt-5-nano" => Some(ModelUsageRates::new(dec!(0.0000003), dec!(0.0000012))), // OpenAI — GPT-4.x - "gpt-4.1" => Some((dec!(0.000002), dec!(0.000008))), - "gpt-4.1-mini" => Some((dec!(0.0000004), dec!(0.0000016))), - "gpt-4.1-nano" => Some((dec!(0.0000001), dec!(0.0000004))), + "gpt-4.1" => Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))), + "gpt-4.1-mini" => Some(ModelUsageRates::new(dec!(0.0000004), dec!(0.0000016))), + "gpt-4.1-nano" => Some(ModelUsageRates::new(dec!(0.0000001), dec!(0.0000004))), "gpt-4o" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" => { - Some((dec!(0.0000025), dec!(0.00001))) + Some(ModelUsageRates::new(dec!(0.0000025), dec!(0.00001))) + } + "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" => { + Some(ModelUsageRates::new(dec!(0.00000015), dec!(0.0000006))) + } + "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" => { + Some(ModelUsageRates::new(dec!(0.00001), dec!(0.00003))) + } + "gpt-4" | "gpt-4-0613" => Some(ModelUsageRates::new(dec!(0.00003), dec!(0.00006))), + "gpt-3.5-turbo" | "gpt-3.5-turbo-0125" => { + Some(ModelUsageRates::new(dec!(0.0000005), dec!(0.0000015))) } - "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" => Some((dec!(0.00000015), dec!(0.0000006))), - "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" => Some((dec!(0.00001), dec!(0.00003))), - "gpt-4" | "gpt-4-0613" => Some((dec!(0.00003), dec!(0.00006))), - "gpt-3.5-turbo" | "gpt-3.5-turbo-0125" => Some((dec!(0.0000005), dec!(0.0000015))), // OpenAI — reasoning - "o3" => Some((dec!(0.000002), dec!(0.000008))), - "o3-mini" | "o3-mini-2025-01-31" => Some((dec!(0.0000011), dec!(0.0000044))), - "o4-mini" => Some((dec!(0.0000011), dec!(0.0000044))), - "o1" | "o1-2024-12-17" => Some((dec!(0.000015), dec!(0.00006))), - "o1-mini" | "o1-mini-2024-09-12" => Some((dec!(0.000003), dec!(0.000012))), + "o3" => Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))), + "o3-mini" | "o3-mini-2025-01-31" => { + Some(ModelUsageRates::new(dec!(0.0000011), dec!(0.0000044))) + } + "o4-mini" => Some(ModelUsageRates::new(dec!(0.0000011), dec!(0.0000044))), + "o1" | "o1-2024-12-17" => Some(ModelUsageRates::new(dec!(0.000015), dec!(0.00006))), + "o1-mini" | "o1-mini-2024-09-12" => { + Some(ModelUsageRates::new(dec!(0.000003), dec!(0.000012))) + } // Anthropic "claude-opus-4-6" @@ -58,7 +122,7 @@ pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> { | "claude-opus-4-0" | "claude-opus-4-20250514" | "claude-3-opus-20240229" - | "claude-3-opus-latest" => Some((dec!(0.000015), dec!(0.000075))), + | "claude-3-opus-latest" => Some(ModelUsageRates::new(dec!(0.000015), dec!(0.000075))), "claude-sonnet-4-6" | "claude-sonnet-4-5" | "claude-sonnet-4-5-20250929" @@ -67,27 +131,31 @@ pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> { | "claude-3-7-sonnet-20250219" | "claude-3-7-sonnet-latest" | "claude-3-5-sonnet-20241022" - | "claude-3-5-sonnet-latest" => Some((dec!(0.000003), dec!(0.000015))), + | "claude-3-5-sonnet-latest" => Some(ModelUsageRates::new(dec!(0.000003), dec!(0.000015))), "claude-haiku-4-5" | "claude-haiku-4-5-20251001" | "claude-3-5-haiku-20241022" - | "claude-3-5-haiku-latest" => Some((dec!(0.0000008), dec!(0.000004))), - "claude-3-haiku-20240307" => Some((dec!(0.00000025), dec!(0.00000125))), + | "claude-3-5-haiku-latest" => Some(ModelUsageRates::new(dec!(0.0000008), dec!(0.000004))), + "claude-3-haiku-20240307" => Some(ModelUsageRates::new(dec!(0.00000025), dec!(0.00000125))), // DeepSeek - "DeepSeek-V4-Flash" | "deepseek-v4-flash" => Some((dec!(0.00000014), dec!(0.00000028))), + "deepseek-v4-flash" => Some(ModelUsageRates::with_cache_read( + dec!(0.00000014), + dec!(0.00000028), + dec!(0.0000000028), + )), // Ollama / local models -- free - _ if is_local_model(id) => Some((Decimal::ZERO, Decimal::ZERO)), + _ if is_local_model(&id) => Some(ModelUsageRates::zero()), // Family fallbacks: a new GPT-5.x minor release shouldn't need a table // edit just to be budgeted. Exact arms above win for known per-model // pricing; these only catch unrecognized `gpt-5*` slugs. `*-mini` / // `*-nano` bill at the small tier, everything else at the standard tier. _ if id.starts_with("gpt-5") && (id.ends_with("-mini") || id.ends_with("-nano")) => { - Some((dec!(0.0000003), dec!(0.0000012))) + Some(ModelUsageRates::new(dec!(0.0000003), dec!(0.0000012))) } - _ if id.starts_with("gpt-5") => Some((dec!(0.000002), dec!(0.000008))), + _ if id.starts_with("gpt-5") => Some(ModelUsageRates::new(dec!(0.000002), dec!(0.000008))), _ => None, } @@ -99,6 +167,38 @@ pub fn default_cost() -> (Decimal, Decimal) { (dec!(0.0000025), dec!(0.00001)) } +/// Returns true when the model identifier explicitly represents a free route. +pub fn is_explicit_free_model(model_id: &str) -> bool { + model_id.ends_with(":free") || model_id == "openrouter/free" || model_id == "free" +} + +/// Shared fallback rates for remote model usage accounting. +/// +/// The static table reports local/self-hosted model families as free. Remote +/// providers can expose similarly named models, so callers that are accounting +/// for a remote provider should use this fallback to avoid silently reporting +/// unknown remote usage as zero-cost unless the model is explicitly free. +pub fn remote_model_fallback_usage_rates(model_id: &str) -> ModelUsageRates { + let rates = model_usage_rates(model_id).unwrap_or_else(|| { + let (input_per_token, output_per_token) = default_cost(); + ModelUsageRates::new(input_per_token, output_per_token) + }); + if rates.input_per_token == Decimal::ZERO + && rates.output_per_token == Decimal::ZERO + && !is_explicit_free_model(model_id) + { + let (input_per_token, output_per_token) = default_cost(); + return ModelUsageRates::new(input_per_token, output_per_token); + } + rates +} + +/// Shared fallback costs for remote model usage accounting. +pub fn remote_model_fallback_cost(model_id: &str) -> (Decimal, Decimal) { + let rates = remote_model_fallback_usage_rates(model_id); + (rates.input_per_token, rates.output_per_token) +} + /// Heuristic to detect local/self-hosted models (Ollama, llama.cpp, etc.). fn is_local_model(model_id: &str) -> bool { let lower = model_id.to_lowercase(); @@ -177,6 +277,32 @@ mod tests { model_cost("deepseek-ai/deepseek-v4-flash"), Some((dec!(0.00000014), dec!(0.00000028))) ); + let mixed_case = model_cost("deepseek-ai/Deepseek-v4-Flash"); + assert_eq!(mixed_case, Some((dec!(0.00000014), dec!(0.00000028)))); + assert_ne!(mixed_case, Some((Decimal::ZERO, Decimal::ZERO))); + assert_eq!( + model_usage_rates("deepseek-ai/Deepseek-v4-Flash") + .unwrap() + .cache_read_input_per_token, + Some(dec!(0.0000000028)) + ); + } + + #[test] + fn test_remote_model_fallback_rates_do_not_treat_remote_local_slugs_as_free() { + let rates = remote_model_fallback_usage_rates("Qwen/Qwen3-32B"); + let (default_input, default_output) = default_cost(); + assert_eq!(rates.input_per_token, default_input); + assert_eq!(rates.output_per_token, default_output); + assert_eq!(rates.cache_read_input_per_token, None); + } + + #[test] + fn test_remote_model_fallback_rates_preserve_explicit_free_models() { + let rates = remote_model_fallback_usage_rates("stepfun/step-3.5-flash:free"); + assert_eq!(rates.input_per_token, Decimal::ZERO); + assert_eq!(rates.output_per_token, Decimal::ZERO); + assert_eq!(rates.cache_read_input_per_token, None); } #[test] diff --git a/crates/ironclaw_llm/src/nearai_chat.rs b/crates/ironclaw_llm/src/nearai_chat.rs index 2ea6775def1..c1f8c6b69b3 100644 --- a/crates/ironclaw_llm/src/nearai_chat.rs +++ b/crates/ironclaw_llm/src/nearai_chat.rs @@ -1071,7 +1071,7 @@ impl LlmProvider for NearAiChatProvider { { return rates; } - remote_model_fallback_cost(&model) + costs::remote_model_fallback_cost(&model) } async fn list_models(&self) -> Result, LlmError> { @@ -1204,18 +1204,6 @@ struct ChatCompletionMessage { tool_calls: Option>, } -fn remote_model_fallback_cost(model: &str) -> (Decimal, Decimal) { - let rates = costs::model_cost(model).unwrap_or_else(costs::default_cost); - if rates == (Decimal::ZERO, Decimal::ZERO) && !is_explicit_free_model(model) { - return costs::default_cost(); - } - rates -} - -fn is_explicit_free_model(model: &str) -> bool { - model.ends_with(":free") || model == "openrouter/free" || model == "free" -} - // -- Pricing fetch types and logic ----------------------------------------- /// Cost amount from the NEAR AI `/v1/model/list` response. diff --git a/crates/ironclaw_runner/src/model_gateway.rs b/crates/ironclaw_runner/src/model_gateway.rs index 8b6f65951a9..763242047e4 100644 --- a/crates/ironclaw_runner/src/model_gateway.rs +++ b/crates/ironclaw_runner/src/model_gateway.rs @@ -19,7 +19,10 @@ use ironclaw_llm::{ ChatMessage, CompletionRequest, CompletionResponse, CompletionStreamSink, ContentPart, FinishReason, ImageUrl, LlmError, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, clean_response, contains_codex_text_tool_call_syntax, - costs::{default_cost, model_cost}, + costs::{ + ModelUsageRates, default_cost, model_usage_rates, remote_model_fallback_cost, + remote_model_fallback_usage_rates, + }, recover_codex_text_tool_calls_from_tool_names, vision_models::is_vision_model, }; @@ -88,31 +91,6 @@ fn normalized_request_model_override(model: Option<&str>) -> Option<&str> { } } -fn fallback_usage_rates_for_model(model: &str) -> (Decimal, Decimal) { - let rates = model_cost(model).unwrap_or_else(default_cost); - if rates == (Decimal::ZERO, Decimal::ZERO) && !is_explicit_free_model(model) { - return default_cost(); - } - rates -} - -fn is_explicit_free_model(model: &str) -> bool { - model.ends_with(":free") || model == "openrouter/free" || model == "free" -} - -fn cache_read_input_rate_for_model(model: &str) -> Option { - let id = model.rsplit('/').next().unwrap_or(model); - if id.eq_ignore_ascii_case("deepseek-v4-flash") { - Some(Decimal::new(28, 10)) - } else { - None - } -} - -fn locked_usage_rates_for_model(model: &str) -> Option<(Decimal, Decimal)> { - cache_read_input_rate_for_model(model).and_then(|_| model_cost(model)) -} - fn model_usage_context( provider: &P, model_override: Option<&str>, @@ -121,20 +99,27 @@ fn model_usage_context( let effective_model = normalized_request_model_override(model_override) .map(str::to_string) .unwrap_or_else(|| active_model.clone()); - let (input_usd_per_token, output_usd_per_token) = - if let Some(rates) = locked_usage_rates_for_model(&effective_model) { + let static_rates = model_usage_rates(&effective_model); + let rates = if effective_model == active_model { + if let Some(rates) = static_rates.filter(|rates| rates.cache_read_input_per_token.is_some()) + { rates - } else if effective_model == active_model { - provider.cost_per_token() } else { - fallback_usage_rates_for_model(&effective_model) - }; - let cache_read_input_usd_per_token = cache_read_input_rate_for_model(&effective_model); + let (input_per_token, output_per_token) = provider.cost_per_token(); + ModelUsageRates { + input_per_token, + output_per_token, + cache_read_input_per_token: None, + } + } + } else { + remote_model_fallback_usage_rates(&effective_model) + }; ModelUsageContext { model: effective_model, - input_usd_per_token, - cache_read_input_usd_per_token, - output_usd_per_token, + input_usd_per_token: rates.input_per_token, + cache_read_input_usd_per_token: rates.cache_read_input_per_token, + output_usd_per_token: rates.output_per_token, } } @@ -258,7 +243,7 @@ impl LlmModelProfilePolicy { } /// Build a [`StaticModelCostTable`] mapping every allowed `ModelProfileId` - /// to its per-token price via [`ironclaw_llm::costs::model_cost`]. + /// to its per-token price via [`ironclaw_llm::costs::remote_model_fallback_cost`]. /// Profiles whose `model_override` is unknown to the LLM cost table /// fall back to [`ironclaw_llm::costs::default_cost`] (roughly GPT-4o /// pricing) so the accountant always reconciles to a non-zero spend @@ -269,7 +254,7 @@ impl LlmModelProfilePolicy { let cost = route .model_override .as_deref() - .and_then(model_cost) + .map(remote_model_fallback_cost) .unwrap_or_else(default_cost); table.insert( profile_id.clone(), diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index a444d7be5cb..a1e9f274aa6 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -1241,9 +1241,6 @@ def _safe_semantic_judge_payload(value: dict[str, object]) -> dict[str, object]: safe["completed"] = value["completed"] if isinstance(value.get("confidence"), (int, float)): safe["confidence"] = value["confidence"] - reason = value.get("reason") - if isinstance(reason, str): - safe["reason"] = reason[:300] usage = _safe_semantic_judge_usage(value.get("inference_usage")) if usage is not None: safe["inference_usage"] = usage @@ -6947,6 +6944,25 @@ def _decimal(value: object) -> Decimal | None: return parsed if parsed >= 0 else None +def _unpriced_product_inference_event( + fields: dict[str, str], + *, + pricing_source: str, +) -> dict[str, object]: + return { + "source": "product", + "operation": str(fields.get("operation") or "unknown"), + "model": str(fields.get("model") or "unknown"), + "input_tokens": _non_negative_int(fields.get("input_tokens")), + "output_tokens": _non_negative_int(fields.get("output_tokens")), + "cache_read_input_tokens": _non_negative_int(fields.get("cache_read_input_tokens")), + "cache_creation_input_tokens": _non_negative_int( + fields.get("cache_creation_input_tokens") + ), + "pricing_source": pricing_source, + } + + def _product_inference_usage(output_dir: Path) -> list[dict[str, object]]: events: list[dict[str, object]] = [] for name in ("ironclaw-reborn-serve.stdout.log", "ironclaw-reborn-serve.stderr.log"): @@ -6966,16 +6982,10 @@ def _product_inference_usage(output_dir: Path) -> list[dict[str, object]]: ) if fields.get("usage_available") == "false": events.append( - { - "source": "product", - "operation": str(fields.get("operation") or "unknown"), - "model": str(fields.get("model") or "unknown"), - "input_tokens": 0, - "output_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - "pricing_source": "provider_usage_unavailable", - } + _unpriced_product_inference_event( + fields, + pricing_source="provider_usage_unavailable", + ) ) continue input_rate = _decimal(fields.get("input_usd_per_token")) @@ -6983,6 +6993,12 @@ def _product_inference_usage(output_dir: Path) -> list[dict[str, object]]: output_rate = _decimal(fields.get("output_usd_per_token")) estimated_usd = _decimal(fields.get("estimated_usd")) if input_rate is None or output_rate is None or estimated_usd is None: + events.append( + _unpriced_product_inference_event( + fields, + pricing_source="provider_usage_malformed", + ) + ) continue event = { "source": "product", @@ -7053,7 +7069,7 @@ def _summarize_inference_usage(events: list[dict[str, object]]) -> dict[str, obj input_tokens, ) uncached_input_tokens = max(input_tokens - cache_read_input_tokens, 0) - cache_read_input_rate = rates[2] or rates[0] + cache_read_input_rate = rates[2] if rates[2] is not None else rates[0] cost = ( Decimal(uncached_input_tokens) * rates[0] + Decimal(cache_read_input_tokens) * cache_read_input_rate diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index 0d22888def1..dc1aa381768 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -1305,7 +1305,9 @@ async def fake_sleep(_seconds: float) -> None: assert raised_error is not None self.assertIn("semantic_judge=", str(raised_error)) self.assertNotIn("response_excerpt", str(raised_error)) + self.assertNotIn("The response does not complete the task.", str(raised_error)) self.assertEqual(raised_error.semantic_judge["completed"], False) + self.assertNotIn("reason", raised_error.semantic_judge) self.assertNotIn("response_excerpt", raised_error.semantic_judge) def test_semantic_judge_passed_respects_confidence_threshold(self): @@ -5433,6 +5435,41 @@ def test_case_inference_usage_aggregates_product_and_judge_calls(self): "matched_product_rate", ) + def test_case_inference_usage_counts_malformed_priced_product_calls_as_unpriced(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + (output_dir / "ironclaw-reborn-serve.stderr.log").write_text( + "--- ironclaw-reborn serve start 2026-07-10T12:00:00Z ---\n" + "INFO REBORN_INFERENCE_USAGE operation=provider_complete " + "model=deepseek-ai/DeepSeek-V4-Flash input_tokens=100 " + "output_tokens=20 cache_read_input_tokens=7 " + "cache_creation_input_tokens=3 input_usd_per_token=NaN " + "output_usd_per_token=0.00000028 estimated_usd=0.000014\n", + encoding="utf-8", + ) + result = run_live_qa.ProbeResult( + provider="test", + mode="live:test", + success=True, + latency_ms=1, + details={}, + ) + + summary = run_live_qa._case_inference_usage(output_dir, result) + + self.assertEqual(summary["call_count"], 1) + self.assertEqual(summary["input_tokens"], 100) + self.assertEqual(summary["output_tokens"], 20) + self.assertEqual(summary["cache_read_input_tokens"], 7) + self.assertEqual(summary["cache_creation_input_tokens"], 3) + self.assertEqual(summary["estimated_usd"], "0") + self.assertEqual(summary["unpriced_call_count"], 1) + self.assertEqual( + summary["events"][0]["pricing_source"], + "provider_usage_malformed", + ) + self.assertNotIn("estimated_usd", summary["events"][0]) + def test_case_inference_usage_charges_cache_reads_at_cache_rate(self): with tempfile.TemporaryDirectory() as tmpdir: output_dir = Path(tmpdir) From 90d0cbc27b090fb08b0ca81ca69d78ca7680c846 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Fri, 10 Jul 2026 19:54:22 +0300 Subject: [PATCH 6/6] fix(canary): keep semantic judge reason in artifacts (#5931) --- scripts/reborn_webui_v2_live_qa/run_live_qa.py | 3 +++ scripts/reborn_webui_v2_live_qa/test_run_live_qa.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index a1e9f274aa6..0cb6ad5cb34 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -1241,6 +1241,9 @@ def _safe_semantic_judge_payload(value: dict[str, object]) -> dict[str, object]: safe["completed"] = value["completed"] if isinstance(value.get("confidence"), (int, float)): safe["confidence"] = value["confidence"] + reason = value.get("reason") + if isinstance(reason, str): + safe["reason"] = reason usage = _safe_semantic_judge_usage(value.get("inference_usage")) if usage is not None: safe["inference_usage"] = usage diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index dc1aa381768..d09a6ccd0fa 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -1305,9 +1305,12 @@ async def fake_sleep(_seconds: float) -> None: assert raised_error is not None self.assertIn("semantic_judge=", str(raised_error)) self.assertNotIn("response_excerpt", str(raised_error)) - self.assertNotIn("The response does not complete the task.", str(raised_error)) + self.assertIn("The response does not complete the task.", str(raised_error)) self.assertEqual(raised_error.semantic_judge["completed"], False) - self.assertNotIn("reason", raised_error.semantic_judge) + self.assertEqual( + raised_error.semantic_judge["reason"], + "The response does not complete the task.", + ) self.assertNotIn("response_excerpt", raised_error.semantic_judge) def test_semantic_judge_passed_respects_confidence_threshold(self):