From c369e82f336a7ed25341971d49f6d53c9621da36 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 3 Aug 2026 14:46:33 -0400 Subject: [PATCH 1/4] feat(code): refresh genai-prices catalog hourly in the background The bundled pricing catalog only refreshes when the genai-prices pin is bumped, while upstream merges new model pricing continuously. Start genai-prices' UpdatePrices daemon on the first successful pricing import (never at CLI startup) so calc_price uses the fresher upstream catalog within the hour. DEEPAGENTS_CODE_PRICES_AUTO_UPDATE opts out; a failed updater start logs once and pricing falls back to the bundled data. --- libs/code/deepagents_code/_env_vars.py | 9 ++ libs/code/deepagents_code/config_manifest.py | 10 ++ libs/code/deepagents_code/cost_tracking.py | 57 ++++++++- libs/code/tests/unit_tests/conftest.py | 16 +++ .../tests/unit_tests/test_cost_tracking.py | 114 +++++++++++++++++- 5 files changed, 202 insertions(+), 4 deletions(-) diff --git a/libs/code/deepagents_code/_env_vars.py b/libs/code/deepagents_code/_env_vars.py index 9ded8e30e8..a4040b842c 100644 --- a/libs/code/deepagents_code/_env_vars.py +++ b/libs/code/deepagents_code/_env_vars.py @@ -315,6 +315,15 @@ When unset, plugins are stored under `DEFAULT_CONFIG_DIR / "plugins"`. """ +PRICES_AUTO_UPDATE = "DEEPAGENTS_CODE_PRICES_AUTO_UPDATE" +"""Toggle hourly background refresh of the `genai-prices` pricing catalog. + +Enabled by default; set to a falsy value (`0`, `false`, `no`, `off`, or empty) +to keep using only the pricing data bundled with the installed `genai-prices` +package. Parsed by `is_env_truthy` at first pricing use, so the updater +thread is never started when disabled. +""" + RECURSION_LIMIT = "DEEPAGENTS_CODE_RECURSION_LIMIT" """Override the main agent's LangGraph `recursion_limit` (graph step budget). diff --git a/libs/code/deepagents_code/config_manifest.py b/libs/code/deepagents_code/config_manifest.py index 6faad45744..1d62dc8c25 100644 --- a/libs/code/deepagents_code/config_manifest.py +++ b/libs/code/deepagents_code/config_manifest.py @@ -1430,6 +1430,16 @@ def _credential_options() -> tuple[ConfigOption, ...]: toml_keys=("update", "check"), invert_toml_bool=True, ), + ConfigOption( + key="update.prices_auto_update", + group="Updates", + summary=( + "Refresh the model pricing catalog from upstream hourly in the background." + ), + kind=OptionKind.BOOL, + default=True, + env_var=_env_vars.PRICES_AUTO_UPDATE, + ), # --- Runtime -------------------------------------------------------- ConfigOption( key="runtime.recursion_limit", diff --git a/libs/code/deepagents_code/cost_tracking.py b/libs/code/deepagents_code/cost_tracking.py index c88a74ebd3..bedd5a5465 100644 --- a/libs/code/deepagents_code/cost_tracking.py +++ b/libs/code/deepagents_code/cost_tracking.py @@ -29,8 +29,11 @@ Every caller uses `estimate_cost`, the only function that imports or calls `genai-prices`. The import is lazy so the package and its bundled pricing data -stay off the CLI startup path. Unsupported models and malformed usage return -`None`; pricing must never interrupt a model turn. +stay off the CLI startup path. On that first successful import a daemon-thread +updater starts refreshing the catalog from upstream hourly (see +`_start_price_updater`); `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` opts out. +Unsupported models and malformed usage return `None`; pricing must never +interrupt a model turn. """ from __future__ import annotations @@ -56,6 +59,7 @@ from langchain_core.runnables.config import ensure_config from langgraph.types import Overwrite +from deepagents_code._env_vars import PRICES_AUTO_UPDATE, is_env_truthy from deepagents_code.resume_state import ResumeState if TYPE_CHECKING: @@ -305,6 +309,50 @@ def _clamped_detail( intersection, which is noise rather than news. """ +_PRICE_UPDATER_ATTEMPTED = False +"""Whether starting the genai-prices background updater has been attempted. + +Latched rather than cleared like the health flags above: `UpdatePrices` +raises when a second instance is started, and a failed start attempt is most +plausibly an API incompatibility that retrying on every request cannot fix, +so the attempt happens exactly once per process either way. +""" + + +def _start_price_updater() -> None: + """Start the genai-prices background catalog refresh once per process. + + `UpdatePrices` fetches the upstream `data.json` from + `raw.githubusercontent.com` hourly and installs it via + `set_custom_snapshot`, after which every `calc_price` call transparently + uses the fresher catalog. Fetch failures are logged by genai-prices and + pricing falls back to the bundled data. A fetched snapshot + wholesale-replaces the bundled catalog, which is safe because the fetched + file is the complete upstream catalog rather than a patch. + + There is deliberately no `stop()`/context-manager pairing: the updater + thread is a daemon and dcode sessions are process-scoped, so the thread + simply exits with the process. + """ + global _PRICE_UPDATER_ATTEMPTED # noqa: PLW0603 + if _PRICE_UPDATER_ATTEMPTED or not is_env_truthy(PRICES_AUTO_UPDATE, default=True): + return + _PRICE_UPDATER_ATTEMPTED = True + try: + from genai_prices import UpdatePrices + + UpdatePrices().start(wait=False) + except Exception: + # A raise here is a genai-prices API change, not something a caller can + # fix -- and pricing with the bundled catalog keeps working regardless, + # so this must never propagate into a model turn. + logger.warning( + "Could not start the genai-prices background updater; cost " + "estimates will use the pricing data bundled with the installed " + "package.", + exc_info=True, + ) + def pricing_data_available() -> bool: """Report whether `genai-prices` is currently able to price a request. @@ -333,7 +381,9 @@ def _load_pricing() -> tuple[Any, Any] | None: The pair is deliberately re-imported rather than cached: `sys.modules` makes that cheap, and holding a reference would pin the first-seen `calc_price`, - defeating any later patch of it. + defeating any later patch of it. A successful import is also where the + background catalog updater kicks off, so the updater only runs once pricing + is actually used -- never at CLI startup or import time. Returns: The `(Usage, calc_price)` pair, or `None` when the package is @@ -355,6 +405,7 @@ def _load_pricing() -> tuple[Any, Any] | None: # A success clears the flag: the earlier failure was transient, and leaving # it set would keep telling the user to reinstall a working package. _PRICING_UNAVAILABLE = False + _start_price_updater() return Usage, calc_price diff --git a/libs/code/tests/unit_tests/conftest.py b/libs/code/tests/unit_tests/conftest.py index 53ecf9a72d..1e5bc92926 100644 --- a/libs/code/tests/unit_tests/conftest.py +++ b/libs/code/tests/unit_tests/conftest.py @@ -274,6 +274,22 @@ def _pin_invoked_name(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, log_nonstandard_invoked_name.cache_clear() +@pytest.fixture(autouse=True) +def _disable_prices_auto_update(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep cost-pricing tests from starting the genai-prices updater thread. + + The first `estimate_cost` call of a process starts the `UpdatePrices` + daemon thread, whose hourly catalog fetch hits the network — pytest-socket + reports it under `--disable-socket`, and the thread would otherwise linger + for the whole test session. Set the production opt-out env var by default + so subprocess tests inherit the same no-network behavior. Tests that cover + the updater mock `UpdatePrices` and override this env var themselves. + """ + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.setenv(PRICES_AUTO_UPDATE, "0") + + @pytest.fixture(autouse=True) def _clear_update_env( request: pytest.FixtureRequest, diff --git a/libs/code/tests/unit_tests/test_cost_tracking.py b/libs/code/tests/unit_tests/test_cost_tracking.py index e127bb1ea2..0645d31dc5 100644 --- a/libs/code/tests/unit_tests/test_cost_tracking.py +++ b/libs/code/tests/unit_tests/test_cost_tracking.py @@ -10,7 +10,7 @@ import warnings from types import SimpleNamespace from typing import TYPE_CHECKING, Any, cast, get_type_hints -from unittest.mock import patch +from unittest.mock import MagicMock, patch from uuid import uuid4 import pytest @@ -338,6 +338,118 @@ def test_azure_fallback_overrides_generic_openai_metadata(self) -> None: def test_codex_subscription_usage_is_not_priced_as_openai_api(self) -> None: assert estimate_cost(_usage(), "gpt-5.4", "openai_codex") is None + +class TestPriceUpdater: + """Process-wide background refresh of the genai-prices catalog.""" + + @pytest.fixture(autouse=True) + def _reset_updater_guard(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Isolate the process-wide start latch and snapshot between tests. + + `monkeypatch.setattr` restores `_PRICE_UPDATER_ATTEMPTED` after each + test, but an earlier test in the same process may already have started + the real updater -- `UpdatePrices.start` refuses a second instance + process-wide, and a real one left running would keep fetching hourly. + Roll back genai-prices' module-level updater handle and custom snapshot + as well so a leak cannot fail or reorder later tests. + """ + monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER_ATTEMPTED", False) + import genai_prices.data_snapshot + import genai_prices.update_prices + + monkeypatch.setattr(genai_prices.update_prices, "_global_update_prices", None) + monkeypatch.setattr(genai_prices.data_snapshot, "_custom_snapshot", None) + + def _patch_update_prices( + self, monkeypatch: pytest.MonkeyPatch, instance: MagicMock + ) -> MagicMock: + """Bind `genai_prices.UpdatePrices` to a constructor returning *instance*.""" + import genai_prices + + constructor = MagicMock(return_value=instance) + monkeypatch.setattr(genai_prices, "UpdatePrices", constructor) + return constructor + + def test_first_pricing_call_starts_updater_once( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import genai_prices + + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + updater = MagicMock() + constructor = self._patch_update_prices(monkeypatch, updater) + + # Price with a stubbed `calc_price` so repeated calls keep exercising + # the real `_load_pricing` import path without catalog work. + monkeypatch.setattr( + genai_prices, + "calc_price", + lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), + ) + + for _ in range(3): + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + constructor.assert_called_once_with() + updater.start.assert_called_once_with(wait=False) + + def test_disabled_env_var_never_starts_updater( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import genai_prices + + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.setenv(PRICES_AUTO_UPDATE, "false") + updater = MagicMock() + constructor = self._patch_update_prices(monkeypatch, updater) + monkeypatch.setattr( + genai_prices, + "calc_price", + lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), + ) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + constructor.assert_not_called() + updater.start.assert_not_called() + + def test_failed_start_prices_with_bundled_data_and_logs_once( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + """A broken updater API must not break pricing or spam the log. + + The failure is latched because retrying every request cannot fix an + incompatible genai-prices release -- it would only repeat the warning + on every model call while the bundled catalog prices fine. + """ + import genai_prices + + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + updater = MagicMock() + updater.start.side_effect = RuntimeError("unexpected genai-prices API") + self._patch_update_prices(monkeypatch, updater) + monkeypatch.setattr( + genai_prices, + "calc_price", + lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), + ) + + with caplog.at_level(logging.WARNING, logger="deepagents_code.cost_tracking"): + for _ in range(3): + assert estimate_cost( + _usage(), KNOWN_MODEL, KNOWN_PROVIDER + ) == pytest.approx(0.01) + + updater.start.assert_called_once_with(wait=False) + warning = "Could not start the genai-prices background updater" + assert caplog.text.count(warning) == 1 + assert cost_tracking.pricing_data_available() + def test_cache_read_is_priced_separately(self) -> None: uncached = estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) cached = estimate_cost( From f4846b914c7f60615b02968ad344cc127b047c7e Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 3 Aug 2026 15:19:25 -0400 Subject: [PATCH 2/4] fix(code): resolve empty prices auto-update env var as disabled An explicitly empty DEEPAGENTS_CODE_PRICES_AUTO_UPDATE disables the updater at runtime via is_env_truthy, but manifest resolution skipped empty env values, so 'dcode config get update.prices_auto_update' reported the default true while the updater was actually off. --- libs/code/deepagents_code/config_manifest.py | 1 + libs/code/tests/unit_tests/test_config_manifest.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/libs/code/deepagents_code/config_manifest.py b/libs/code/deepagents_code/config_manifest.py index 1d62dc8c25..5b36422934 100644 --- a/libs/code/deepagents_code/config_manifest.py +++ b/libs/code/deepagents_code/config_manifest.py @@ -1439,6 +1439,7 @@ def _credential_options() -> tuple[ConfigOption, ...]: kind=OptionKind.BOOL, default=True, env_var=_env_vars.PRICES_AUTO_UPDATE, + empty_env_is_false=True, ), # --- Runtime -------------------------------------------------------- ConfigOption( diff --git a/libs/code/tests/unit_tests/test_config_manifest.py b/libs/code/tests/unit_tests/test_config_manifest.py index f80f25fbcf..4754391ac5 100644 --- a/libs/code/tests/unit_tests/test_config_manifest.py +++ b/libs/code/tests/unit_tests/test_config_manifest.py @@ -1219,6 +1219,17 @@ def test_no_update_check_resolves_inverted_persisted_check() -> None: ) +def test_prices_auto_update_empty_env_disables(monkeypatch) -> None: + """An explicitly empty env value opts out, matching `_start_price_updater`.""" + opt = get_option("update.prices_auto_update") + assert opt is not None + monkeypatch.setenv(_env_vars.PRICES_AUTO_UPDATE, "") + assert resolve_scalar(opt, toml_data={}) == ( + False, + f"env ({_env_vars.PRICES_AUTO_UPDATE})", + ) + + def test_resolve_ptc_delegates_to_parser() -> None: """The PTC kind routes through the dedicated allowlist parser.""" opt = get_option("interpreter.ptc") From c94d56bdfdf74652b1d48013dbc82538d3429028 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 3 Aug 2026 15:37:43 -0400 Subject: [PATCH 3/4] fix(code): harden the background price catalog updater Refuse an upstream catalog listing fewer providers than the bundled data. genai-prices validates only that the payload is an array of well-formed providers, so an empty or mid-publish data.json installs cleanly and makes every calc_price raise LookupError -- which this module reports as "no published rates for this model", leaving costs to stop accruing with no visible cause. The guard raises rather than returning None, because _update_prices installs whatever fetch returns and None would discard a healthy earlier fetch. Quiet the genai-prices logger. Its background thread logs a failed hourly refresh at ERROR, and with no handler attached that clears logging.lastResort's WARNING threshold and prints over the alternate-screen TUI once an hour for any offline or proxied session. _start_price_updater also claims the logger itself, for the server graph and embedded hosts that never run _quiet_sdk_logging. Honor DEEPAGENTS_CODE_OFFLINE alongside the dedicated opt-out, and record the new unpinned egress as T11b in the threat model. Serialize the start attempt under a lock. estimate_cost runs both on the event loop and on the executor threads that price drained records, so an unguarded check-then-set let two threads reach start(); the loser tripped genai-prices' process-wide singleton guard and its RuntimeError was reported as a failed start while the winner's updater ran fine. The warning now names the exception type and message, which differ in what the user should do about them. Make update.prices_auto_update settable in config.toml, and tie the manifest default to the one _start_price_updater applies. Tests stand UpdatePrices in with an autospec, so a renamed keyword in a patch release fails the suite instead of silently killing the feature in production. TestPriceUpdater moves out of the middle of TestEstimateCost, where it had re-parented 21 pricing tests into itself. --- libs/code/THREAT_MODEL.md | 7 + libs/code/deepagents_code/_env_vars.py | 11 +- libs/code/deepagents_code/config.py | 12 +- libs/code/deepagents_code/config_manifest.py | 1 + libs/code/deepagents_code/cost_tracking.py | 186 +++++-- libs/code/tests/unit_tests/conftest.py | 10 +- libs/code/tests/unit_tests/test_config.py | 13 + .../tests/unit_tests/test_config_manifest.py | 41 ++ .../tests/unit_tests/test_cost_tracking.py | 478 +++++++++++++----- 9 files changed, 605 insertions(+), 154 deletions(-) diff --git a/libs/code/THREAT_MODEL.md b/libs/code/THREAT_MODEL.md index db5a5c9bba..acbf703c48 100644 --- a/libs/code/THREAT_MODEL.md +++ b/libs/code/THREAT_MODEL.md @@ -419,6 +419,13 @@ - **Mitigations**: (1) SHA-256 verified against the pinned hash table before move — a mismatch aborts the install and leaves `BIN_DIR` clean. (2) Network egress is limited to `github.com`. (3) Opt-out via `DEEPAGENTS_CODE_OFFLINE` for air-gapped environments, or `DEEPAGENTS_CODE_RIPGREP_INSTALLER=system` to defer to the OS package manager instead of the managed binary. (4) Pinned version + checksums are bumped in-tree, so a compromised upstream release is detected on the next Deep Agents Code release rather than silently propagating. (5) Atomic move-into-place avoids partial installs when concurrent CLI invocations race. (6) The eager install-script path is non-`sudo` (no system package manager is invoked in the default `managed` mode). - **Preconditions**: User has not installed `rg` via their package manager, `DEEPAGENTS_CODE_OFFLINE` is unset, `DEEPAGENTS_CODE_RIPGREP_INSTALLER` is not `system`, and the host can reach `github.com`. The pinned SHA-256 in `RIPGREP_ASSETS` would need to be incorrect (a supply-chain compromise of the deepagents-code release) for a tampered binary to be installed. +#### T11b: Unpinned Pricing Catalog Fetched Hourly from a Mutable Upstream Ref + +- **Flow**: Background daemon thread started by `cost_tracking._start_price_updater` on the first priced model request. +- **Description**: Unless opted out, Deep Agents Code starts `genai_prices.UpdatePrices`, which fetches `raw.githubusercontent.com/pydantic/genai-prices/refs/heads/main/prices/new_data/v2/data.json` every hour and installs it via `set_custom_snapshot`. The fetched catalog wholesale-replaces the pricing data bundled with the installed package for the life of the process. Unlike the ripgrep download (T11), the payload is **not** checksummed and the URL names a mutable branch ref rather than a pinned release, so the content can change between any two fetches. The blast radius is confined to displayed cost estimates — the catalog is parsed as data by `genai-prices`, never executed — but corrupt, regressed, or hostile upstream data silently changes every cost figure the user sees, and a catalog that omits providers makes lookups fail in a way that reads as "this model has no published rates." +- **Mitigations**: (1) Opt-out via `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE=0`, or `DEEPAGENTS_CODE_OFFLINE` for air-gapped environments, both checked before the thread starts. (2) `cost_tracking._build_price_updater` refuses a fetched catalog listing fewer providers than the bundled one, so a truncated or mid-publish `data.json` cannot take effect. (3) A refused or failed fetch leaves the previously installed catalog in place rather than clearing it. (4) `genai-prices` rejects any payload that is not a JSON array of schema-valid providers. (5) Network egress is limited to `raw.githubusercontent.com`. (6) The updater is started lazily on first pricing, never at CLI startup, so a session that prices nothing makes no request. +- **Preconditions**: `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` is not falsy, `DEEPAGENTS_CODE_OFFLINE` is unset, the host can reach `raw.githubusercontent.com`, and at least one model request is priced. For tampered data to be installed, the upstream repository or the CDN path would need to be compromised **and** the substituted catalog would need to list at least as many providers as the bundled data. + #### T12: Project `.env` Injects Shell Interpreter Startup Hooks - **Flow**: DF10 (project `.env` → process environment) → bash subprocess startup (DF19) diff --git a/libs/code/deepagents_code/_env_vars.py b/libs/code/deepagents_code/_env_vars.py index a4040b842c..cf1aea8097 100644 --- a/libs/code/deepagents_code/_env_vars.py +++ b/libs/code/deepagents_code/_env_vars.py @@ -320,8 +320,15 @@ Enabled by default; set to a falsy value (`0`, `false`, `no`, `off`, or empty) to keep using only the pricing data bundled with the installed `genai-prices` -package. Parsed by `is_env_truthy` at first pricing use, so the updater -thread is never started when disabled. +package. `DEEPAGENTS_CODE_OFFLINE` suppresses the refresh too, along with +every other network fetch. + +Parsed by `is_env_truthy` on each pricing call until the updater starts, and +never read again after that -- so disabling it mid-process has no effect on a +running updater, while enabling it mid-process starts one on the next priced +request. Also the escape hatch for hosts embedding this package that manage +`genai_prices.UpdatePrices` themselves: genai-prices permits one updater per +process, so an embedder that starts its own would otherwise race this one. """ RECURSION_LIMIT = "DEEPAGENTS_CODE_RECURSION_LIMIT" diff --git a/libs/code/deepagents_code/config.py b/libs/code/deepagents_code/config.py index 153aaa08a2..6992feec19 100644 --- a/libs/code/deepagents_code/config.py +++ b/libs/code/deepagents_code/config.py @@ -531,6 +531,7 @@ class _LangSmithProfileConfig(Protocol): _QUIET_SDK_LOGGER_NAMES = ( "deepagents.profiles.harness.harness_profiles", + "genai-prices", "langchain", "langsmith", ) @@ -539,10 +540,13 @@ class _LangSmithProfileConfig(Protocol): def _quiet_sdk_logging() -> None: """Keep non-actionable SDK diagnostics off the terminal. - The harness-profile resolver and tracing SDKs emit diagnostics on their own - logger hierarchies. With no handler attached, warnings reach Python's - last-resort stderr handler and can bleed into command output or the - alternate-screen TUI. Route them to the debug log when + The harness-profile resolver, tracing SDKs, and the `genai-prices` price + updater emit diagnostics on their own logger hierarchies. With no handler + attached, warnings reach Python's last-resort stderr handler and can bleed + into command output or the alternate-screen TUI -- the price updater logs + at ERROR from a background thread once an hour, so an offline or + proxied session would otherwise get a stderr line over the TUI on every + failed refresh. Route them to the debug log when `DEEPAGENTS_CODE_DEBUG` is set, otherwise attach a `NullHandler` so they stay off the terminal. Other Deep Agents loggers remain untouched so actionable runtime warnings are still visible. diff --git a/libs/code/deepagents_code/config_manifest.py b/libs/code/deepagents_code/config_manifest.py index 5b36422934..e25128104d 100644 --- a/libs/code/deepagents_code/config_manifest.py +++ b/libs/code/deepagents_code/config_manifest.py @@ -1439,6 +1439,7 @@ def _credential_options() -> tuple[ConfigOption, ...]: kind=OptionKind.BOOL, default=True, env_var=_env_vars.PRICES_AUTO_UPDATE, + toml_keys=("update", "prices_auto_update"), empty_env_is_false=True, ), # --- Runtime -------------------------------------------------------- diff --git a/libs/code/deepagents_code/cost_tracking.py b/libs/code/deepagents_code/cost_tracking.py index bedd5a5465..6db53df12c 100644 --- a/libs/code/deepagents_code/cost_tracking.py +++ b/libs/code/deepagents_code/cost_tracking.py @@ -31,7 +31,8 @@ `genai-prices`. The import is lazy so the package and its bundled pricing data stay off the CLI startup path. On that first successful import a daemon-thread updater starts refreshing the catalog from upstream hourly (see -`_start_price_updater`); `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` opts out. +`_start_price_updater`); `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` opts out, and +`DEEPAGENTS_CODE_OFFLINE` suppresses it along with every other network fetch. Unsupported models and malformed usage return `None`; pricing must never interrupt a model turn. """ @@ -59,12 +60,14 @@ from langchain_core.runnables.config import ensure_config from langgraph.types import Overwrite -from deepagents_code._env_vars import PRICES_AUTO_UPDATE, is_env_truthy +from deepagents_code._env_vars import OFFLINE, PRICES_AUTO_UPDATE, is_env_truthy from deepagents_code.resume_state import ResumeState if TYPE_CHECKING: from uuid import UUID + from genai_prices import UpdatePrices + from genai_prices.data_snapshot import DataSnapshot from langchain_core.outputs import LLMResult from langgraph.runtime import Runtime @@ -309,49 +312,170 @@ def _clamped_detail( intersection, which is noise rather than news. """ +_PRICE_UPDATER_LOCK = threading.Lock() +"""Serializes the check-then-set of `_PRICE_UPDATER_ATTEMPTED`. + +`estimate_cost` runs both inline on the event loop and from the executor +threads that price drained records, so two turns really can reach the start +attempt at once. Unguarded, both would pass the flag check; the loser then +trips the process-wide singleton guard inside `UpdatePrices.start` and its +`RuntimeError` gets reported as a failed start even though the winner's +updater is running fine. +""" + _PRICE_UPDATER_ATTEMPTED = False """Whether starting the genai-prices background updater has been attempted. -Latched rather than cleared like the health flags above: `UpdatePrices` -raises when a second instance is started, and a failed start attempt is most -plausibly an API incompatibility that retrying on every request cannot fix, -so the attempt happens exactly once per process either way. +Latched rather than cleared like the health flags above: every way the start +can fail -- an incompatible `UpdatePrices` API, another component already +owning the process-wide singleton, a thread that cannot be spawned -- lasts +for the life of the process, so retrying on the next request would only +repeat the warning. Set under `_PRICE_UPDATER_LOCK` before the attempt, so +success and failure both latch. +""" + +_PRICE_UPDATER: UpdatePrices | None = None +"""The `UpdatePrices` instance this process started, or `None`. + +Held purely so the updater stays queryable: genai-prices records the most +recent background fetch failure on the instance, and dropping the reference +would put that out of reach for the rest of the process. +""" + +_TRUNCATED_CATALOG_REPORTED = False +"""Whether a refused upstream catalog has been reported on this package's logger. + +Not cleared, for the same reason as `_AUDIO_CACHE_OVERLAP_REPORTED`: the +refusal recurs every hour for as long as upstream stays broken, and repeating +the warning on each retry is noise rather than news. genai-prices logs its own +line per attempt, which the debug log captures. """ +def _build_price_updater(update_prices_cls: type[UpdatePrices]) -> UpdatePrices: + """Build an `UpdatePrices` that refuses a catalog smaller than the bundled one. + + A fetched snapshot wholesale-replaces the bundled catalog rather than + merging into it, and genai-prices validates only that the payload is a JSON + array of well-formed providers. An empty or half-published upstream file + therefore installs cleanly and makes every subsequent `calc_price` raise + `LookupError` -- which this module reports as "no published rates for this + model" rather than as a broken catalog, so the user watches costs stop + accruing with no indication why. Comparing provider counts is a coarse + check, but it is the one that catches that failure. + + Args: + update_prices_cls: The lazily imported `genai_prices.UpdatePrices`. + + Returns: + An unstarted instance of a guarded `UpdatePrices` subclass. + """ + from genai_prices.data import providers as bundled_providers + + bundled_count = len(bundled_providers) + + # Subclassed from the lazily imported class rather than declared at module + # scope, which would drag `genai_prices` onto the CLI startup path. + class _GuardedUpdatePrices(update_prices_cls): # ty: ignore[unsupported-base] + """`UpdatePrices` that validates a fetch before it can be installed.""" + + def fetch(self) -> DataSnapshot | None: + """Fetch upstream prices, rejecting a payload that lost providers. + + Returns: + The fetched snapshot when it is at least as complete as the + bundled catalog. + + Raises: + ValueError: When the fetched catalog lists fewer providers than + the bundled one. + """ + global _TRUNCATED_CATALOG_REPORTED # noqa: PLW0603 + snapshot = super().fetch() + fetched_count = len(snapshot.providers) if snapshot else 0 + if fetched_count >= bundled_count: + return snapshot + if not _TRUNCATED_CATALOG_REPORTED: + _TRUNCATED_CATALOG_REPORTED = True + logger.warning( + "Refusing an upstream pricing catalog listing %d providers " + "against %d bundled with the installed package; continuing " + "with the catalog already in use. Upstream data.json may be " + "mid-publish.", + fetched_count, + bundled_count, + ) + # Raising rather than returning `None` keeps the last good catalog: + # `_update_prices` installs whatever `fetch` returns, `None` + # included, so returning would discard a healthy earlier fetch. The + # background loop treats a raise as a failed refresh and retries on + # the next interval. + msg = ( + f"Refused pricing catalog with {fetched_count} providers " + f"({bundled_count} bundled)" + ) + raise ValueError(msg) + + return _GuardedUpdatePrices() + + def _start_price_updater() -> None: """Start the genai-prices background catalog refresh once per process. `UpdatePrices` fetches the upstream `data.json` from `raw.githubusercontent.com` hourly and installs it via `set_custom_snapshot`, after which every `calc_price` call transparently - uses the fresher catalog. Fetch failures are logged by genai-prices and - pricing falls back to the bundled data. A fetched snapshot - wholesale-replaces the bundled catalog, which is safe because the fetched - file is the complete upstream catalog rather than a patch. - - There is deliberately no `stop()`/context-manager pairing: the updater - thread is a daemon and dcode sessions are process-scoped, so the thread - simply exits with the process. + uses the fresher catalog. `_build_price_updater` guards what may be + installed. + + A failed or refused fetch leaves the previously installed snapshot in + place -- the bundled catalog until the first fetch succeeds, the last good + fetch after that. genai-prices never reverts to bundled data on its own; + only `stop()` does, and there is deliberately no `stop()`/context-manager + pairing here because the updater thread is a daemon and there is exactly + one per process, so it exits with the process. + + Does nothing when `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` is falsy or + `DEEPAGENTS_CODE_OFFLINE` is truthy. """ - global _PRICE_UPDATER_ATTEMPTED # noqa: PLW0603 - if _PRICE_UPDATER_ATTEMPTED or not is_env_truthy(PRICES_AUTO_UPDATE, default=True): + global _PRICE_UPDATER, _PRICE_UPDATER_ATTEMPTED # noqa: PLW0603 + if not is_env_truthy(PRICES_AUTO_UPDATE, default=True) or is_env_truthy(OFFLINE): return - _PRICE_UPDATER_ATTEMPTED = True - try: - from genai_prices import UpdatePrices - - UpdatePrices().start(wait=False) - except Exception: - # A raise here is a genai-prices API change, not something a caller can - # fix -- and pricing with the bundled catalog keeps working regardless, - # so this must never propagate into a model turn. - logger.warning( - "Could not start the genai-prices background updater; cost " - "estimates will use the pricing data bundled with the installed " - "package.", - exc_info=True, - ) + with _PRICE_UPDATER_LOCK: + if _PRICE_UPDATER_ATTEMPTED: + return + _PRICE_UPDATER_ATTEMPTED = True + try: + from genai_prices import UpdatePrices + + # `config._quiet_sdk_logging` already claims the `genai-prices` + # logger on the CLI path, but the server graph and embedded hosts + # never run it. Without a handler, the hourly ERROR from a failed + # refresh reaches `logging.lastResort` and prints over the TUI. + gp_logger = logging.getLogger("genai-prices") + if not gp_logger.handlers: + gp_logger.addHandler(logging.NullHandler()) + updater = _build_price_updater(UpdatePrices) + updater.start(wait=False) + except Exception as exc: + # Deliberately broad: pricing must never interrupt a model turn, so + # nothing raised while starting a best-effort refresh may escape. + # The type and message are interpolated because the causes differ + # in what the user should do -- an incompatible genai-prices API, + # another component already holding the process-wide singleton (in + # which case an updater *is* running, just not ours), or a thread + # that could not be spawned. + logger.warning( + "Could not start the genai-prices background updater (%s: %s); " + "cost estimates will use whichever pricing catalog is already " + "installed, which is the data bundled with the package unless " + "another component started an updater first.", + type(exc).__name__, + exc, + exc_info=True, + ) + else: + _PRICE_UPDATER = updater def pricing_data_available() -> bool: diff --git a/libs/code/tests/unit_tests/conftest.py b/libs/code/tests/unit_tests/conftest.py index 1e5bc92926..57fa09638d 100644 --- a/libs/code/tests/unit_tests/conftest.py +++ b/libs/code/tests/unit_tests/conftest.py @@ -280,10 +280,12 @@ def _disable_prices_auto_update(monkeypatch: pytest.MonkeyPatch) -> None: The first `estimate_cost` call of a process starts the `UpdatePrices` daemon thread, whose hourly catalog fetch hits the network — pytest-socket - reports it under `--disable-socket`, and the thread would otherwise linger - for the whole test session. Set the production opt-out env var by default - so subprocess tests inherit the same no-network behavior. Tests that cover - the updater mock `UpdatePrices` and override this env var themselves. + reports it under `--disable-socket` (which `make test` passes), and the + thread would otherwise linger for the whole test session. Set the + production opt-out env var by default so subprocess tests inherit the same + no-network behavior. Tests that cover the updater stand `UpdatePrices` in + with an autospec and override this env var themselves, so the real thread + never starts anywhere in the suite. """ from deepagents_code._env_vars import PRICES_AUTO_UPDATE diff --git a/libs/code/tests/unit_tests/test_config.py b/libs/code/tests/unit_tests/test_config.py index 8414682b35..5fa7c75a42 100644 --- a/libs/code/tests/unit_tests/test_config.py +++ b/libs/code/tests/unit_tests/test_config.py @@ -3659,6 +3659,19 @@ def test_attaches_null_handler_without_debug( # last-resort stderr handler. assert logger.propagate is True + def test_covers_the_logger_genai_prices_actually_uses(self) -> None: + """The price updater's logger must be quieted under its real name. + + Its background thread logs a failed hourly catalog refresh at ERROR. + Unhandled, that clears `logging.lastResort`'s WARNING threshold and + prints over the alternate-screen TUI once an hour for any offline or + proxied session. Reading the name off the module pins the coupling, so + an upstream rename fails here rather than in a user's terminal. + """ + import genai_prices.update_prices + + assert genai_prices.update_prices.logger.name in _QUIET_SDK_LOGGER_NAMES + def test_idempotent(self, monkeypatch: pytest.MonkeyPatch) -> None: """Repeated calls do not stack duplicate handlers.""" from deepagents_code._env_vars import DEBUG diff --git a/libs/code/tests/unit_tests/test_config_manifest.py b/libs/code/tests/unit_tests/test_config_manifest.py index 4754391ac5..37dd40bb10 100644 --- a/libs/code/tests/unit_tests/test_config_manifest.py +++ b/libs/code/tests/unit_tests/test_config_manifest.py @@ -10,6 +10,7 @@ import argparse import os from typing import Any +from unittest.mock import MagicMock import pytest @@ -1219,6 +1220,46 @@ def test_no_update_check_resolves_inverted_persisted_check() -> None: ) +def test_prices_auto_update_default_matches_runtime(monkeypatch) -> None: + """The manifest default must agree with the one `_start_price_updater` uses. + + They are declared independently, so a drift makes `dcode config get + update.prices_auto_update` report the opposite of what the updater does. + """ + from deepagents_code import cost_tracking + + opt = get_option("update.prices_auto_update") + assert opt is not None + monkeypatch.delenv(_env_vars.PRICES_AUTO_UPDATE, raising=False) + # Suite-wide fixtures set both; the updater honors OFFLINE as well. + monkeypatch.delenv(_env_vars.OFFLINE, raising=False) + + started: list[object] = [] + monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER_ATTEMPTED", False) + monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER", None) + monkeypatch.setattr( + cost_tracking, + "_build_price_updater", + lambda cls: started.append(cls) or MagicMock(), + ) + cost_tracking._start_price_updater() + + assert opt.default is True + assert bool(started) is opt.default + + +def test_prices_auto_update_persists_in_toml(monkeypatch) -> None: + """The dotted key resolves from `config.toml`, not env vars only.""" + opt = get_option("update.prices_auto_update") + assert opt is not None + monkeypatch.delenv(_env_vars.PRICES_AUTO_UPDATE, raising=False) + assert opt.toml_keys == ("update", "prices_auto_update") + assert resolve_scalar(opt, toml_data={"update": {"prices_auto_update": False}}) == ( + False, + "config.toml", + ) + + def test_prices_auto_update_empty_env_disables(monkeypatch) -> None: """An explicitly empty env value opts out, matching `_start_price_updater`.""" opt = get_option("update.prices_auto_update") diff --git a/libs/code/tests/unit_tests/test_cost_tracking.py b/libs/code/tests/unit_tests/test_cost_tracking.py index 0645d31dc5..7b741890ca 100644 --- a/libs/code/tests/unit_tests/test_cost_tracking.py +++ b/libs/code/tests/unit_tests/test_cost_tracking.py @@ -4,13 +4,15 @@ import asyncio import builtins +import inspect import logging import subprocess import sys +import threading import warnings from types import SimpleNamespace from typing import TYPE_CHECKING, Any, cast, get_type_hints -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, create_autospec, patch from uuid import uuid4 import pytest @@ -55,6 +57,7 @@ if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator + from genai_prices import UpdatePrices from langchain_core.callbacks import CallbackManagerForLLMRun from langchain_core.language_models import BaseChatModel from langchain_core.runnables import RunnableConfig @@ -338,118 +341,6 @@ def test_azure_fallback_overrides_generic_openai_metadata(self) -> None: def test_codex_subscription_usage_is_not_priced_as_openai_api(self) -> None: assert estimate_cost(_usage(), "gpt-5.4", "openai_codex") is None - -class TestPriceUpdater: - """Process-wide background refresh of the genai-prices catalog.""" - - @pytest.fixture(autouse=True) - def _reset_updater_guard(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Isolate the process-wide start latch and snapshot between tests. - - `monkeypatch.setattr` restores `_PRICE_UPDATER_ATTEMPTED` after each - test, but an earlier test in the same process may already have started - the real updater -- `UpdatePrices.start` refuses a second instance - process-wide, and a real one left running would keep fetching hourly. - Roll back genai-prices' module-level updater handle and custom snapshot - as well so a leak cannot fail or reorder later tests. - """ - monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER_ATTEMPTED", False) - import genai_prices.data_snapshot - import genai_prices.update_prices - - monkeypatch.setattr(genai_prices.update_prices, "_global_update_prices", None) - monkeypatch.setattr(genai_prices.data_snapshot, "_custom_snapshot", None) - - def _patch_update_prices( - self, monkeypatch: pytest.MonkeyPatch, instance: MagicMock - ) -> MagicMock: - """Bind `genai_prices.UpdatePrices` to a constructor returning *instance*.""" - import genai_prices - - constructor = MagicMock(return_value=instance) - monkeypatch.setattr(genai_prices, "UpdatePrices", constructor) - return constructor - - def test_first_pricing_call_starts_updater_once( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - import genai_prices - - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) - updater = MagicMock() - constructor = self._patch_update_prices(monkeypatch, updater) - - # Price with a stubbed `calc_price` so repeated calls keep exercising - # the real `_load_pricing` import path without catalog work. - monkeypatch.setattr( - genai_prices, - "calc_price", - lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), - ) - - for _ in range(3): - assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None - - constructor.assert_called_once_with() - updater.start.assert_called_once_with(wait=False) - - def test_disabled_env_var_never_starts_updater( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - import genai_prices - - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.setenv(PRICES_AUTO_UPDATE, "false") - updater = MagicMock() - constructor = self._patch_update_prices(monkeypatch, updater) - monkeypatch.setattr( - genai_prices, - "calc_price", - lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), - ) - - assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None - - constructor.assert_not_called() - updater.start.assert_not_called() - - def test_failed_start_prices_with_bundled_data_and_logs_once( - self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture - ) -> None: - """A broken updater API must not break pricing or spam the log. - - The failure is latched because retrying every request cannot fix an - incompatible genai-prices release -- it would only repeat the warning - on every model call while the bundled catalog prices fine. - """ - import genai_prices - - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) - updater = MagicMock() - updater.start.side_effect = RuntimeError("unexpected genai-prices API") - self._patch_update_prices(monkeypatch, updater) - monkeypatch.setattr( - genai_prices, - "calc_price", - lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), - ) - - with caplog.at_level(logging.WARNING, logger="deepagents_code.cost_tracking"): - for _ in range(3): - assert estimate_cost( - _usage(), KNOWN_MODEL, KNOWN_PROVIDER - ) == pytest.approx(0.01) - - updater.start.assert_called_once_with(wait=False) - warning = "Could not start the genai-prices background updater" - assert caplog.text.count(warning) == 1 - assert cost_tracking.pricing_data_available() - def test_cache_read_is_priced_separately(self) -> None: uncached = estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) cached = estimate_cost( @@ -942,6 +833,367 @@ def test_module_import_does_not_import_genai_prices(self) -> None: assert result.returncode == 0, result.stderr +class TestPriceUpdater: + """Process-wide background refresh of the genai-prices catalog.""" + + @pytest.fixture(autouse=True) + def _reset_updater_guard(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Isolate the process-wide updater state between tests. + + `monkeypatch.setattr` restores this module's latches after each test. + genai-prices' own module globals are rolled back too: a leaked handle + in `_global_update_prices` would make a later `start()` raise, and a + leaked `_custom_snapshot` would silently re-price every later test off + fetched data. Note this does *not* stop a daemon thread that a real + `start()` already spawned -- it only clears the guard and the catalog. + Nothing here starts a real one; the conftest opt-out keeps the rest of + the suite from doing so either. + + `DEEPAGENTS_CODE_OFFLINE` is cleared because the suite-wide + `_skip_managed_tool_downloads` fixture sets it, and it now suppresses + this updater too -- leaving it set would make every test here pass + vacuously. The test that covers the offline gate sets it back. + """ + from deepagents_code._env_vars import OFFLINE + + monkeypatch.delenv(OFFLINE, raising=False) + monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER_ATTEMPTED", False) + monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER", None) + monkeypatch.setattr(cost_tracking, "_TRUNCATED_CATALOG_REPORTED", False) + import genai_prices.data_snapshot + import genai_prices.update_prices + + monkeypatch.setattr(genai_prices.update_prices, "_global_update_prices", None) + monkeypatch.setattr(genai_prices.data_snapshot, "_custom_snapshot", None) + + def _patch_updater( + self, monkeypatch: pytest.MonkeyPatch, instance: MagicMock + ) -> MagicMock: + """Bind `_build_price_updater` to a factory returning *instance*. + + The factory rather than `genai_prices.UpdatePrices` is patched because + `_build_price_updater` subclasses whatever it is handed, which a + `MagicMock` cannot stand in for. Callers pass an autospec instance so + `start(wait=False)` is checked against the real signature. + """ + factory = MagicMock(return_value=instance) + monkeypatch.setattr(cost_tracking, "_build_price_updater", factory) + return factory + + @staticmethod + def _autospec_updater() -> MagicMock: + """An `UpdatePrices` stand-in that enforces the real method signatures. + + Returns: + An autospec instance whose `start` rejects a call the real class + would reject. + """ + from genai_prices import UpdatePrices + + return create_autospec(UpdatePrices, instance=True) + + @staticmethod + def _stub_calc_price(monkeypatch: pytest.MonkeyPatch) -> None: + """Price every request at $0.01 without touching the catalog.""" + import genai_prices + + monkeypatch.setattr( + genai_prices, + "calc_price", + lambda *_args, **_kwargs: SimpleNamespace(total_price=0.01), + ) + + def test_first_pricing_call_starts_updater_once( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from genai_prices import UpdatePrices + + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + updater = self._autospec_updater() + factory = self._patch_updater(monkeypatch, updater) + # Repeated calls keep exercising the real `_load_pricing` import path. + self._stub_calc_price(monkeypatch) + + for _ in range(3): + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + factory.assert_called_once_with(UpdatePrices) + updater.start.assert_called_once_with(wait=False) + assert cost_tracking._PRICE_UPDATER is updater + + def test_starting_claims_the_genai_prices_logger( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The updater's logger never reaches `logging.lastResort`. + + `config._quiet_sdk_logging` covers the CLI, but the server graph and + embedded hosts never call it -- and an unhandled ERROR from the hourly + refresh prints over the TUI. Starting the updater must claim the logger + on its own. + """ + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + gp_logger = logging.getLogger("genai-prices") + monkeypatch.setattr(gp_logger, "handlers", []) + self._patch_updater(monkeypatch, self._autospec_updater()) + self._stub_calc_price(monkeypatch) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + assert gp_logger.handlers + + def test_real_updater_accepts_the_start_call_this_module_makes(self) -> None: + """Pin the genai-prices API the start call depends on. + + Every other test here stands the updater in with an autospec, so this + is what fails if a `>=0.1.1,<0.2.0` release renames `start`'s keyword. + Without it, that drift kills the feature in production -- the broad + `except` downgrades it to a warning -- while the suite stays green. + """ + from genai_prices import UpdatePrices + + signature = inspect.signature(UpdatePrices.start) + wait = signature.parameters["wait"] + assert wait.kind is inspect.Parameter.KEYWORD_ONLY + assert wait.default is False + + @pytest.mark.parametrize("value", ["false", "0", "off", ""]) + def test_disabled_env_var_never_starts_updater( + self, monkeypatch: pytest.MonkeyPatch, value: str + ) -> None: + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.setenv(PRICES_AUTO_UPDATE, value) + factory = self._patch_updater(monkeypatch, self._autospec_updater()) + self._stub_calc_price(monkeypatch) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + factory.assert_not_called() + + def test_offline_mode_never_starts_updater( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """`DEEPAGENTS_CODE_OFFLINE` suppresses this fetch like every other.""" + from deepagents_code._env_vars import OFFLINE, PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + monkeypatch.setenv(OFFLINE, "1") + factory = self._patch_updater(monkeypatch, self._autospec_updater()) + self._stub_calc_price(monkeypatch) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + factory.assert_not_called() + + def test_the_start_attempt_is_serialized_by_a_lock( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The latch is read and set inside `_PRICE_UPDATER_LOCK`. + + `estimate_cost` runs on the event loop and on the executor threads that + price drained records, so an unguarded check-then-set lets two threads + both reach `start()`. The loser trips genai-prices' process-wide + singleton guard and its `RuntimeError` gets reported as a failed start + -- a warning claiming the updater is down while the winner's runs fine. + + Holding the lock here and watching a pricing thread block on it tests + that directly, rather than trying to lose a race on purpose. + """ + from genai_prices import UpdatePrices + + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + self._stub_calc_price(monkeypatch) + factory = self._patch_updater(monkeypatch, self._autospec_updater()) + + priced = threading.Event() + + def _price() -> None: + estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) + priced.set() + + thread = threading.Thread(target=_price) + with cost_tracking._PRICE_UPDATER_LOCK: + thread.start() + # Without the lock the thread sails through the start attempt. + assert not priced.wait(0.25) + factory.assert_not_called() + + thread.join(timeout=5) + assert priced.is_set() + factory.assert_called_once_with(UpdatePrices) + + def test_failed_start_still_prices_and_logs_once( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + """A broken updater must not break pricing or spam the log. + + The failure is latched because nothing a later request does can fix an + incompatible genai-prices release or a claimed singleton -- retrying + would only repeat the warning on every model call while pricing works. + """ + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + updater = self._autospec_updater() + updater.start.side_effect = RuntimeError("unexpected genai-prices API") + self._patch_updater(monkeypatch, updater) + self._stub_calc_price(monkeypatch) + + with caplog.at_level(logging.WARNING, logger="deepagents_code.cost_tracking"): + for _ in range(3): + assert estimate_cost( + _usage(), KNOWN_MODEL, KNOWN_PROVIDER + ) == pytest.approx(0.01) + + updater.start.assert_called_once_with(wait=False) + warning = "Could not start the genai-prices background updater" + records = [r for r in caplog.records if warning in r.getMessage()] + assert len(records) == 1 + # Asserted on the formatted message, not `caplog.text`: `exc_info=True` + # appends a traceback naming the same exception, so a check against the + # full text would pass even if the message dropped the type entirely. + assert "RuntimeError: unexpected genai-prices API" in records[0].getMessage() + assert cost_tracking.pricing_data_available() + assert cost_tracking._PRICE_UPDATER is None + + def test_a_start_failure_does_not_price_from_a_half_built_updater( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A failure inside the factory itself is caught like any other.""" + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + monkeypatch.setattr( + cost_tracking, + "_build_price_updater", + MagicMock(side_effect=AttributeError("no UpdatePrices")), + ) + self._stub_calc_price(monkeypatch) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) == pytest.approx( + 0.01 + ) + assert cost_tracking._PRICE_UPDATER is None + + +class TestPriceCatalogGuard: + """`_build_price_updater` gating on what a fetch is allowed to install.""" + + @pytest.fixture(autouse=True) + def _reset_report_latch(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cost_tracking, "_TRUNCATED_CATALOG_REPORTED", False) + + @staticmethod + def _guarded_updater( + monkeypatch: pytest.MonkeyPatch, fetched: int | None + ) -> tuple[UpdatePrices, int]: + """Build the real guarded updater over a fetch returning *fetched* providers. + + `fetched=None` stands in for the `None` snapshot that + `UpdatePrices.fetch` is typed to allow. + + Returns: + The guarded instance and the bundled provider count it is judged + against. + """ + from genai_prices import UpdatePrices + from genai_prices.data import providers as bundled + + snapshot = ( + None if fetched is None else SimpleNamespace(providers=[object()] * fetched) + ) + monkeypatch.setattr(UpdatePrices, "fetch", lambda _self: snapshot) + return cost_tracking._build_price_updater(UpdatePrices), len(bundled) + + def test_a_complete_catalog_is_installed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from genai_prices.data import providers as bundled + + updater, _ = self._guarded_updater(monkeypatch, len(bundled)) + + snapshot = updater.fetch() + + assert snapshot is not None + assert len(snapshot.providers) == len(bundled) + + def test_a_grown_catalog_is_installed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Upstream adding providers is the normal case, not a truncation.""" + from genai_prices.data import providers as bundled + + updater, _ = self._guarded_updater(monkeypatch, len(bundled) + 5) + + snapshot = updater.fetch() + + assert snapshot is not None + assert len(snapshot.providers) == len(bundled) + 5 + + @pytest.mark.parametrize("fetched", [0, 1, None]) + def test_a_truncated_catalog_is_refused( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + fetched: int | None, + ) -> None: + """An empty or half-published data.json must not replace bundled data. + + genai-prices validates only that the payload is an array of well-formed + providers, so `[]` installs cleanly and makes every later `calc_price` + raise `LookupError` -- which this module reports as an unpriced model + rather than a broken catalog, so costs stop with no visible cause. + """ + updater, bundled_count = self._guarded_updater(monkeypatch, fetched) + + with ( + caplog.at_level(logging.WARNING, logger="deepagents_code.cost_tracking"), + pytest.raises(ValueError, match="Refused pricing catalog"), + ): + updater.fetch() + + assert f"{bundled_count} bundled" in caplog.text + + def test_a_refusal_is_reported_once_not_hourly( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + """The refusal recurs every hour while upstream stays broken.""" + updater, _ = self._guarded_updater(monkeypatch, 0) + + with caplog.at_level(logging.WARNING, logger="deepagents_code.cost_tracking"): + for _ in range(3): + with pytest.raises(ValueError, match="Refused pricing catalog"): + updater.fetch() + + assert caplog.text.count("Refusing an upstream pricing catalog") == 1 + + def test_a_refusal_leaves_the_installed_catalog_alone( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Raising, not returning `None`, is what preserves the last good fetch. + + `UpdatePrices._update_prices` installs whatever `fetch` returns -- + including `None`, which reverts to bundled data. Raising instead makes + the background loop treat the refresh as failed and retry later. + """ + import genai_prices.data_snapshot as snapshot_mod + + updater, _ = self._guarded_updater(monkeypatch, 0) + good = SimpleNamespace(providers=[object()], from_auto_update=True) + monkeypatch.setattr(snapshot_mod, "_custom_snapshot", good) + + with pytest.raises(ValueError, match="Refused pricing catalog"): + updater._update_prices() + + assert snapshot_mod._custom_snapshot is good + + class TestCostTrackingMiddleware: """Tests for cumulative cost writes on the model checkpoint path.""" From 257c53731e2a003765ca81c5833cd7a5108ef969 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 3 Aug 2026 16:00:59 -0400 Subject: [PATCH 4/4] fix(code): honor the persisted prices auto-update opt-out The runtime gate for the background price updater read DEEPAGENTS_CODE_PRICES_AUTO_UPDATE directly, so a user who set [update].prices_auto_update = false in config.toml saw 'config get' report false while the hourly network fetch still started. Route the gate through the manifest resolver so env, TOML, and default resolve with the same precedence the config surface reports. --- libs/code/THREAT_MODEL.md | 2 +- libs/code/deepagents_code/cost_tracking.py | 43 ++++++++++-- .../tests/unit_tests/test_config_manifest.py | 3 + .../tests/unit_tests/test_cost_tracking.py | 67 ++++++++++++++----- 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/libs/code/THREAT_MODEL.md b/libs/code/THREAT_MODEL.md index acbf703c48..1ee2d677e4 100644 --- a/libs/code/THREAT_MODEL.md +++ b/libs/code/THREAT_MODEL.md @@ -423,7 +423,7 @@ - **Flow**: Background daemon thread started by `cost_tracking._start_price_updater` on the first priced model request. - **Description**: Unless opted out, Deep Agents Code starts `genai_prices.UpdatePrices`, which fetches `raw.githubusercontent.com/pydantic/genai-prices/refs/heads/main/prices/new_data/v2/data.json` every hour and installs it via `set_custom_snapshot`. The fetched catalog wholesale-replaces the pricing data bundled with the installed package for the life of the process. Unlike the ripgrep download (T11), the payload is **not** checksummed and the URL names a mutable branch ref rather than a pinned release, so the content can change between any two fetches. The blast radius is confined to displayed cost estimates — the catalog is parsed as data by `genai-prices`, never executed — but corrupt, regressed, or hostile upstream data silently changes every cost figure the user sees, and a catalog that omits providers makes lookups fail in a way that reads as "this model has no published rates." -- **Mitigations**: (1) Opt-out via `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE=0`, or `DEEPAGENTS_CODE_OFFLINE` for air-gapped environments, both checked before the thread starts. (2) `cost_tracking._build_price_updater` refuses a fetched catalog listing fewer providers than the bundled one, so a truncated or mid-publish `data.json` cannot take effect. (3) A refused or failed fetch leaves the previously installed catalog in place rather than clearing it. (4) `genai-prices` rejects any payload that is not a JSON array of schema-valid providers. (5) Network egress is limited to `raw.githubusercontent.com`. (6) The updater is started lazily on first pricing, never at CLI startup, so a session that prices nothing makes no request. +- **Mitigations**: (1) Opt-out via `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE=0` or `[update].prices_auto_update = false` in `config.toml`, or `DEEPAGENTS_CODE_OFFLINE` for air-gapped environments, all checked before the thread starts. (2) `cost_tracking._build_price_updater` refuses a fetched catalog listing fewer providers than the bundled one, so a truncated or mid-publish `data.json` cannot take effect. (3) A refused or failed fetch leaves the previously installed catalog in place rather than clearing it. (4) `genai-prices` rejects any payload that is not a JSON array of schema-valid providers. (5) Network egress is limited to `raw.githubusercontent.com`. (6) The updater is started lazily on first pricing, never at CLI startup, so a session that prices nothing makes no request. - **Preconditions**: `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` is not falsy, `DEEPAGENTS_CODE_OFFLINE` is unset, the host can reach `raw.githubusercontent.com`, and at least one model request is priced. For tampered data to be installed, the upstream repository or the CDN path would need to be compromised **and** the substituted catalog would need to list at least as many providers as the bundled data. #### T12: Project `.env` Injects Shell Interpreter Startup Hooks diff --git a/libs/code/deepagents_code/cost_tracking.py b/libs/code/deepagents_code/cost_tracking.py index 6db53df12c..eb6f8814e2 100644 --- a/libs/code/deepagents_code/cost_tracking.py +++ b/libs/code/deepagents_code/cost_tracking.py @@ -31,7 +31,8 @@ `genai-prices`. The import is lazy so the package and its bundled pricing data stay off the CLI startup path. On that first successful import a daemon-thread updater starts refreshing the catalog from upstream hourly (see -`_start_price_updater`); `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` opts out, and +`_start_price_updater`); `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE=0` or +`[update].prices_auto_update = false` in `config.toml` opts out, and `DEEPAGENTS_CODE_OFFLINE` suppresses it along with every other network fetch. Unsupported models and malformed usage return `None`; pricing must never interrupt a model turn. @@ -60,7 +61,7 @@ from langchain_core.runnables.config import ensure_config from langgraph.types import Overwrite -from deepagents_code._env_vars import OFFLINE, PRICES_AUTO_UPDATE, is_env_truthy +from deepagents_code._env_vars import OFFLINE, is_env_truthy from deepagents_code.resume_state import ResumeState if TYPE_CHECKING: @@ -419,6 +420,31 @@ def fetch(self) -> DataSnapshot | None: return _GuardedUpdatePrices() +def _prices_auto_update_enabled() -> bool: + """Resolve the `update.prices_auto_update` option through the manifest. + + Routing the gate through `resolve_scalar` keeps env-over-TOML precedence + and the `config get update.prices_auto_update` report in lockstep with what + the updater actually does; reading the env var inline would show a user who + opted out in `config.toml` `false` while the hourly fetch still started. + + Returns: + `True` unless the option resolved to disabled or its manifest entry is + missing. + """ + from deepagents_code.config_manifest import ( + get_option, + load_config_toml, + resolve_scalar, + ) + + option = get_option("update.prices_auto_update") + if option is None: + return True + value, _ = resolve_scalar(option, toml_data=load_config_toml()) + return bool(value) + + def _start_price_updater() -> None: """Start the genai-prices background catalog refresh once per process. @@ -435,16 +461,23 @@ def _start_price_updater() -> None: pairing here because the updater thread is a daemon and there is exactly one per process, so it exits with the process. - Does nothing when `DEEPAGENTS_CODE_PRICES_AUTO_UPDATE` is falsy or - `DEEPAGENTS_CODE_OFFLINE` is truthy. + Does nothing when the `update.prices_auto_update` option resolves to + disabled or `DEEPAGENTS_CODE_OFFLINE` is truthy. Either opt-out still marks + the start as attempted: config is read once at process start in practice, + so a later flip would not take effect anyway, and re-resolving on every + priced request would re-read `config.toml` each time. """ global _PRICE_UPDATER, _PRICE_UPDATER_ATTEMPTED # noqa: PLW0603 - if not is_env_truthy(PRICES_AUTO_UPDATE, default=True) or is_env_truthy(OFFLINE): + if is_env_truthy(OFFLINE): return with _PRICE_UPDATER_LOCK: if _PRICE_UPDATER_ATTEMPTED: return _PRICE_UPDATER_ATTEMPTED = True + # Resolved inside the lock so the option load (env + `config.toml`) + # never delays a pricing thread blocked on an in-flight start. + if not _prices_auto_update_enabled(): + return try: from genai_prices import UpdatePrices diff --git a/libs/code/tests/unit_tests/test_config_manifest.py b/libs/code/tests/unit_tests/test_config_manifest.py index 37dd40bb10..1ed440a112 100644 --- a/libs/code/tests/unit_tests/test_config_manifest.py +++ b/libs/code/tests/unit_tests/test_config_manifest.py @@ -1233,6 +1233,9 @@ def test_prices_auto_update_default_matches_runtime(monkeypatch) -> None: monkeypatch.delenv(_env_vars.PRICES_AUTO_UPDATE, raising=False) # Suite-wide fixtures set both; the updater honors OFFLINE as well. monkeypatch.delenv(_env_vars.OFFLINE, raising=False) + # The runtime gate resolves the option, which reads the user's real + # `config.toml` here; an empty table keeps this test on defaults. + monkeypatch.setattr("deepagents_code.config_manifest.load_config_toml", dict) started: list[object] = [] monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER_ATTEMPTED", False) diff --git a/libs/code/tests/unit_tests/test_cost_tracking.py b/libs/code/tests/unit_tests/test_cost_tracking.py index 7b741890ca..a09c0f263e 100644 --- a/libs/code/tests/unit_tests/test_cost_tracking.py +++ b/libs/code/tests/unit_tests/test_cost_tracking.py @@ -860,6 +860,10 @@ def _reset_updater_guard(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER_ATTEMPTED", False) monkeypatch.setattr(cost_tracking, "_PRICE_UPDATER", None) monkeypatch.setattr(cost_tracking, "_TRUNCATED_CATALOG_REPORTED", False) + # The opt-out gate reads the user's real `config.toml` unless the + # read is stubbed, so a local `[update].prices_auto_update = false` + # would silently disable the updater under test. + monkeypatch.setattr("deepagents_code.config_manifest.load_config_toml", dict) import genai_prices.data_snapshot import genai_prices.update_prices @@ -880,6 +884,13 @@ def _patch_updater( monkeypatch.setattr(cost_tracking, "_build_price_updater", factory) return factory + @staticmethod + def _enable_auto_update(monkeypatch: pytest.MonkeyPatch) -> None: + """Clear the conftest opt-out so the updater under test may start.""" + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + @staticmethod def _autospec_updater() -> MagicMock: """An `UpdatePrices` stand-in that enforces the real method signatures. @@ -908,9 +919,7 @@ def test_first_pricing_call_starts_updater_once( ) -> None: from genai_prices import UpdatePrices - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + self._enable_auto_update(monkeypatch) updater = self._autospec_updater() factory = self._patch_updater(monkeypatch, updater) # Repeated calls keep exercising the real `_load_pricing` import path. @@ -933,9 +942,7 @@ def test_starting_claims_the_genai_prices_logger( refresh prints over the TUI. Starting the updater must claim the logger on its own. """ - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + self._enable_auto_update(monkeypatch) gp_logger = logging.getLogger("genai-prices") monkeypatch.setattr(gp_logger, "handlers", []) self._patch_updater(monkeypatch, self._autospec_updater()) @@ -974,13 +981,45 @@ def test_disabled_env_var_never_starts_updater( factory.assert_not_called() + def test_toml_opt_out_never_starts_updater( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The TOML opt-out gates the updater, not just `config get`.""" + monkeypatch.setattr( + "deepagents_code.config_manifest.load_config_toml", + lambda: {"update": {"prices_auto_update": False}}, + ) + factory = self._patch_updater(monkeypatch, self._autospec_updater()) + self._stub_calc_price(monkeypatch) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + factory.assert_not_called() + + def test_env_var_overrides_toml_opt_out( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A truthy env var wins over a persisted TOML opt-out.""" + from deepagents_code._env_vars import PRICES_AUTO_UPDATE + + monkeypatch.setenv(PRICES_AUTO_UPDATE, "1") + monkeypatch.setattr( + "deepagents_code.config_manifest.load_config_toml", + lambda: {"update": {"prices_auto_update": False}}, + ) + factory = self._patch_updater(monkeypatch, self._autospec_updater()) + self._stub_calc_price(monkeypatch) + + assert estimate_cost(_usage(), KNOWN_MODEL, KNOWN_PROVIDER) is not None + + factory.assert_called_once() + def test_offline_mode_never_starts_updater( self, monkeypatch: pytest.MonkeyPatch ) -> None: """`DEEPAGENTS_CODE_OFFLINE` suppresses this fetch like every other.""" - from deepagents_code._env_vars import OFFLINE, PRICES_AUTO_UPDATE + from deepagents_code._env_vars import OFFLINE - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) monkeypatch.setenv(OFFLINE, "1") factory = self._patch_updater(monkeypatch, self._autospec_updater()) self._stub_calc_price(monkeypatch) @@ -1005,9 +1044,7 @@ def test_the_start_attempt_is_serialized_by_a_lock( """ from genai_prices import UpdatePrices - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + self._enable_auto_update(monkeypatch) self._stub_calc_price(monkeypatch) factory = self._patch_updater(monkeypatch, self._autospec_updater()) @@ -1037,9 +1074,7 @@ def test_failed_start_still_prices_and_logs_once( incompatible genai-prices release or a claimed singleton -- retrying would only repeat the warning on every model call while pricing works. """ - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + self._enable_auto_update(monkeypatch) updater = self._autospec_updater() updater.start.side_effect = RuntimeError("unexpected genai-prices API") self._patch_updater(monkeypatch, updater) @@ -1066,9 +1101,7 @@ def test_a_start_failure_does_not_price_from_a_half_built_updater( self, monkeypatch: pytest.MonkeyPatch ) -> None: """A failure inside the factory itself is caught like any other.""" - from deepagents_code._env_vars import PRICES_AUTO_UPDATE - - monkeypatch.delenv(PRICES_AUTO_UPDATE, raising=False) + self._enable_auto_update(monkeypatch) monkeypatch.setattr( cost_tracking, "_build_price_updater",