fix(discovery): persist shared metadata refresh evidence

contextual_orchestrator/model_discovery.py

@@ -15,6 +15,7 @@
 
 from decimal import Decimal
 from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
 import json
 import logging
 import math
@@ -291,6 +292,11 @@ def probe_discovered_model_tool_call_capability(
 _OPENROUTER_CREDITS_URL = "https://openrouter.ai/api/v1/credits"
 
 
+def _now_utc() -> datetime:
+    """Return a timezone-aware UTC timestamp for secret-free discovery evidence."""
+    return datetime.now(timezone.utc)
+
+
 def _provider_discovery_error_code(exc: Exception) -> str:
     """Map provider failures to stable codes without retaining provider response text."""
     if isinstance(exc, urllib.error.HTTPError):
@@ -499,6 +505,18 @@ class DiscoveredModel:
     spend_admitted: bool = True
 
 
+@dataclass(frozen=True)
+class ExternalMetadataFetchEvidence:
+    """Secret-free evidence for one shared third-party metadata fetch."""
+
+    metadata_source_name: str
+    refresh_status: str
+    consumer_provider_count: int
+    error_code: str | None
+    started_at: datetime
+    finished_at: datetime
+
+
 class ProviderDiscoveryError(RuntimeError):
     """Raised when a provider's model list could not be fetched (network/auth failure)."""
 
@@ -600,13 +618,49 @@ def _fetch_models_dev_metadata(*, timeout: float) -> Any | None:
     bounded attempt has failed; a successful attempt returns immediately
     without spending the rest of the retry budget.
     """
+    metadata, _evidence = fetch_models_dev_metadata_with_evidence(
+        timeout=timeout,
+        consumer_provider_count=0,
+    )
+    return metadata
+
+
+def fetch_models_dev_metadata_with_evidence(
+    *,
+    timeout: float,
+    consumer_provider_count: int,
+) -> tuple[Any | None, ExternalMetadataFetchEvidence]:
+    """Fetch Models.dev once and return both payload and secret-free refresh evidence."""
+    started_at = _now_utc()
+    last_exc: Exception | None = None
     for attempt in range(_MODELS_DEV_FETCH_ATTEMPTS):
         try:
-            return _fetch_json(_MODELS_DEV_URL, timeout=timeout)
-        except (urllib.error.URLError, TimeoutError, ValueError, OSError):
+            payload = _fetch_json(_MODELS_DEV_URL, timeout=timeout)
+            return payload, ExternalMetadataFetchEvidence(
+                metadata_source_name="models_dev",
+                refresh_status="succeeded",
+                consumer_provider_count=consumer_provider_count,
+                error_code=None,
+                started_at=started_at,
+                finished_at=_now_utc(),
+            )
+        except (urllib.error.URLError, TimeoutError, ValueError, OSError) as exc:
+            last_exc = exc
             if attempt < _MODELS_DEV_FETCH_ATTEMPTS - 1:
                 time.sleep(_MODELS_DEV_FETCH_RETRY_DELAY_SECONDS)
-    return None
+    error_code = (
+        _provider_discovery_error_code(last_exc)
+        if last_exc is not None
+        else "unknown_error"
+    )
+    return None, ExternalMetadataFetchEvidence(
+        metadata_source_name="models_dev",
+        refresh_status="failed",
+        consumer_provider_count=consumer_provider_count,
+        error_code=error_code,
+        started_at=started_at,
+        finished_at=_now_utc(),
+    )
 
 
 def _fetch_configured_gateway_json(
@@ -1658,6 +1712,25 @@ def discover_all_models(
     timeout: float = DISCOVERY_TIMEOUT_SECONDS,
     ca_bundle: str | None = None,
 ) -> tuple[list[DiscoveredModel], list[ProviderDiscoveryError]]:
+    """Discover models across every provider with a registered credential."""
+    discovered, errors, _metadata_refreshes = discover_all_models_with_metadata_evidence(
+        sources,
+        timeout=timeout,
+        ca_bundle=ca_bundle,
+    )
+    return discovered, errors
+
+
+def discover_all_models_with_metadata_evidence(
+    sources: tuple[ProviderModelSource, ...] = PROVIDER_MODEL_SOURCES,
+    *,
+    timeout: float = DISCOVERY_TIMEOUT_SECONDS,
+    ca_bundle: str | None = None,
+) -> tuple[
+    list[DiscoveredModel],
+    list[ProviderDiscoveryError],
+    tuple[ExternalMetadataFetchEvidence, ...],
+]:
     """Discover models across every provider with a registered credential.
 
     One provider's failure never blocks the others: errors are collected and
@@ -1672,12 +1745,19 @@ def discover_all_models(
     """
     discovered: list[DiscoveredModel] = []
     errors: list[ProviderDiscoveryError] = []
+    metadata_refreshes: tuple[ExternalMetadataFetchEvidence, ...] = ()
     models_dev_metadata: Any = _NOT_FETCHED
-    if any(
-        source.models_dev_provider_id and get_credential(source.credential_name)
+    metadata_consumers = tuple(
+        source
         for source in sources
-    ):
-        models_dev_metadata = _fetch_models_dev_metadata(timeout=timeout)
+        if source.models_dev_provider_id and get_credential(source.credential_name)
+    )
+    if metadata_consumers:
+        models_dev_metadata, metadata_refresh = fetch_models_dev_metadata_with_evidence(
+            timeout=timeout,
+            consumer_provider_count=len(metadata_consumers),
+        )
+        metadata_refreshes = (metadata_refresh,)
     for source in sources:
         try:
             discovered.extend(
@@ -1713,7 +1793,7 @@ def discover_all_models(
             len(routed),
             len(errors),
         )
-    return routed, errors
+    return routed, errors, metadata_refreshes
 
 
 def openrouter_paid_inference_available(

contextual_orchestrator/provider_catalog_bootstrap.py

@@ -24,13 +24,14 @@
 )
 from .kv_config import InMemoryConfigStore
 from .model_discovery import (
+    ExternalMetadataFetchEvidence,
     PROVIDER_MODEL_SOURCES,
     DiscoveredModel,
     ProviderDiscoveryError,
     ProviderModelSource,
     apply_openrouter_spend_admission,
     agent_id_for,
-    discover_all_models,
+    discover_all_models_with_metadata_evidence,
     openrouter_paid_inference_available,
     refresh_price_book,
 )
@@ -50,6 +51,7 @@
 )
 from .provider_catalog_store import (
     CatalogRefreshEvidence,
+    ExternalMetadataRefreshEvidence,
     InMemoryProviderCatalogStore,
     PostgresProviderCatalogStore,
     ProviderCatalogStore,
@@ -146,6 +148,7 @@ class ProviderCatalogBootstrapReport:
     priced_model_count: int
     privacy_assessment_count: int
     catalog_refreshes: tuple[CatalogRefreshEvidence, ...]
+    external_metadata_refreshes: tuple[ExternalMetadataRefreshEvidence, ...]
 
     def as_dict(self) -> dict[str, object]:
         """Return the stable JSON evidence contract without secret values."""
@@ -177,6 +180,17 @@ def as_dict(self) -> dict[str, object]:
                 }
                 for evidence in self.catalog_refreshes
             ],
+            "external_metadata_refreshes": [
+                {
+                    "metadata_source_name": evidence.metadata_source_name,
+                    "refresh_status": evidence.refresh_status,
+                    "consumer_provider_count": evidence.consumer_provider_count,
+                    "error_code": evidence.error_code,
+                    "started_at": evidence.started_at.isoformat(),
+                    "finished_at": evidence.finished_at.isoformat(),
+                }
+                for evidence in self.external_metadata_refreshes
+            ],
         }
 
 
@@ -538,18 +552,37 @@ def bootstrap_provider_catalog_runtime(
     try:
         store = catalog_store or build_provider_catalog_store()
         source_tuple = tuple(sources)
-        discover = discovery or (
-            lambda requested_sources: discover_all_models(requested_sources)
-        )
-        live_models, errors = discover(source_tuple)
+        metadata_refreshes: tuple[ExternalMetadataFetchEvidence, ...] = ()
+        if discovery is None:
+            live_models, errors, metadata_refreshes = (
+                discover_all_models_with_metadata_evidence(source_tuple)
+            )
+        else:
+            live_models, errors = discovery(source_tuple)
         privacy_assessments: list[PrivacyPolicyAssessment] = []
         if analyze_privacy_policies:
             live_models, privacy_assessments = privacy_analysis(live_models)
         # The store evidence log is shared process state. Keep the offset,
         # refresh writes, and tail capture in one atomic boundary so concurrent
         # bootstrap reports cannot claim one another's provider attempts.
+        external_metadata_refreshes: tuple[ExternalMetadataRefreshEvidence, ...] = ()
         with _CATALOG_REFRESH_EVIDENCE_LOCK:
             evidence_offset = len(store.refresh_evidence())
+            if metadata_refreshes:
+                external_evidence_offset = len(
+                    store.external_metadata_refresh_evidence()
+                )
+                for metadata_refresh in metadata_refreshes:
+                    store.record_external_metadata_refresh(
+                        ExternalMetadataRefreshEvidence(
+                            metadata_source_name=metadata_refresh.metadata_source_name,
+                            refresh_status=metadata_refresh.refresh_status,
+                            consumer_provider_count=metadata_refresh.consumer_provider_count,
+                            error_code=metadata_refresh.error_code,
+                            started_at=metadata_refresh.started_at,
+                            finished_at=metadata_refresh.finished_at,
+                        )
+                    )
             snapshot = refresh_persisted_provider_catalog(
                 store,
                 sources=source_tuple,
@@ -558,6 +591,12 @@ def bootstrap_provider_catalog_runtime(
                 errors=errors,
             )
             catalog_refreshes = store.refresh_evidence()[evidence_offset:]
+            if metadata_refreshes:
+                external_metadata_refreshes = (
+                    store.external_metadata_refresh_evidence()[
+                        external_evidence_offset:
+                    ]
+                )
         assessments_by_account: dict[tuple[str, str], list[PrivacyPolicyAssessment]] = {}
         for assessment in privacy_assessments:
             assessments_by_account.setdefault(
@@ -657,6 +696,7 @@ def bootstrap_provider_catalog_runtime(
             priced_model_count=priced_count,
             privacy_assessment_count=privacy_assessment_count,
             catalog_refreshes=catalog_refreshes,
+            external_metadata_refreshes=external_metadata_refreshes,
         )
     except Exception:
         try: