From c7c389666c4a48068892ef735c88a6f252266d2f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Apr 2026 21:34:39 +0000 Subject: [PATCH 1/2] Add client-side stability metrics with OTel telemetry sent by default Move OpenTelemetry from optional to required dependency in cua-core so that operational telemetry (latency, errors, throughput) is collected out of the box. Add stability-specific metrics: API request tracking (success/error/latency), a configurable latency target threshold for SLO monitoring, and a churn rate metric inspired by customer-happiness models that flags requests as "unhappy" when they fail or exceed the latency target. Key changes: - cua-core: OTel packages are now required dependencies - New OTel instruments: api_requests_total, api_request_duration, api_errors_total, api_requests_exceeding_latency_target - StabilityTracker class for in-process success/error/churn computation - CloudAPIClient._request instrumented with request-level telemetry - Removed try/except ImportError guards in agent/computer OTel code - 16 new tests covering tracker math, metric recording, and config https://claude.ai/code/session_013SPeuLkNwmAWgRvcT1pzPB --- libs/python/agent/cua_agent/callbacks/otel.py | 43 +-- libs/python/computer/computer/computer.py | 27 +- libs/python/computer/computer/otel_wrapper.py | 46 +-- .../core/cua_core/telemetry/__init__.py | 9 + libs/python/core/cua_core/telemetry/otel.py | 249 +++++++++++++ libs/python/core/pyproject.toml | 18 +- .../core/tests/test_stability_metrics.py | 332 ++++++++++++++++++ libs/python/cua-cli/cua_cli/api/client.py | 49 ++- 8 files changed, 679 insertions(+), 94 deletions(-) create mode 100644 libs/python/core/tests/test_stability_metrics.py diff --git a/libs/python/agent/cua_agent/callbacks/otel.py b/libs/python/agent/cua_agent/callbacks/otel.py index 2a3e0e156d..52bf5aefbb 100644 --- a/libs/python/agent/cua_agent/callbacks/otel.py +++ b/libs/python/agent/cua_agent/callbacks/otel.py @@ -13,23 +13,14 @@ from .base import AsyncCallbackHandler -# Import OTEL functions - these are available when cua-core[telemetry] is installed -try: - from core.telemetry import ( - create_span, - is_otel_enabled, - record_error, - record_operation, - record_tokens, - track_concurrent, - ) - - OTEL_AVAILABLE = True -except ImportError: - OTEL_AVAILABLE = False - - def is_otel_enabled() -> bool: - return False +from cua_core.telemetry import ( + create_span, + is_otel_enabled, + record_error, + record_operation, + record_tokens, + track_concurrent, +) class OtelCallback(AsyncCallbackHandler): @@ -75,7 +66,7 @@ def _get_agent_type(self) -> str: async def on_run_start(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]]) -> None: """Called at the start of an agent run loop.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return self.run_start_time = time.perf_counter() @@ -89,7 +80,7 @@ async def on_run_end( new_items: List[Dict[str, Any]], ) -> None: """Called at the end of an agent run loop.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return if self.run_start_time is not None: @@ -108,7 +99,7 @@ async def on_run_end( async def on_responses(self, kwargs: Dict[str, Any], responses: Dict[str, Any]) -> None: """Called when responses are received (each step).""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return self.step_count += 1 @@ -130,7 +121,7 @@ async def on_responses(self, kwargs: Dict[str, Any], responses: Dict[str, Any]) async def on_usage(self, usage: Dict[str, Any]) -> None: """Called when usage information is received.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return prompt_tokens = usage.get("prompt_tokens", 0) @@ -145,14 +136,14 @@ async def on_usage(self, usage: Dict[str, Any]) -> None: async def on_computer_call_start(self, item: Dict[str, Any]) -> None: """Called when a computer call is about to start.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return async def on_computer_call_end( self, item: Dict[str, Any], result: List[Dict[str, Any]] ) -> None: """Called when a computer call has completed.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return action = item.get("action", {}) @@ -170,12 +161,12 @@ async def on_computer_call_end( async def on_api_start(self, kwargs: Dict[str, Any]) -> None: """Called when an LLM API call is about to start.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return async def on_api_end(self, kwargs: Dict[str, Any], result: Any) -> None: """Called when an LLM API call has completed.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return @@ -198,7 +189,7 @@ def __init__(self, agent: Any): async def on_error(self, error: Exception, context: Dict[str, Any]) -> None: """Called when an error occurs during agent execution.""" - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return error_type = type(error).__name__ diff --git a/libs/python/computer/computer/computer.py b/libs/python/computer/computer/computer.py index f73fa9d8d0..d944aaccc9 100644 --- a/libs/python/computer/computer/computer.py +++ b/libs/python/computer/computer/computer.py @@ -43,19 +43,11 @@ from .tracing_wrapper import TracingInterfaceWrapper # Import OTEL functions for session-level metrics -try: - from cua_core.telemetry import ( - is_otel_enabled, - record_operation, - track_concurrent, - ) - - OTEL_AVAILABLE = True -except ImportError: - OTEL_AVAILABLE = False - - def is_otel_enabled() -> bool: - return False +from cua_core.telemetry import ( + is_otel_enabled, + record_operation, + track_concurrent, +) SYSTEM_INFO = { @@ -681,7 +673,7 @@ async def run(self) -> Optional[str]: self.logger.info("Computer successfully initialized") # Record session start in OTEL - if OTEL_AVAILABLE and is_otel_enabled() and self._telemetry_enabled: + if is_otel_enabled() and self._telemetry_enabled: duration_seconds = time.time() - start_time record_operation( operation="computer.session.start", @@ -692,7 +684,7 @@ async def run(self) -> Optional[str]: ) except Exception as e: # Record failed session start - if OTEL_AVAILABLE and is_otel_enabled() and self._telemetry_enabled: + if is_otel_enabled() and self._telemetry_enabled: duration_seconds = time.time() - start_time record_operation( operation="computer.session.start", @@ -743,7 +735,7 @@ async def stop(self) -> None: self.logger.info("Computer stopped") # Record session stop in OTEL - if OTEL_AVAILABLE and is_otel_enabled() and self._telemetry_enabled: + if is_otel_enabled() and self._telemetry_enabled: duration_seconds = time.time() - start_time record_operation( operation="computer.session.stop", @@ -1057,8 +1049,7 @@ def interface(self): # Apply OTEL wrapper if enabled and telemetry is on if ( - OTEL_AVAILABLE - and is_otel_enabled() + is_otel_enabled() and self._telemetry_enabled and hasattr(self, "_original_interface") and self._original_interface is not None diff --git a/libs/python/computer/computer/otel_wrapper.py b/libs/python/computer/computer/otel_wrapper.py index e7731f7509..e16f23907f 100644 --- a/libs/python/computer/computer/otel_wrapper.py +++ b/libs/python/computer/computer/otel_wrapper.py @@ -12,21 +12,12 @@ from .interface.base import BaseComputerInterface -# Import OTEL functions - available when cua-core[telemetry] is installed -try: - from cua_core.telemetry import ( - create_span, - is_otel_enabled, - record_error, - record_operation, - ) - - OTEL_AVAILABLE = True -except ImportError: - OTEL_AVAILABLE = False - - def is_otel_enabled() -> bool: - return False +from cua_core.telemetry import ( + create_span, + is_otel_enabled, + record_error, + record_operation, +) # Actions that should be instrumented @@ -95,7 +86,7 @@ def __init__( """ self._original_interface = original_interface self._os_type = os_type - self._enabled = OTEL_AVAILABLE and is_otel_enabled() + self._enabled = is_otel_enabled() def __getattr__(self, name: str) -> Any: """ @@ -149,20 +140,19 @@ async def instrumented(*args: Any, **kwargs: Any) -> Any: duration = time.perf_counter() - start_time # Record operation metrics - if OTEL_AVAILABLE: - record_operation( + record_operation( + operation=f"computer.action.{name}", + duration_seconds=duration, + status=status, + os_type=self._os_type, + ) + + if error_type: + record_error( + error_type=error_type, operation=f"computer.action.{name}", - duration_seconds=duration, - status=status, - os_type=self._os_type, ) - if error_type: - record_error( - error_type=error_type, - operation=f"computer.action.{name}", - ) - return instrumented @@ -180,7 +170,7 @@ def wrap_interface_with_otel( Returns: The wrapped interface (or original if OTEL disabled) """ - if not OTEL_AVAILABLE or not is_otel_enabled(): + if not is_otel_enabled(): return interface return OtelInterfaceWrapper(interface, os_type) # type: ignore diff --git a/libs/python/core/cua_core/telemetry/__init__.py b/libs/python/core/cua_core/telemetry/__init__.py index f7e3044548..6c5172ffc8 100644 --- a/libs/python/core/cua_core/telemetry/__init__.py +++ b/libs/python/core/cua_core/telemetry/__init__.py @@ -6,10 +6,14 @@ # OpenTelemetry instrumentation for Four Golden Signals from cua_core.telemetry.otel import ( + StabilityTracker, create_span, + get_stability_tracker, instrument_async, instrument_sync, is_otel_enabled, + record_api_error, + record_api_request, record_error, record_operation, record_tokens, @@ -35,4 +39,9 @@ "create_span", "instrument_async", "instrument_sync", + # Stability metrics + "record_api_request", + "record_api_error", + "StabilityTracker", + "get_stability_tracker", ] diff --git a/libs/python/core/cua_core/telemetry/otel.py b/libs/python/core/cua_core/telemetry/otel.py index 2587bb458e..0dd8cfea69 100644 --- a/libs/python/core/cua_core/telemetry/otel.py +++ b/libs/python/core/cua_core/telemetry/otel.py @@ -5,6 +5,11 @@ - Traffic: Operation counters - Errors: Error counters - Saturation: Concurrent operation gauges + +Plus stability metrics: +- API request success/failure tracking +- Request latency histograms with SLO threshold tracking +- Customer churn rate (% of requests that failed or exceeded latency target) """ from __future__ import annotations @@ -26,6 +31,10 @@ # Default OTEL endpoint DEFAULT_OTEL_ENDPOINT = "https://otel.cua.ai" +# Default latency target in seconds — requests exceeding this are counted as +# "unhappy" for churn rate purposes. Override via CUA_LATENCY_TARGET_SECONDS. +DEFAULT_LATENCY_TARGET_SECONDS = 30.0 + # Lazy initialization state _initialized = False _init_failed = False @@ -44,6 +53,12 @@ _concurrent_operations: Optional[Any] = None # UpDownCounter _tokens_total: Optional[Any] = None # Counter +# Stability metrics (lazily initialized) +_api_requests_total: Optional[Any] = None # Counter +_api_request_duration: Optional[Any] = None # Histogram +_api_errors_total: Optional[Any] = None # Counter +_api_requests_exceeding_target: Optional[Any] = None # Counter + def is_otel_enabled() -> bool: """Check if OpenTelemetry is enabled. @@ -82,6 +97,14 @@ def _get_service_name() -> str: return os.environ.get("CUA_OTEL_SERVICE_NAME", "cua-sdk") +def _get_latency_target() -> float: + """Get the latency target in seconds for churn rate calculation.""" + try: + return float(os.environ.get("CUA_LATENCY_TARGET_SECONDS", str(DEFAULT_LATENCY_TARGET_SECONDS))) + except (ValueError, TypeError): + return DEFAULT_LATENCY_TARGET_SECONDS + + def _initialize_otel() -> bool: """Initialize OpenTelemetry components. @@ -91,6 +114,8 @@ def _initialize_otel() -> bool: global _initialized, _init_failed, _meter, _tracer, _meter_provider, _tracer_provider global _operation_duration, _operations_total, _errors_total global _concurrent_operations, _tokens_total + global _api_requests_total, _api_request_duration, _api_errors_total + global _api_requests_exceeding_target if _initialized: return True @@ -188,6 +213,31 @@ def _initialize_otel() -> bool: unit="1", ) + # --- Stability metrics --- + _api_requests_total = _meter.create_counter( + name="cua_sdk_api_requests_total", + description="Total API requests by endpoint and status", + unit="1", + ) + + _api_request_duration = _meter.create_histogram( + name="cua_sdk_api_request_duration_seconds", + description="API request latency in seconds", + unit="s", + ) + + _api_errors_total = _meter.create_counter( + name="cua_sdk_api_errors_total", + description="Total API request errors by type and endpoint", + unit="1", + ) + + _api_requests_exceeding_target = _meter.create_counter( + name="cua_sdk_api_requests_exceeding_latency_target", + description="API requests that exceeded the latency target (contributes to churn)", + unit="1", + ) + # Register shutdown handler atexit.register(_shutdown_otel) @@ -531,3 +581,202 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper # type: ignore return decorator + + +# --- Stability Metrics API --- + + +def record_api_request( + endpoint: str, + method: str, + status_code: int, + duration_seconds: float, + error_type: Optional[str] = None, + **extra_attributes: Any, +) -> None: + """Record an API request for stability tracking. + + Tracks success/failure, latency, and whether the request exceeded + the latency target (contributing to churn rate). + + Args: + endpoint: API endpoint path (e.g., "/v1/images") + method: HTTP method (e.g., "GET", "POST") + status_code: HTTP response status code (0 for connection failures) + duration_seconds: Request duration in seconds + error_type: Error class name if the request failed with an exception + **extra_attributes: Additional attributes to record + """ + if not _initialize_otel(): + return + + is_success = 200 <= status_code < 400 and error_type is None + status = "success" if is_success else "error" + + attributes: Dict[str, str] = { + "endpoint": endpoint, + "method": method, + "status_code": str(status_code), + "status": status, + } + for key, value in extra_attributes.items(): + if value is not None: + attributes[key] = str(value) + + try: + # Track request count + if _api_requests_total is not None: + _api_requests_total.add(1, attributes) + + # Track latency + if _api_request_duration is not None: + _api_request_duration.record(duration_seconds, attributes) + + # Track errors + if not is_success and _api_errors_total is not None: + error_attrs = {**attributes} + if error_type: + error_attrs["error_type"] = error_type + _api_errors_total.add(1, error_attrs) + + # Track latency target breaches (contributes to churn) + latency_target = _get_latency_target() + if duration_seconds > latency_target and _api_requests_exceeding_target is not None: + _api_requests_exceeding_target.add( + 1, + { + "endpoint": endpoint, + "method": method, + "latency_target_seconds": str(latency_target), + }, + ) + + except Exception as e: + logger.debug(f"Failed to record API request metric: {e}") + + +def record_api_error( + endpoint: str, + method: str, + error_type: str, + duration_seconds: float = 0.0, + **extra_attributes: Any, +) -> None: + """Record an API request that failed with an exception (no HTTP status). + + Use this for connection errors, timeouts, DNS failures, etc. where + no HTTP response was received. + + Args: + endpoint: API endpoint path + method: HTTP method + error_type: Exception class name (e.g., "ConnectionError", "TimeoutError") + duration_seconds: Time elapsed before the error + **extra_attributes: Additional attributes to record + """ + record_api_request( + endpoint=endpoint, + method=method, + status_code=0, + duration_seconds=duration_seconds, + error_type=error_type, + **extra_attributes, + ) + + +class StabilityTracker: + """In-process tracker that computes client-side stability scores. + + This supplements the OTel counters/histograms with a simple rolling view + that can be queried locally (e.g. for adaptive retry logic or health + checks). + + Thread-safe. + """ + + def __init__(self, latency_target: Optional[float] = None): + self._lock = Lock() + self._total_requests = 0 + self._failed_requests = 0 + self._slow_requests = 0 # exceeded latency target + self._latency_target = latency_target or _get_latency_target() + + def record( + self, + success: bool, + duration_seconds: float, + ) -> None: + """Record an API request outcome.""" + with self._lock: + self._total_requests += 1 + if not success: + self._failed_requests += 1 + if duration_seconds > self._latency_target: + self._slow_requests += 1 + + @property + def total_requests(self) -> int: + with self._lock: + return self._total_requests + + @property + def failed_requests(self) -> int: + with self._lock: + return self._failed_requests + + @property + def success_rate(self) -> float: + """Fraction of requests that succeeded (0.0 – 1.0).""" + with self._lock: + if self._total_requests == 0: + return 1.0 + return (self._total_requests - self._failed_requests) / self._total_requests + + @property + def error_rate(self) -> float: + """Fraction of requests that failed (0.0 – 1.0).""" + return 1.0 - self.success_rate + + @property + def churn_rate(self) -> float: + """Fraction of requests that were 'unhappy' — either failed or exceeded + the latency target (0.0 – 1.0). Inspired by customer-happiness models + in tycoon-style simulations. + """ + with self._lock: + if self._total_requests == 0: + return 0.0 + unhappy = self._failed_requests + self._slow_requests + # A request can be both failed *and* slow; cap at total. + return min(unhappy, self._total_requests) / self._total_requests + + @property + def stability_score(self) -> float: + """Overall stability score (0.0 – 1.0). + + ``1.0`` means all requests succeeded within the latency target. + ``0.0`` means every request was unhappy. + """ + return 1.0 - self.churn_rate + + def reset(self) -> None: + """Reset all counters (useful for windowed tracking).""" + with self._lock: + self._total_requests = 0 + self._failed_requests = 0 + self._slow_requests = 0 + + +# Global singleton tracker +_stability_tracker: Optional[StabilityTracker] = None +_tracker_lock = Lock() + + +def get_stability_tracker() -> StabilityTracker: + """Return the global :class:`StabilityTracker` instance, creating it if needed.""" + global _stability_tracker + if _stability_tracker is None: + with _tracker_lock: + if _stability_tracker is None: + _stability_tracker = StabilityTracker() + return _stability_tracker diff --git a/libs/python/core/pyproject.toml b/libs/python/core/pyproject.toml index da872114dd..b2988f6f1a 100644 --- a/libs/python/core/pyproject.toml +++ b/libs/python/core/pyproject.toml @@ -12,22 +12,16 @@ authors = [ ] dependencies = [ "posthog>=3.20.0", -] -requires-python = ">=3.11,<3.14" - -[project.optional-dependencies] -# OpenTelemetry for operational metrics (Four Golden Signals) -otel = [ - "opentelemetry-api>=1.20.0", - "opentelemetry-sdk>=1.20.0", - "opentelemetry-exporter-otlp-proto-http>=1.20.0", -] -# All telemetry features -telemetry = [ "opentelemetry-api>=1.20.0", "opentelemetry-sdk>=1.20.0", "opentelemetry-exporter-otlp-proto-http>=1.20.0", ] +requires-python = ">=3.11,<3.14" + +[project.optional-dependencies] +# Kept for backwards compatibility with existing `pip install cua-core[otel]` usage +otel = [] +telemetry = [] [tool.pdm] distribution = true diff --git a/libs/python/core/tests/test_stability_metrics.py b/libs/python/core/tests/test_stability_metrics.py new file mode 100644 index 0000000000..2fdca1fbc5 --- /dev/null +++ b/libs/python/core/tests/test_stability_metrics.py @@ -0,0 +1,332 @@ +"""Tests for stability metrics in the OTel telemetry module. + +Tests cover: +- StabilityTracker success/error/churn rate computations +- record_api_request / record_api_error telemetry recording +- Latency target threshold behaviour +""" + +import sys +import types +from unittest.mock import MagicMock, patch + +import pytest + +# The posthog module imports ``from core import __version__`` which relies on +# a namespace alias that may not be present in every environment. Provide a +# stub so that importing cua_core.telemetry doesn't blow up during tests. +_core_stub = types.ModuleType("core") +_core_stub.__version__ = "0.0.0-test" +sys.modules.setdefault("core", _core_stub) + + +class TestStabilityTracker: + """Unit tests for the in-process StabilityTracker.""" + + def _make_tracker(self, latency_target=30.0): + from cua_core.telemetry.otel import StabilityTracker + + return StabilityTracker(latency_target=latency_target) + + def test_initial_state(self): + tracker = self._make_tracker() + assert tracker.total_requests == 0 + assert tracker.failed_requests == 0 + assert tracker.success_rate == 1.0 + assert tracker.error_rate == 0.0 + assert tracker.churn_rate == 0.0 + assert tracker.stability_score == 1.0 + + def test_all_successes(self): + tracker = self._make_tracker() + for _ in range(10): + tracker.record(success=True, duration_seconds=1.0) + assert tracker.total_requests == 10 + assert tracker.failed_requests == 0 + assert tracker.success_rate == 1.0 + assert tracker.error_rate == 0.0 + assert tracker.churn_rate == 0.0 + assert tracker.stability_score == 1.0 + + def test_all_failures(self): + tracker = self._make_tracker() + for _ in range(5): + tracker.record(success=False, duration_seconds=1.0) + assert tracker.total_requests == 5 + assert tracker.failed_requests == 5 + assert tracker.success_rate == 0.0 + assert tracker.error_rate == 1.0 + + def test_mixed_success_failure(self): + tracker = self._make_tracker() + for _ in range(7): + tracker.record(success=True, duration_seconds=1.0) + for _ in range(3): + tracker.record(success=False, duration_seconds=1.0) + assert tracker.total_requests == 10 + assert tracker.success_rate == pytest.approx(0.7) + assert tracker.error_rate == pytest.approx(0.3) + + def test_slow_requests_increase_churn(self): + tracker = self._make_tracker(latency_target=5.0) + # 8 fast successes, 2 slow successes + for _ in range(8): + tracker.record(success=True, duration_seconds=1.0) + for _ in range(2): + tracker.record(success=True, duration_seconds=10.0) # exceeds 5s target + assert tracker.total_requests == 10 + assert tracker.failed_requests == 0 + assert tracker.success_rate == 1.0 + # Churn = slow / total = 2/10 + assert tracker.churn_rate == pytest.approx(0.2) + assert tracker.stability_score == pytest.approx(0.8) + + def test_churn_combines_failures_and_slow(self): + tracker = self._make_tracker(latency_target=5.0) + # 6 fast successes, 2 slow successes, 2 fast failures + for _ in range(6): + tracker.record(success=True, duration_seconds=1.0) + for _ in range(2): + tracker.record(success=True, duration_seconds=10.0) + for _ in range(2): + tracker.record(success=False, duration_seconds=1.0) + # unhappy = 2 failed + 2 slow = 4 / 10 + assert tracker.churn_rate == pytest.approx(0.4) + assert tracker.stability_score == pytest.approx(0.6) + + def test_churn_capped_at_1(self): + """A request that is both failed AND slow should not double-count beyond 1.0.""" + tracker = self._make_tracker(latency_target=5.0) + # All requests are both failed and slow + for _ in range(5): + tracker.record(success=False, duration_seconds=10.0) + # unhappy = 5 failed + 5 slow = 10, but capped at total=5 + assert tracker.churn_rate == pytest.approx(1.0) + assert tracker.stability_score == pytest.approx(0.0) + + def test_reset(self): + tracker = self._make_tracker() + tracker.record(success=True, duration_seconds=1.0) + tracker.record(success=False, duration_seconds=1.0) + assert tracker.total_requests == 2 + tracker.reset() + assert tracker.total_requests == 0 + assert tracker.failed_requests == 0 + assert tracker.success_rate == 1.0 + assert tracker.churn_rate == 0.0 + + +class TestGetStabilityTracker: + """Test the singleton get_stability_tracker function.""" + + def test_returns_singleton(self): + import cua_core.telemetry.otel as otel_mod + + # Reset global state + otel_mod._stability_tracker = None + + from cua_core.telemetry import get_stability_tracker + + t1 = get_stability_tracker() + t2 = get_stability_tracker() + assert t1 is t2 + + # Clean up + otel_mod._stability_tracker = None + + +class TestRecordApiRequest: + """Test record_api_request sends correct OTel metrics.""" + + def test_successful_request_records_metrics(self, monkeypatch): + """Verify a successful request records count + latency via OTel.""" + monkeypatch.setenv("CUA_TELEMETRY_ENABLED", "true") + + import cua_core.telemetry.otel as otel_mod + + # Mock the metric instruments + mock_counter = MagicMock() + mock_histogram = MagicMock() + mock_error_counter = MagicMock() + mock_exceed_counter = MagicMock() + + otel_mod._api_requests_total = mock_counter + otel_mod._api_request_duration = mock_histogram + otel_mod._api_errors_total = mock_error_counter + otel_mod._api_requests_exceeding_target = mock_exceed_counter + # Pretend already initialized + otel_mod._initialized = True + + from cua_core.telemetry import record_api_request + + record_api_request( + endpoint="/v1/images", + method="GET", + status_code=200, + duration_seconds=0.5, + ) + + # Should record request count + mock_counter.add.assert_called_once() + attrs = mock_counter.add.call_args[0][1] + assert attrs["status"] == "success" + assert attrs["endpoint"] == "/v1/images" + + # Should record latency + mock_histogram.record.assert_called_once() + assert mock_histogram.record.call_args[0][0] == 0.5 + + # Should NOT record error + mock_error_counter.add.assert_not_called() + + # Should NOT record latency breach (0.5s < 30s default) + mock_exceed_counter.add.assert_not_called() + + # Clean up + otel_mod._initialized = False + otel_mod._api_requests_total = None + otel_mod._api_request_duration = None + otel_mod._api_errors_total = None + otel_mod._api_requests_exceeding_target = None + + def test_error_request_records_error_counter(self, monkeypatch): + """Verify a 500 response records an error metric.""" + monkeypatch.setenv("CUA_TELEMETRY_ENABLED", "true") + + import cua_core.telemetry.otel as otel_mod + + mock_counter = MagicMock() + mock_histogram = MagicMock() + mock_error_counter = MagicMock() + mock_exceed_counter = MagicMock() + + otel_mod._api_requests_total = mock_counter + otel_mod._api_request_duration = mock_histogram + otel_mod._api_errors_total = mock_error_counter + otel_mod._api_requests_exceeding_target = mock_exceed_counter + otel_mod._initialized = True + + from cua_core.telemetry import record_api_request + + record_api_request( + endpoint="/v1/images", + method="POST", + status_code=500, + duration_seconds=1.2, + ) + + # Should record error + mock_error_counter.add.assert_called_once() + error_attrs = mock_error_counter.add.call_args[0][1] + assert error_attrs["status"] == "error" + + # Clean up + otel_mod._initialized = False + otel_mod._api_requests_total = None + otel_mod._api_request_duration = None + otel_mod._api_errors_total = None + otel_mod._api_requests_exceeding_target = None + + def test_slow_request_records_latency_breach(self, monkeypatch): + """Verify a slow request records a latency target breach.""" + monkeypatch.setenv("CUA_TELEMETRY_ENABLED", "true") + monkeypatch.setenv("CUA_LATENCY_TARGET_SECONDS", "2.0") + + import cua_core.telemetry.otel as otel_mod + + mock_counter = MagicMock() + mock_histogram = MagicMock() + mock_error_counter = MagicMock() + mock_exceed_counter = MagicMock() + + otel_mod._api_requests_total = mock_counter + otel_mod._api_request_duration = mock_histogram + otel_mod._api_errors_total = mock_error_counter + otel_mod._api_requests_exceeding_target = mock_exceed_counter + otel_mod._initialized = True + + from cua_core.telemetry import record_api_request + + record_api_request( + endpoint="/v1/images", + method="GET", + status_code=200, + duration_seconds=5.0, # exceeds 2s target + ) + + # Should record latency breach + mock_exceed_counter.add.assert_called_once() + + # Clean up + otel_mod._initialized = False + otel_mod._api_requests_total = None + otel_mod._api_request_duration = None + otel_mod._api_errors_total = None + otel_mod._api_requests_exceeding_target = None + + +class TestRecordApiError: + """Test record_api_error for connection-level failures.""" + + def test_connection_error_records_with_status_code_zero(self, monkeypatch): + monkeypatch.setenv("CUA_TELEMETRY_ENABLED", "true") + + import cua_core.telemetry.otel as otel_mod + + mock_counter = MagicMock() + mock_histogram = MagicMock() + mock_error_counter = MagicMock() + mock_exceed_counter = MagicMock() + + otel_mod._api_requests_total = mock_counter + otel_mod._api_request_duration = mock_histogram + otel_mod._api_errors_total = mock_error_counter + otel_mod._api_requests_exceeding_target = mock_exceed_counter + otel_mod._initialized = True + + from cua_core.telemetry import record_api_error + + record_api_error( + endpoint="/v1/images", + method="GET", + error_type="ConnectionError", + duration_seconds=0.1, + ) + + # Should record with status_code=0 + attrs = mock_counter.add.call_args[0][1] + assert attrs["status_code"] == "0" + assert attrs["status"] == "error" + + # Should record error with error_type + error_attrs = mock_error_counter.add.call_args[0][1] + assert error_attrs["error_type"] == "ConnectionError" + + # Clean up + otel_mod._initialized = False + otel_mod._api_requests_total = None + otel_mod._api_request_duration = None + otel_mod._api_errors_total = None + otel_mod._api_requests_exceeding_target = None + + +class TestLatencyTarget: + """Test latency target configuration.""" + + def test_default_target(self, monkeypatch): + monkeypatch.delenv("CUA_LATENCY_TARGET_SECONDS", raising=False) + from cua_core.telemetry.otel import _get_latency_target + + assert _get_latency_target() == 30.0 + + def test_custom_target(self, monkeypatch): + monkeypatch.setenv("CUA_LATENCY_TARGET_SECONDS", "10.0") + from cua_core.telemetry.otel import _get_latency_target + + assert _get_latency_target() == 10.0 + + def test_invalid_target_falls_back_to_default(self, monkeypatch): + monkeypatch.setenv("CUA_LATENCY_TARGET_SECONDS", "not_a_number") + from cua_core.telemetry.otel import _get_latency_target + + assert _get_latency_target() == 30.0 diff --git a/libs/python/cua-cli/cua_cli/api/client.py b/libs/python/cua-cli/cua_cli/api/client.py index b1faf4d014..a6d7670c54 100644 --- a/libs/python/cua-cli/cua_cli/api/client.py +++ b/libs/python/cua-cli/cua_cli/api/client.py @@ -2,12 +2,14 @@ import hashlib import os +import time from pathlib import Path from typing import Any, Optional from urllib.parse import quote import aiohttp from cua_core.http import cua_version_headers +from cua_core.telemetry import get_stability_tracker, record_api_error, record_api_request from cua_cli.auth.store import require_api_key DEFAULT_API_BASE = "https://api.cua.ai" @@ -50,16 +52,43 @@ async def _request( if json is not None: headers["Content-Type"] = "application/json" - async with aiohttp.ClientSession() as session: - timeout_obj = aiohttp.ClientTimeout(total=timeout) - async with session.request( - method, url, headers=headers, json=json, timeout=timeout_obj - ) as resp: - try: - data = await resp.json(content_type=None) - except Exception: - data = await resp.text() - return resp.status, data + tracker = get_stability_tracker() + start_time = time.perf_counter() + status_code = 0 + + try: + async with aiohttp.ClientSession() as session: + timeout_obj = aiohttp.ClientTimeout(total=timeout) + async with session.request( + method, url, headers=headers, json=json, timeout=timeout_obj + ) as resp: + status_code = resp.status + try: + data = await resp.json(content_type=None) + except Exception: + data = await resp.text() + + duration = time.perf_counter() - start_time + is_success = 200 <= status_code < 400 + record_api_request( + endpoint=path, + method=method, + status_code=status_code, + duration_seconds=duration, + ) + tracker.record(success=is_success, duration_seconds=duration) + + return status_code, data + except Exception as e: + duration = time.perf_counter() - start_time + record_api_error( + endpoint=path, + method=method, + error_type=type(e).__name__, + duration_seconds=duration, + ) + tracker.record(success=False, duration_seconds=duration) + raise # Image API methods From a4246db5b2ac43469fe6840a71d5e09e98aae2d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Apr 2026 21:35:11 +0000 Subject: [PATCH 2/2] Update uv.lock for new OTel required dependencies https://claude.ai/code/session_013SPeuLkNwmAWgRvcT1pzPB --- uv.lock | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/uv.lock b/uv.lock index e4d3a00e20..9a89e1bb85 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12, <3.14" resolution-markers = [ "python_full_version >= '3.13' and sys_platform == 'darwin'", @@ -1057,7 +1057,7 @@ wheels = [ [[package]] name = "cua-agent" -version = "0.7.39" +version = "0.8.1" source = { editable = "libs/python/agent" } dependencies = [ { name = "aiohttp" }, @@ -1269,7 +1269,7 @@ requires-dist = [ [[package]] name = "cua-computer" -version = "0.5.17" +version = "0.5.18" source = { editable = "libs/python/computer" } dependencies = [ { name = "aiohttp" }, @@ -1313,7 +1313,7 @@ provides-extras = ["lume", "lumier", "ui", "all"] [[package]] name = "cua-computer-server" -version = "0.3.28" +version = "0.3.33" source = { editable = "libs/python/computer-server" } dependencies = [ { name = "aiohttp" }, @@ -1389,22 +1389,13 @@ provides-extras = ["macos", "linux", "windows", "vnc"] [[package]] name = "cua-core" -version = "0.1.19" +version = "0.3.0" source = { editable = "libs/python/core" } dependencies = [ - { name = "posthog" }, -] - -[package.optional-dependencies] -otel = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, -] -telemetry = [ { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, + { name = "posthog" }, ] [package.dev-dependencies] @@ -1414,12 +1405,9 @@ dev = [ [package.metadata] requires-dist = [ - { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.20.0" }, - { name = "opentelemetry-api", marker = "extra == 'telemetry'", specifier = ">=1.20.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.20.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'telemetry'", specifier = ">=1.20.0" }, - { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.20.0" }, - { name = "opentelemetry-sdk", marker = "extra == 'telemetry'", specifier = ">=1.20.0" }, + { name = "opentelemetry-api", specifier = ">=1.20.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.20.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.20.0" }, { name = "posthog", specifier = ">=3.20.0" }, ] provides-extras = ["otel", "telemetry"]