diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 542bff1f60d6..1a0f100075b9 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -148,7 +148,7 @@ "bippy": "0.5.43", "concurrently": "10.0.4", "cross-env": "10.1.0", - "electron": "41.10.3", + "electron": "40.10.2", "electron-builder": "26.15.3", "esbuild": "0.28.1", "jsdom": "29.1.1", diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index 0d14abe519a9..860d916e2172 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -55,8 +55,9 @@ dependency does not change the collection or privacy policy. ## Current Slices -The current vertical slices record logical model calls, top-level task runs, -tool and approval outcomes, and skill lifecycle and reuse: +The current vertical slices record pseudonymous profile activity, logical +model calls, top-level task runs, tool and approval outcomes, and skill +lifecycle and reuse: ```text Hermes turn, API, tool, and approval hooks @@ -79,6 +80,15 @@ New calls use `hermes.model_route.count`. The previous `hermes.model_call.count` contract remains readable only so pending local counters created by older builds can be exported without losing data. +The first consented session start emits an empty `hermes.client.active` Relay +mark. The profile-scoped subscriber creates a random UUID install identity and +uses a transactional compare-and-set to record at most one client-active +counter in any rolling 24-hour window. The metric has no dimensions; Hermes +version, OS family, architecture, and install method remain bounded package +resources. Concurrent Hermes processes share the SQLite latch, so simultaneous +starts cannot double-count one install. A later session or task can attempt the +mark again, but the subscriber suppresses it until the rolling window expires. + Each task run is a Relay `Function` scope named `hermes.task_run`, parented to the owning Hermes session. The start counter contains only bounded execution surface and entrypoint values. The terminal counter contains bounded outcome, @@ -126,9 +136,13 @@ $HERMES_HOME/telemetry/shared_metrics/outbox/*.json The database keeps transactional aggregate and package-outbox state. Package files are immutable delta documents that conform to a closed JSON schema and -are written with atomic replacement. Fully packaged aggregate rows and -successfully exported package rows and files are retained locally for 30 days. -Pending package rows and counters with unexported deltas are never pruned. +are written with atomic replacement. Each package records the Hermes version, +OS family, architecture, and install method as bounded client resources. +Unrecognized platform or installation values are exported as `unknown`; raw +platform strings, hostnames, and paths are never included. Fully packaged +aggregate rows and successfully exported package rows and files are retained +locally for 30 days. Pending package rows and counters with unexported deltas +are never pruned. Package schema v1 remains unchanged for existing outbox files. New packages use v2, which accepts both the retired model-call contract and the current model-route contract so upgrades can drain pending counters safely. @@ -146,6 +160,13 @@ the persistent local identifier by default. It requires a separate product and privacy decision covering consent, identity scope, rotation or keyed pseudonymization, reset behavior, retention, and deletion. +The install identity is scoped to one `HERMES_HOME`. To reset it, stop Hermes +processes and remove `$HERMES_HOME/telemetry/shared_metrics`. This deliberately +removes the old identity, aggregate database, and queued local packages +together; the next consented session creates a new identity. Disabling shared +metrics stops new collection but does not silently delete previously collected +local state. + ## Smoke Test Run a real Hermes CLI turn against the deterministic local model server: @@ -162,6 +183,6 @@ The smoke has the local model request a real `read_file` tool call before its final response, then drives create, load, reuse, patch, edit, stale, archive, restore, and install skill transitions through the installed Relay binding. It verifies model, provider, task, tool, and skill counters in SQLite, validates -all exported delta packages against the closed schema, and checks that prompt, -response, tool-call ID, tool-result, and skill-name canaries are absent from the -packages. +all exported delta packages against the closed schema, verifies the +pseudonymous client-active counter, and checks that prompt, response, tool-call +ID, tool-result, and skill-name canaries are absent from the packages. diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index e1d8c43ca594..756c6caa8836 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -16,6 +16,7 @@ from .shared_metrics import SharedMetricsStore from .shared_metrics_contract import ( + CLIENT_ACTIVE_MARK, MODEL_CALL_PROFILE_MODEL, MODEL_CALL_SCOPE, SCHEMA_KEY, @@ -166,6 +167,26 @@ def ensure_session(self, event: dict[str, Any]) -> _MetricsSession | None: return None return session + def record_client_active(self, event: dict[str, Any]) -> None: + """Emit one payload-free activation attempt under the session scope.""" + session = self.ensure_session(event) + if session is None: + return + self._emit_client_active(session) + + def _emit_client_active(self, session: _MetricsSession) -> None: + with session.lock: + if session.closing: + return + self._run_in_session( + session, + self.relay.scope.event, + CLIENT_ACTIVE_MARK, + handle=session.relay_session.handle, + data={}, + metadata=self._event_metadata(), + ) + def _run_in_session( self, session: _MetricsSession, @@ -210,6 +231,7 @@ def start_task(self, event: dict[str, Any]) -> _TaskRun | None: or session.relay_session.context is None ): return None + self._emit_client_active(session) task_context = session.relay_session.context.copy() start_fields = task_start_fields(event) active_turn = relay_runtime.active_turn(session.session_id) @@ -1089,7 +1111,7 @@ def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: return try: if hook_name == "on_session_start": - runtime.ensure_session(kwargs) + runtime.record_client_active(kwargs) elif hook_name == "pre_llm_call": runtime.start_task(kwargs) elif hook_name == "pre_api_request": diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json index 925622885348..5d0bea610a7b 100644 --- a/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json +++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json @@ -41,13 +41,28 @@ "type": "object", "additionalProperties": false, "required": [ - "hermes_version" + "architecture", + "hermes_version", + "install_method", + "os_family" ], "properties": { + "architecture": { + "type": "string", + "enum": ["arm", "arm64", "unknown", "x86", "x86_64"] + }, "hermes_version": { "type": "string", "minLength": 1, "maxLength": 64 + }, + "install_method": { + "type": "string", + "enum": ["docker", "git", "homebrew", "nixos", "pip", "unknown"] + }, + "os_family": { + "type": "string", + "enum": ["linux", "macos", "unknown", "windows"] } } }, @@ -56,6 +71,9 @@ "minItems": 1, "items": { "oneOf": [ + { + "$ref": "#/$defs/client_active_counter" + }, { "$ref": "#/$defs/model_call_counter" }, @@ -85,6 +103,32 @@ } }, "$defs": { + "client_active_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.client.active" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "maxProperties": 0 + }, + "value": { + "const": 1 + } + } + }, "uuid": { "type": "string", "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py index 186924576d98..fd42b0623065 100644 --- a/hermes_cli/observability/shared_metrics.py +++ b/hermes_cli/observability/shared_metrics.py @@ -17,17 +17,21 @@ from utils import atomic_json_write from .shared_metrics_contract import ( + CLIENT_ACTIVE_METRIC, COUNTER_METRICS, MODEL_ROUTE_METRIC, + client_resource_is_valid, counter_dimensions_are_valid, ) _PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v2" -_STORE_SCHEMA_VERSION = "1" +_STORE_SCHEMA_VERSION = "2" _BUSY_TIMEOUT_MS = 250 _SCHEMA_BUSY_TIMEOUT_MS = 5_000 _LOCAL_HISTORY_RETENTION_DAYS = 30 +_ACTIVE_INSTALL_STATE_KEY = "client_active_recorded_at" +_ACTIVE_INSTALL_INTERVAL = timedelta(hours=24) logger = logging.getLogger(__name__) @@ -59,54 +63,138 @@ def __init__( def record_model_call( self, dimensions: dict[str, str], - hermes_version: str, + resource: dict[str, str], ) -> None: """Increment the terminal model-call counter for the current UTC day.""" - self.record_counter(MODEL_ROUTE_METRIC, dimensions, hermes_version) + self.record_counter(MODEL_ROUTE_METRIC, dimensions, resource) + + def record_client_active(self, resource: dict[str, str]) -> bool: + """Record this install at most once in any rolling 24-hour window.""" + dimensions: dict[str, str] = {} + self._validate_counter(CLIENT_ACTIVE_METRIC, dimensions, resource) + now = _utc_now() + with self._connection() as connection: + with write_txn(connection): + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", + (_ACTIVE_INSTALL_STATE_KEY,), + ).fetchone() + if row is not None: + last_recorded = self._parse_state_timestamp(row["value"]) + if last_recorded is not None and last_recorded > now: + # A wall-clock correction must not suppress activity until + # the stale future timestamp plus another full interval. + connection.execute( + """ + UPDATE telemetry_state + SET value = ? + WHERE key = ? + """, + (_isoformat(now), _ACTIVE_INSTALL_STATE_KEY), + ) + return False + if ( + last_recorded is not None + and now < last_recorded + _ACTIVE_INSTALL_INTERVAL + ): + return False + + self._install_id(connection) + self._record_counter_in_transaction( + connection, + CLIENT_ACTIVE_METRIC, + dimensions, + resource, + period_start=now.date().isoformat(), + ) + connection.execute( + """ + INSERT INTO telemetry_state(key, value) + VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (_ACTIVE_INSTALL_STATE_KEY, _isoformat(now)), + ) + return True def record_counter( self, metric_name: str, dimensions: dict[str, str], - hermes_version: str, + resource: dict[str, str], ) -> None: """Increment one allowlisted counter for the current UTC day.""" + self._validate_counter(metric_name, dimensions, resource) + with self._connection() as connection: + self._record_counter_in_transaction( + connection, + metric_name, + dimensions, + resource, + period_start=_utc_now().date().isoformat(), + ) + + @staticmethod + def _validate_counter( + metric_name: str, + dimensions: dict[str, str], + resource: dict[str, str], + ) -> None: if metric_name not in COUNTER_METRICS: raise ValueError(f"Unsupported shared metric: {metric_name}") if not counter_dimensions_are_valid(metric_name, dimensions): raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}") + if not client_resource_is_valid(resource): + raise ValueError("Unsupported shared-metrics client resource") + + @staticmethod + def _record_counter_in_transaction( + connection: sqlite3.Connection, + metric_name: str, + dimensions: dict[str, str], + resource: dict[str, str], + *, + period_start: str, + ) -> None: dimensions_json = json.dumps( dimensions, sort_keys=True, separators=(",", ":"), ) - period_start = _utc_now().date().isoformat() - with self._connection() as connection: - connection.execute( - """ - INSERT INTO counter_aggregates( - period_start, - metric_name, - hermes_version, - dimensions_json, - value, - packaged_value - ) VALUES (?, ?, ?, ?, 1, 0) - ON CONFLICT( - period_start, - metric_name, - hermes_version, - dimensions_json - ) - DO UPDATE SET value = value + 1 - """, - ( - period_start, - metric_name, - hermes_version or "unknown", - dimensions_json, - ), + connection.execute( + """ + INSERT INTO counter_aggregates( + period_start, + metric_name, + hermes_version, + os_family, + architecture, + install_method, + dimensions_json, + value, + packaged_value + ) VALUES (?, ?, ?, ?, ?, ?, ?, 1, 0) + ON CONFLICT( + period_start, + metric_name, + hermes_version, + os_family, + architecture, + install_method, + dimensions_json ) + DO UPDATE SET value = value + 1 + """, + ( + period_start, + metric_name, + resource["hermes_version"], + resource["os_family"], + resource["architecture"], + resource["install_method"], + dimensions_json, + ), + ) def create_and_export_package(self) -> list[Path]: """Commit one pending delta package, then atomically export the outbox.""" @@ -141,18 +229,33 @@ def counter_snapshot(self) -> list[dict[str, Any]]: period_start, metric_name, hermes_version, + os_family, + architecture, + install_method, dimensions_json, value, packaged_value FROM counter_aggregates - ORDER BY period_start, hermes_version, metric_name, dimensions_json + ORDER BY + period_start, + hermes_version, + os_family, + architecture, + install_method, + metric_name, + dimensions_json """ ).fetchall() return [ { "period_start": row["period_start"], "metric_name": row["metric_name"], - "hermes_version": row["hermes_version"], + "resource": { + "hermes_version": row["hermes_version"], + "os_family": row["os_family"], + "architecture": row["architecture"], + "install_method": row["install_method"], + }, "dimensions": json.loads(row["dimensions_json"]), "value": row["value"], "packaged_value": row["packaged_value"], @@ -213,17 +316,47 @@ def _ensure_schema_in_transaction(connection: sqlite3.Connection) -> None: schema_row = connection.execute( "SELECT value FROM telemetry_state WHERE key = 'schema_version'" ).fetchone() - if schema_row is not None and str(schema_row["value"]) != _STORE_SCHEMA_VERSION: + schema_version = str(schema_row["value"]) if schema_row is not None else None + if schema_version == "1": + SharedMetricsStore._migrate_v1_counter_aggregates(connection) + schema_version = _STORE_SCHEMA_VERSION + if schema_version is not None and schema_version != _STORE_SCHEMA_VERSION: raise RuntimeError( - "Unsupported shared-metrics store schema version: " - f"{schema_row['value']}" + f"Unsupported shared-metrics store schema version: {schema_version}" ) + SharedMetricsStore._create_counter_aggregates_table(connection) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS package_outbox ( + package_id TEXT PRIMARY KEY, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL, + exported_at TEXT + ) + """ + ) + connection.execute( + """ + INSERT INTO telemetry_state(key, value) + VALUES ('schema_version', ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (_STORE_SCHEMA_VERSION,), + ) + + @staticmethod + def _create_counter_aggregates_table(connection: sqlite3.Connection) -> None: connection.execute( """ CREATE TABLE IF NOT EXISTS counter_aggregates ( period_start TEXT NOT NULL, metric_name TEXT NOT NULL, hermes_version TEXT NOT NULL, + os_family TEXT NOT NULL, + architecture TEXT NOT NULL, + install_method TEXT NOT NULL, dimensions_json TEXT NOT NULL, value INTEGER NOT NULL, packaged_value INTEGER NOT NULL DEFAULT 0, @@ -231,30 +364,48 @@ def _ensure_schema_in_transaction(connection: sqlite3.Connection) -> None: period_start, metric_name, hermes_version, + os_family, + architecture, + install_method, dimensions_json ) ) """ ) + + @staticmethod + def _migrate_v1_counter_aggregates(connection: sqlite3.Connection) -> None: connection.execute( - """ - CREATE TABLE IF NOT EXISTS package_outbox ( - package_id TEXT PRIMARY KEY, - period_start TEXT NOT NULL, - period_end TEXT NOT NULL, - payload_json TEXT NOT NULL, - created_at TEXT NOT NULL, - exported_at TEXT - ) - """ + "ALTER TABLE counter_aggregates RENAME TO counter_aggregates_v1" ) + SharedMetricsStore._create_counter_aggregates_table(connection) connection.execute( """ - INSERT OR IGNORE INTO telemetry_state(key, value) - VALUES ('schema_version', ?) - """, - (_STORE_SCHEMA_VERSION,), + INSERT INTO counter_aggregates( + period_start, + metric_name, + hermes_version, + os_family, + architecture, + install_method, + dimensions_json, + value, + packaged_value + ) + SELECT + period_start, + metric_name, + hermes_version, + 'unknown', + 'unknown', + 'unknown', + dimensions_json, + value, + packaged_value + FROM counter_aggregates_v1 + """ ) + connection.execute("DROP TABLE counter_aggregates_v1") def _install_id(self, connection: sqlite3.Connection) -> str: row = connection.execute( @@ -274,16 +425,36 @@ def _install_id(self, connection: sqlite3.Connection) -> str: raise RuntimeError("Unable to create the shared-metrics install identity") return str(row["value"]) + @staticmethod + def _parse_state_timestamp(value: Any) -> datetime | None: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + if parsed.tzinfo is None: + return None + return parsed.astimezone(timezone.utc) + def _pending_period_count(self) -> int: with self._connection() as connection: row = connection.execute( """ SELECT COUNT(*) AS period_count FROM ( - SELECT period_start, hermes_version + SELECT + period_start, + hermes_version, + os_family, + architecture, + install_method FROM counter_aggregates WHERE value > packaged_value - GROUP BY period_start, hermes_version + GROUP BY + period_start, + hermes_version, + os_family, + architecture, + install_method ) """ ).fetchone() @@ -322,10 +493,20 @@ def _create_package_in_transaction( ) -> dict[str, Any] | None: period_row = connection.execute( """ - SELECT period_start, hermes_version + SELECT + period_start, + hermes_version, + os_family, + architecture, + install_method FROM counter_aggregates WHERE value > packaged_value - ORDER BY period_start, hermes_version + ORDER BY + period_start, + hermes_version, + os_family, + architecture, + install_method LIMIT 1 """ ).fetchone() @@ -339,16 +520,33 @@ def _create_package_in_transaction( FROM counter_aggregates WHERE period_start = ? AND hermes_version = ? + AND os_family = ? + AND architecture = ? + AND install_method = ? AND value > packaged_value ORDER BY metric_name, dimensions_json """, - (period_value, period_row["hermes_version"]), + ( + period_value, + period_row["hermes_version"], + period_row["os_family"], + period_row["architecture"], + period_row["install_method"], + ), ).fetchall() period_start = datetime.fromisoformat(str(period_value)).replace( tzinfo=timezone.utc ) period_end = period_start + timedelta(days=1) package_id = str(uuid.uuid4()) + resource = { + "hermes_version": period_row["hermes_version"], + "os_family": period_row["os_family"], + "architecture": period_row["architecture"], + "install_method": period_row["install_method"], + } + if not client_resource_is_valid(resource): + raise ValueError("Unsupported shared-metrics client resource") payload = { "schema_version": _PACKAGE_SCHEMA_VERSION, "package_id": package_id, @@ -356,7 +554,7 @@ def _create_package_in_transaction( "period_start": _isoformat(period_start), "period_end": _isoformat(period_end), "generated_at": _isoformat(now), - "resource": {"hermes_version": period_row["hermes_version"]}, + "resource": resource, "metrics": [self._package_metric(row) for row in rows], } payload_json = json.dumps( @@ -390,12 +588,18 @@ def _create_package_in_transaction( WHERE period_start = ? AND metric_name = ? AND hermes_version = ? + AND os_family = ? + AND architecture = ? + AND install_method = ? AND dimensions_json = ? """, ( period_value, row["metric_name"], period_row["hermes_version"], + period_row["os_family"], + period_row["architecture"], + period_row["install_method"], row["dimensions_json"], ), ) diff --git a/hermes_cli/observability/shared_metrics_contract.py b/hermes_cli/observability/shared_metrics_contract.py index 0d8d3bbbb4eb..b7637c9317c5 100644 --- a/hermes_cli/observability/shared_metrics_contract.py +++ b/hermes_cli/observability/shared_metrics_contract.py @@ -18,10 +18,12 @@ MODEL_CALL_PROFILE_MODEL = "unknown" TASK_SCOPE = "hermes.task_run" TOOL_CALL_SCOPE = "hermes.tool_call" +CLIENT_ACTIVE_MARK = "hermes.client.active" TOOL_APPROVAL_MARK = "hermes.tool_approval" SKILL_LIFECYCLE_MARK = "hermes.skill.lifecycle" SKILL_LOAD_MARK = "hermes.skill.load" SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics" +CLIENT_ACTIVE_METRIC = "hermes.client.active" LEGACY_MODEL_CALL_METRIC = "hermes.model_call.count" MODEL_ROUTE_METRIC = "hermes.model_route.count" TASK_STARTED_METRIC = "hermes.task_run.started" @@ -178,6 +180,99 @@ "not_applicable", "reused_after_patch", }) +CLIENT_OS_FAMILIES: frozenset[str] = frozenset({ + "linux", + "macos", + "unknown", + "windows", +}) +CLIENT_ARCHITECTURES: frozenset[str] = frozenset({ + "arm", + "arm64", + "unknown", + "x86", + "x86_64", +}) +CLIENT_INSTALL_METHODS: frozenset[str] = frozenset({ + "docker", + "git", + "homebrew", + "nixos", + "pip", + "unknown", +}) +CLIENT_RESOURCE_KEYS: frozenset[str] = frozenset({ + "architecture", + "hermes_version", + "install_method", + "os_family", +}) + +def client_os_family(value: Any) -> str: + """Map a platform system name to the shared-metrics OS taxonomy.""" + normalized = str(value or "").strip().lower() + return { + "darwin": "macos", + "linux": "linux", + "macos": "macos", + "windows": "windows", + }.get(normalized, "unknown") + + +def client_architecture(value: Any) -> str: + """Map a machine architecture to the shared-metrics taxonomy.""" + normalized = str(value or "").strip().lower().replace("-", "_") + if normalized in {"amd64", "x64", "x86_64"}: + return "x86_64" + if normalized in {"aarch64", "arm64"}: + return "arm64" + if normalized in {"i386", "i486", "i586", "i686", "x86"}: + return "x86" + if normalized.startswith("armv"): + return "arm" + return "unknown" + + +def client_install_method(value: Any) -> str: + """Return an allowlisted Hermes installation method.""" + normalized = str(value or "").strip().lower() + if normalized == "nix": + return "nixos" + return normalized if normalized in CLIENT_INSTALL_METHODS else "unknown" + + +def client_resource( + hermes_version: Any, + *, + os_name: Any, + architecture: Any, + install_method: Any, +) -> dict[str, str]: + """Build the bounded client resource attached to aggregate packages.""" + normalized_version = str(hermes_version or "").strip() + if not normalized_version or len(normalized_version) > 64: + normalized_version = "unknown" + return { + "architecture": client_architecture(architecture), + "hermes_version": normalized_version, + "install_method": client_install_method(install_method), + "os_family": client_os_family(os_name), + } + + +def client_resource_is_valid(resource: Any) -> bool: + """Return whether a package resource exactly matches the bounded contract.""" + if not isinstance(resource, dict) or set(resource) != CLIENT_RESOURCE_KEYS: + return False + version = resource.get("hermes_version") + return ( + isinstance(version, str) + and 0 < len(version) <= 64 + and resource.get("os_family") in CLIENT_OS_FAMILIES + and resource.get("architecture") in CLIENT_ARCHITECTURES + and resource.get("install_method") in CLIENT_INSTALL_METHODS + ) + _LEGACY_PROVIDER_FAMILIES = frozenset({ "aggregator", @@ -213,6 +308,7 @@ }) _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = { + CLIENT_ACTIVE_METRIC: {}, # Retained only so pre-v2 pending rows remain packageable. LEGACY_MODEL_CALL_METRIC: { "call_role": frozenset({"primary"}), @@ -259,6 +355,7 @@ }, } COUNTER_METRICS: frozenset[str] = frozenset({ + CLIENT_ACTIVE_METRIC, MODEL_ROUTE_METRIC, SKILL_LIFECYCLE_METRIC, SKILL_LOAD_METRIC, @@ -307,6 +404,22 @@ def _event_metadata_is_valid(event: Any) -> bool: ) in {"OK", "ERROR"} +def client_active_counter(event: Any) -> tuple[str, dict[str, str]] | None: + """Return the active-install counter for one empty allowlisted mark.""" + if not _event_metadata_is_valid(event): + return None + if ( + str(getattr(event, "kind", "") or "") != "mark" + or str(getattr(event, "name", "") or "") != CLIENT_ACTIVE_MARK + or getattr(event, "category", None) is not None + or getattr(event, "scope_category", None) is not None + or getattr(event, "category_profile", None) is not None + or getattr(event, "data", None) != {} + ): + return None + return CLIENT_ACTIVE_METRIC, {} + + def model_call_dimensions(event: Any) -> dict[str, str] | None: """Return package dimensions for one valid logical model-call end event.""" auxiliary = _auxiliary_model_call_dimensions(event) diff --git a/hermes_cli/observability/shared_metrics_subscriber.py b/hermes_cli/observability/shared_metrics_subscriber.py index b1ae16141e6c..54c05e5780d1 100644 --- a/hermes_cli/observability/shared_metrics_subscriber.py +++ b/hermes_cli/observability/shared_metrics_subscriber.py @@ -3,15 +3,20 @@ from __future__ import annotations import logging +import platform import threading from typing import Any from agent.relay_runtime import RUNTIME_INSTANCE_KEY +from hermes_cli.config import detect_install_method from .shared_metrics import SharedMetricsStore from .shared_metrics_contract import ( + CLIENT_ACTIVE_METRIC, MODEL_ROUTE_METRIC, TOOL_CALL_METRIC, + client_active_counter, + client_resource, model_call_dimensions, skill_counter, task_counter, @@ -33,7 +38,12 @@ def __init__( runtime_id: str | None = None, ) -> None: self.store = store - self._hermes_version = hermes_version or "unknown" + self._client_resource = client_resource( + hermes_version, + os_name=platform.system(), + architecture=platform.machine(), + install_method=detect_install_method(), + ) self._runtime_id = runtime_id self._active = True self._lock = threading.RLock() @@ -51,8 +61,14 @@ def __call__(self, event: Any) -> None: or metadata.get(RUNTIME_INSTANCE_KEY) != self._runtime_id ): return - dimensions = model_call_dimensions(event) - metric_name = MODEL_ROUTE_METRIC + metric = client_active_counter(event) + dimensions = None + metric_name = CLIENT_ACTIVE_METRIC + if metric is not None: + metric_name, dimensions = metric + if dimensions is None: + dimensions = model_call_dimensions(event) + metric_name = MODEL_ROUTE_METRIC if dimensions is None: dimensions = tool_call_dimensions(event) metric_name = TOOL_CALL_METRIC @@ -69,11 +85,14 @@ def __call__(self, event: Any) -> None: if not self._active: return try: - self.store.record_counter( - metric_name, - dimensions, - self._hermes_version, - ) + if metric_name == CLIENT_ACTIVE_METRIC: + self.store.record_client_active(self._client_resource) + else: + self.store.record_counter( + metric_name, + dimensions, + self._client_resource, + ) except Exception: logger.warning( "Unable to persist the Hermes shared metric: %s", diff --git a/package-lock.json b/package-lock.json index 6204d5119aac..d7cd4adf5860 100644 --- a/package-lock.json +++ b/package-lock.json @@ -155,7 +155,7 @@ "bippy": "0.5.43", "concurrently": "10.0.4", "cross-env": "10.1.0", - "electron": "41.10.3", + "electron": "40.10.2", "electron-builder": "26.15.3", "esbuild": "0.28.1", "jsdom": "29.1.1", @@ -1308,16 +1308,6 @@ "react": ">=16.8.0" } }, - "node_modules/@electron-internal/extract-zip": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@electron-internal/extract-zip/-/extract-zip-1.0.5.tgz", - "integrity": "sha512-+bqFCP98pLI0Tt0XQo1TmlXtwjWchISndDOxCkEcIuUgXWpBnLyRI+2DU+mesvnMMX6L1XDqYNA0lXNDHd/yiA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=22.12.0" - } - }, "node_modules/@electron/asar": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.4.1.tgz", @@ -1450,50 +1440,25 @@ } }, "node_modules/@electron/get": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@electron/get/-/get-5.1.0.tgz", - "integrity": "sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-2.0.3.tgz", + "integrity": "sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ==", "dev": true, "license": "MIT", "dependencies": { "debug": "^4.1.1", - "env-paths": "^3.0.0", - "graceful-fs": "^4.2.11", + "env-paths": "^2.2.0", + "fs-extra": "^8.1.0", + "got": "^11.8.5", "progress": "^2.0.3", - "semver": "^7.6.3", + "semver": "^6.2.0", "sumchecker": "^3.0.1" }, "engines": { - "node": ">=22.12.0" + "node": ">=12" }, "optionalDependencies": { - "undici": "^7.24.4" - } - }, - "node_modules/@electron/get/node_modules/env-paths": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", - "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@electron/get/node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" + "global-agent": "^3.0.0" } }, "node_modules/@electron/notarize": { @@ -6347,6 +6312,17 @@ "integrity": "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==", "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.64.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.64.0.tgz", @@ -9218,22 +9194,22 @@ } }, "node_modules/electron": { - "version": "41.10.3", - "resolved": "https://registry.npmjs.org/electron/-/electron-41.10.3.tgz", - "integrity": "sha512-MJuSODPw8siv/I8JjhctW/cS/XNldwI4gLRyyWZx6QkoZJUDgbEvitp7IVOnGrHENTQb6Udo+zMpKhFnhlIhdg==", + "version": "40.10.2", + "resolved": "https://registry.npmjs.org/electron/-/electron-40.10.2.tgz", + "integrity": "sha512-Xj3Hy0Imbu4g0gDIW55w/jJYz94nMO2JRSGYA3LyAn5SwaERCelgZrA21vfH+Bi//SWAWQXddHsMwCqauyMT8g==", "dev": true, "hasInstallScript": true, "license": "MIT", "dependencies": { - "@electron-internal/extract-zip": "^1.0.1", - "@electron/get": "^5.0.0", - "@types/node": "^24.9.0" + "@electron/get": "^2.0.0", + "@types/node": "^24.9.0", + "extract-zip": "^2.0.1" }, "bin": { "electron": "cli.js" }, "engines": { - "node": ">= 22.12.0" + "node": ">= 12.20.55" } }, "node_modules/electron-builder": { @@ -10155,6 +10131,27 @@ "node": ">=0.10.0" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -10400,6 +10397,21 @@ } } }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -13734,9 +13746,9 @@ } }, "node_modules/node-gyp/node_modules/undici": { - "version": "6.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", - "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "version": "6.27.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.27.0.tgz", + "integrity": "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==", "dev": true, "license": "MIT", "engines": { @@ -14079,6 +14091,13 @@ "url": "https://github.com/sponsors/jet2jet" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -16602,9 +16621,9 @@ } }, "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", "dev": true, "license": "MIT", "engines": { @@ -17617,6 +17636,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/yauzl": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz", + "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -17742,7 +17774,7 @@ "ink-text-input": "6.0.0", "nanostores": "1.4.0", "react": "19.2.7", - "undici": "^6.28.0", + "undici": "6.27.0", "unicode-animations": "1.0.3" }, "devDependencies": { @@ -17768,9 +17800,9 @@ } }, "ui-tui/node_modules/undici": { - "version": "6.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", - "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "version": "6.27.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.27.0.tgz", + "integrity": "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==", "license": "MIT", "engines": { "node": ">=18.17" diff --git a/plugins/platforms/photon/sidecar/package-lock.json b/plugins/platforms/photon/sidecar/package-lock.json index 12167c52899b..d9f3f9ff49f3 100644 --- a/plugins/platforms/photon/sidecar/package-lock.json +++ b/plugins/platforms/photon/sidecar/package-lock.json @@ -1726,9 +1726,9 @@ } }, "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", "license": "MIT", "engines": { "node": ">=20.18.1" diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.py index a4dbf0f566d2..ba11567532d2 100644 --- a/scripts/smoke_nemo_relay_shared_metrics.py +++ b/scripts/smoke_nemo_relay_shared_metrics.py @@ -298,6 +298,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]: for counter in counters: by_name.setdefault(counter["name"], []).append(counter) if set(by_name) != { + "hermes.client.active", "hermes.model_route.count", "hermes.skill.lifecycle.count", "hermes.skill.load.count", @@ -308,6 +309,17 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]: raise AssertionError( f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}" ) + if by_name["hermes.client.active"] != [ + { + "name": "hermes.client.active", + "dimensions": {}, + "value": 1, + "packaged_value": 1, + } + ]: + raise AssertionError( + f"Unexpected client-active counter: {by_name['hermes.client.active']}" + ) [model] = by_name["hermes.model_route.count"] expected_model = { "name": "hermes.model_route.count", @@ -429,6 +441,13 @@ def _validate_packages( ] for package in packages: jsonschema.validate(package, schema) + if set(package["resource"]) != { + "architecture", + "hermes_version", + "install_method", + "os_family", + }: + raise AssertionError(f"Unexpected client resource: {package['resource']}") serialized = json.dumps(packages) for prohibited in ( @@ -448,6 +467,7 @@ def _validate_packages( for metric in package.get("metrics", []): metrics.setdefault(metric["name"], []).append(metric) if set(metrics) != { + "hermes.client.active", "hermes.model_route.count", "hermes.skill.lifecycle.count", "hermes.skill.load.count", @@ -458,6 +478,17 @@ def _validate_packages( raise AssertionError( f"Unexpected package metrics:\n{json.dumps(metrics, indent=2)}" ) + if metrics["hermes.client.active"] != [ + { + "name": "hermes.client.active", + "type": "counter", + "dimensions": {}, + "value": 1, + } + ]: + raise AssertionError( + f"Unexpected client-active metric: {metrics['hermes.client.active']}" + ) [model] = metrics["hermes.model_route.count"] if model["dimensions"] != { "model": MODEL_CANARY, diff --git a/tests/agent/test_auxiliary_relay.py b/tests/agent/test_auxiliary_relay.py index 18e7b5ed55d3..9d3c675b1ba7 100644 --- a/tests/agent/test_auxiliary_relay.py +++ b/tests/agent/test_auxiliary_relay.py @@ -251,7 +251,7 @@ def run(task): snapshot = store.counter_snapshot() assert len(snapshot) == 1 assert snapshot[0]["metric_name"] == MODEL_ROUTE_METRIC - assert snapshot[0]["hermes_version"] == "test-version" + assert snapshot[0]["resource"]["hermes_version"] == "test-version" assert snapshot[0]["dimensions"] == { "model": "accepted/model", "provider": "openrouter", diff --git a/tests/hermes_cli/test_relay_shared_metrics.py b/tests/hermes_cli/test_relay_shared_metrics.py index 6e4eaca2e804..e26d3fd2cafd 100644 --- a/tests/hermes_cli/test_relay_shared_metrics.py +++ b/tests/hermes_cli/test_relay_shared_metrics.py @@ -5,6 +5,7 @@ import json import multiprocessing as mp import os +import shutil import sqlite3 import stat import threading @@ -12,7 +13,7 @@ import uuid from concurrent.futures import ThreadPoolExecutor from copy import deepcopy -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace from typing import Any @@ -22,6 +23,10 @@ from hermes_cli.observability import shared_metrics as shared_metrics_module from hermes_cli.observability.shared_metrics import SharedMetricsStore from hermes_cli.observability.shared_metrics_contract import ( + CLIENT_ACTIVE_METRIC, + CLIENT_ARCHITECTURES, + CLIENT_INSTALL_METHODS, + CLIENT_OS_FAMILIES, COUNT_BUCKETS, DURATION_BUCKETS, EXECUTION_SURFACES, @@ -46,6 +51,11 @@ TOOL_LATENCY_BUCKETS, TOOL_OUTCOMES, TOOL_RETRY_BUCKETS, + client_active_counter, + client_architecture, + client_install_method, + client_os_family, + client_resource, count_bucket, duration_bucket, execution_surface, @@ -111,6 +121,20 @@ def _dimensions() -> dict[str, str]: } +def _resource( + hermes_version: str = "test-version", + *, + os_family: str = "linux", + architecture: str = "x86_64", + install_method: str = "git", +) -> dict[str, str]: + return { + "architecture": architecture, + "hermes_version": hermes_version, + "install_method": install_method, + "os_family": os_family, + } + def _legacy_dimensions() -> dict[str, str]: return { "call_role": "primary", @@ -131,15 +155,25 @@ def _record_model_calls_in_process( start_barrier.wait() store = SharedMetricsStore(Path(database_path), Path(outbox_directory)) for _ in range(count): - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) + + +def _record_client_active_in_process( + database_path: str, + outbox_directory: str, + start_barrier: Any, +) -> None: + store = SharedMetricsStore(Path(database_path), Path(outbox_directory)) + start_barrier.wait() + store.record_client_active(_resource()) def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_path): database_path = tmp_path / "metrics.sqlite3" outbox_directory = tmp_path / "outbox" store = SharedMetricsStore(database_path, outbox_directory) - store.record_model_call(_dimensions(), "test-version") - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) + store.record_model_call(_dimensions(), _resource()) first_paths = store.create_and_export_package() @@ -149,7 +183,7 @@ def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_pat uuid.UUID(first_package["package_id"]) uuid.UUID(first_package["install_id"]) assert first_package["schema_version"] == "hermes.shared_metrics.v2" - assert first_package["resource"] == {"hermes_version": "test-version"} + assert first_package["resource"] == _resource() assert first_package["metrics"] == [ { "name": MODEL_ROUTE_METRIC, @@ -165,7 +199,7 @@ def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_pat assert restarted.create_and_export_package() == [] assert len(list(outbox_directory.glob("*.json"))) == 1 - restarted.record_model_call(_dimensions(), "test-version") + restarted.record_model_call(_dimensions(), _resource()) second_paths = restarted.create_and_export_package() assert len(second_paths) == 1 @@ -194,19 +228,33 @@ def test_v2_package_preserves_pending_v1_model_counters(tmp_path): period_start, metric_name, hermes_version, + os_family, + architecture, + install_method, dimensions_json, value, packaged_value - ) VALUES (?, ?, ?, ?, 3, 0) + ) VALUES (?, ?, ?, ?, ?, ?, ?, 3, 0) """, ( period_start, LEGACY_MODEL_CALL_METRIC, "test-version", + "unknown", + "unknown", + "unknown", legacy_dimensions_json, ), ) - store.record_model_call(_dimensions(), "test-version") + store.record_model_call( + _dimensions(), + _resource( + "test-version", + os_family="unknown", + architecture="unknown", + install_method="unknown", + ), + ) [package_path] = store.create_and_export_package() package = json.loads(package_path.read_text(encoding="utf-8")) @@ -298,36 +346,162 @@ def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas( monkeypatch.setattr(shared_metrics_module, "_utc_now", lambda: current_time) store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) assert len(store.create_and_export_package_if_due()) == 1 current_time = datetime(2026, 7, 28, 18, tzinfo=timezone.utc) store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) assert store.create_and_export_package_if_due() == [] assert len(list((tmp_path / "outbox").glob("*.json"))) == 1 assert store.counter_snapshot()[0] == { "period_start": "2026-07-28", "metric_name": MODEL_ROUTE_METRIC, - "hermes_version": "test-version", + "resource": _resource(), "dimensions": _dimensions(), "value": 2, "packaged_value": 1, } current_time = datetime(2026, 7, 29, 9, tzinfo=timezone.utc) - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) assert len(store.create_and_export_package_if_due()) == 2 assert len(list((tmp_path / "outbox").glob("*.json"))) == 3 assert all( row["value"] == row["packaged_value"] for row in store.counter_snapshot() ) - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) assert store.create_and_export_package_if_due() == [] assert len(list((tmp_path / "outbox").glob("*.json"))) == 3 +def test_client_active_uses_a_transactional_rolling_24_hour_latch( + tmp_path, + monkeypatch, +): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + now = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc) + monkeypatch.setattr(shared_metrics_module, "_utc_now", lambda: now) + + assert store.record_client_active(_resource()) + assert not store.record_client_active(_resource()) + + now += timedelta(hours=23, minutes=59, seconds=59) + assert not store.record_client_active(_resource()) + + now += timedelta(seconds=1) + assert store.record_client_active(_resource()) + + active = [ + counter + for counter in store.counter_snapshot() + if counter["metric_name"] == CLIENT_ACTIVE_METRIC + ] + assert [counter["dimensions"] for counter in active] == [{}, {}] + assert [counter["period_start"] for counter in active] == [ + "2026-07-22", + "2026-07-23", + ] + assert [counter["value"] for counter in active] == [1, 1] + + +def test_client_active_recovers_from_an_invalid_latch_and_creates_identity( + tmp_path, + monkeypatch, +): + database_path = tmp_path / "metrics.sqlite3" + store = SharedMetricsStore(database_path, tmp_path / "outbox") + with sqlite3.connect(database_path) as connection: + connection.execute( + "INSERT INTO telemetry_state(key, value) VALUES (?, ?)", + ("client_active_recorded_at", "invalid-timestamp"), + ) + now = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc) + monkeypatch.setattr(shared_metrics_module, "_utc_now", lambda: now) + + assert store.record_client_active(_resource()) + + with sqlite3.connect(database_path) as connection: + state = dict( + connection.execute( + "SELECT key, value FROM telemetry_state WHERE key != 'schema_version'" + ).fetchall() + ) + uuid.UUID(state["install_id"]) + assert state["client_active_recorded_at"] == "2026-07-22T10:00:00Z" + + +def test_client_active_rebases_a_future_latch_without_double_counting( + tmp_path, + monkeypatch, +): + database_path = tmp_path / "metrics.sqlite3" + store = SharedMetricsStore(database_path, tmp_path / "outbox") + now = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc) + monkeypatch.setattr(shared_metrics_module, "_utc_now", lambda: now) + + assert store.record_client_active(_resource()) + with sqlite3.connect(database_path) as connection: + connection.execute( + "UPDATE telemetry_state SET value = ? WHERE key = ?", + ("2026-07-24T10:00:00Z", "client_active_recorded_at"), + ) + + assert not store.record_client_active(_resource()) + with sqlite3.connect(database_path) as connection: + latch = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", + ("client_active_recorded_at",), + ).fetchone()[0] + + assert latch == "2026-07-22T10:00:00Z" + [counter] = store.counter_snapshot() + assert counter["metric_name"] == CLIENT_ACTIVE_METRIC + assert counter["value"] == 1 + + +def test_client_active_package_uses_empty_dimensions_and_stable_install_id(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + + assert store.record_client_active(_resource()) + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + + _schema_validator().validate(package) + uuid.UUID(package["install_id"]) + assert package["metrics"] == [ + { + "name": CLIENT_ACTIVE_METRIC, + "type": "counter", + "dimensions": {}, + "value": 1, + } + ] + + +def test_deleting_local_metrics_state_resets_install_identity(tmp_path): + root = tmp_path / "shared-metrics" + database_path = root / "metrics.sqlite3" + outbox_directory = root / "outbox" + first = SharedMetricsStore(database_path, outbox_directory) + assert first.record_client_active(_resource()) + [first_package_path] = first.create_and_export_package() + first_package = json.loads(first_package_path.read_text(encoding="utf-8")) + + shutil.rmtree(root) + + reset = SharedMetricsStore(database_path, outbox_directory) + assert reset.record_client_active(_resource()) + [reset_package_path] = reset.create_and_export_package() + reset_package = json.loads(reset_package_path.read_text(encoding="utf-8")) + + assert reset_package["install_id"] != first_package["install_id"] + assert reset_package["metrics"][0]["name"] == CLIENT_ACTIVE_METRIC + + def test_package_schema_matches_the_model_call_contract(): schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) properties = _package_dimension_schema()["properties"] @@ -340,6 +514,89 @@ def test_package_schema_matches_the_model_call_contract(): assert "enum" not in properties["provider"] +def test_client_resource_classification_is_bounded(): + assert client_os_family("Darwin") == "macos" + assert client_os_family("Windows") == "windows" + assert client_architecture("AMD64") == "x86_64" + assert client_architecture("aarch64") == "arm64" + assert client_architecture("armv7l") == "arm" + assert client_install_method("Homebrew") == "homebrew" + assert client_install_method("nix") == "nixos" + + assert client_resource( + "", + os_name="privacy-os-canary", + architecture="privacy-arch-canary", + install_method="privacy-install-canary", + ) == _resource( + "unknown", + os_family="unknown", + architecture="unknown", + install_method="unknown", + ) + +def test_package_schema_matches_the_client_resource_contract(): + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + resource = schema["properties"]["resource"] + + # Every v2 package records the complete bounded client resource. + assert set(resource["required"]) == { + "architecture", + "hermes_version", + "install_method", + "os_family", + } + assert set(resource["properties"]) == { + "architecture", + "hermes_version", + "install_method", + "os_family", + } + assert set(resource["properties"]["os_family"]["enum"]) == CLIENT_OS_FAMILIES + assert set(resource["properties"]["architecture"]["enum"]) == (CLIENT_ARCHITECTURES) + assert set(resource["properties"]["install_method"]["enum"]) == ( + CLIENT_INSTALL_METHODS + ) + + +def test_client_active_mark_accepts_only_an_empty_allowlisted_payload(): + event = SimpleNamespace( + kind="mark", + category=None, + category_profile=None, + name="hermes.client.active", + scope_category=None, + metadata={ + "hermes.metrics.schema_version": "hermes.metrics.event.v2", + }, + data={}, + ) + + assert client_active_counter(event) == (CLIENT_ACTIVE_METRIC, {}) + + with_payload = deepcopy(event) + with_payload.data = {"session_id": "privacy-canary"} + assert client_active_counter(with_payload) is None + + wrong_schema = deepcopy(event) + wrong_schema.metadata["hermes.metrics.schema_version"] = "unknown" + assert client_active_counter(wrong_schema) is None + + +def test_package_schema_matches_the_task_contract(): + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + start = _task_dimension_schema("task_started_counter")["properties"] + terminal = _task_dimension_schema("task_finished_counter")["properties"] + + assert set(schema["$defs"]["execution_surface"]["enum"]) == EXECUTION_SURFACES + assert set(schema["$defs"]["task_entrypoint"]["enum"]) == TASK_ENTRYPOINTS + assert set(schema["$defs"]["duration_bucket"]["enum"]) == DURATION_BUCKETS + assert set(schema["$defs"]["count_bucket"]["enum"]) == COUNT_BUCKETS + assert start["entrypoint"] == {"$ref": "#/$defs/task_entrypoint"} + assert set(terminal["end_reason"]["enum"]) == TASK_END_REASONS + assert set(terminal["outcome"]["enum"]) == TASK_OUTCOMES + assert set(terminal["termination"]["enum"]) == TASK_TERMINATIONS + def test_v1_package_schema_retains_the_legacy_model_contract(): schema = json.loads(LEGACY_SCHEMA_PATH.read_text(encoding="utf-8")) model_counter = schema["$defs"]["model_call_counter"] @@ -704,6 +961,156 @@ def test_skill_event_fields_are_bounded_and_reject_malformed_usage(): is None ) +def test_store_rejects_an_unsupported_schema_version(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + connection.execute( + "INSERT INTO telemetry_state(key, value) VALUES ('schema_version', '999')" + ) + + with pytest.raises(RuntimeError, match="Unsupported shared-metrics store schema"): + SharedMetricsStore(database_path, tmp_path / "outbox") + + with sqlite3.connect(database_path) as connection: + [schema_version] = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'schema_version'" + ).fetchone() + assert schema_version == "999" + +def test_store_migrates_v1_counters_with_unknown_client_dimensions(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + install_id = str(uuid.uuid4()) + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + connection.executemany( + "INSERT INTO telemetry_state(key, value) VALUES (?, ?)", + [("schema_version", "1"), ("install_id", install_id)], + ) + connection.execute( + """ + CREATE TABLE counter_aggregates ( + period_start TEXT NOT NULL, + metric_name TEXT NOT NULL, + hermes_version TEXT NOT NULL, + dimensions_json TEXT NOT NULL, + value INTEGER NOT NULL, + packaged_value INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY ( + period_start, + metric_name, + hermes_version, + dimensions_json + ) + ) + """ + ) + connection.execute( + """ + INSERT INTO counter_aggregates( + period_start, + metric_name, + hermes_version, + dimensions_json, + value, + packaged_value + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + "2026-07-21", + LEGACY_MODEL_CALL_METRIC, + "old-version", + json.dumps( + _legacy_dimensions(), + sort_keys=True, + separators=(",", ":"), + ), + 3, + 1, + ), + ) + + store = SharedMetricsStore(database_path, outbox_directory) + + [counter] = store.counter_snapshot() + assert counter["resource"] == _resource( + "old-version", + os_family="unknown", + architecture="unknown", + install_method="unknown", + ) + assert counter["value"] == 3 + assert counter["packaged_value"] == 1 + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + _schema_validator().validate(package) + assert package["install_id"] == install_id + assert package["metrics"][0]["value"] == 2 + +def test_pending_metrics_keep_the_client_resource_recorded_at_event_time(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + resource_a = _resource("version-a", architecture="arm64", install_method="pip") + resource_b = _resource("version-a", os_family="macos") + store.record_model_call(_dimensions(), resource_a) + store.record_model_call(_dimensions(), resource_b) + + packages = [ + json.loads(path.read_text(encoding="utf-8")) + for path in store.create_and_export_package() + ] + + assert {tuple(sorted(package["resource"].items())) for package in packages} == { + tuple(sorted(resource_a.items())), + tuple(sorted(resource_b.items())), + } + assert all(package["metrics"][0]["value"] == 1 for package in packages) + +def test_store_exports_task_started_and_terminal_counters(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_counter( + "hermes.task_run.started", + {"entrypoint": "interactive", "execution_surface": "cli"}, + _resource(), + ) + terminal = task_terminal_fields( + { + "platform": "cli", + "completed": True, + "turn_exit_reason": "text_response(stop)", + }, + duration_ms=2_000, + model_call_count=1, + tool_call_count=2, + retry_count=0, + ) + store.record_counter("hermes.task_run.finished", terminal, _resource()) + + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + _schema_validator().validate(package) + + assert {metric["name"] for metric in package["metrics"]} == { + "hermes.task_run.finished", + "hermes.task_run.started", + } + +def test_package_schema_rejects_unknown_fields(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_model_call(_dimensions(), _resource()) + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + invalid_package = deepcopy(package) + invalid_package["prompt"] = "must-not-be-accepted" + + jsonschema = pytest.importorskip("jsonschema") + with pytest.raises(jsonschema.ValidationError): + _schema_validator().validate(invalid_package) + def test_store_does_not_record_the_retired_model_metric(tmp_path): store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") @@ -711,14 +1118,82 @@ def test_store_does_not_record_the_retired_model_metric(tmp_path): store.record_counter( LEGACY_MODEL_CALL_METRIC, _legacy_dimensions(), - "test-version", + _resource(), ) assert store.counter_snapshot() == [] +def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + with pytest.raises(ValueError, match="Unsupported dimensions"): + store.record_counter( + MODEL_ROUTE_METRIC, + {"prompt": "must-not-be-persisted"}, + _resource(), + ) + assert store.counter_snapshot() == [] + +def test_store_rejects_client_resources_outside_the_contract(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + + with pytest.raises(ValueError, match="Unsupported shared-metrics client resource"): + store.record_model_call( + _dimensions(), + { + **_resource(), + "architecture": "privacy-architecture-canary", + }, + ) + + assert store.counter_snapshot() == [] + +def test_package_builder_rejects_tampered_dimensions(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), _resource()) + with sqlite3.connect(database_path) as connection: + connection.execute( + "UPDATE counter_aggregates SET dimensions_json = ?", + (json.dumps({"prompt": "must-not-be-exported"}),), + ) + + with pytest.raises(ValueError, match="Unsupported dimensions"): + store.create_and_export_package() + + assert list(outbox_directory.glob("*.json")) == [] + + +def test_package_builder_rejects_tampered_client_resources(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), _resource()) + with sqlite3.connect(database_path) as connection: + connection.execute( + "UPDATE counter_aggregates SET os_family = ?", + ("privacy-os-canary",), + ) + + with pytest.raises( + ValueError, + match="Unsupported shared-metrics client resource", + ): + store.create_and_export_package() + + assert list(outbox_directory.glob("*.json")) == [] + + +def test_pending_package_retry_reuses_the_same_package_and_file(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), _resource()) + [package_path] = store.create_and_export_package() + original_payload = package_path.read_bytes() @@ -752,11 +1227,11 @@ def test_retention_prunes_only_expired_exported_history(tmp_path): outbox_directory = tmp_path / "outbox" store = SharedMetricsStore(database_path, outbox_directory) - store.record_model_call(_dimensions(), "expired-version") + store.record_model_call(_dimensions(), _resource("expired-version")) [expired_path] = store.create_and_export_package() - store.record_model_call(_dimensions(), "current-version") + store.record_model_call(_dimensions(), _resource("current-version")) [current_path] = store.create_and_export_package() - store.record_model_call(_dimensions(), "pending-version") + store.record_model_call(_dimensions(), _resource("pending-version")) pending_package = store._create_package() assert pending_package is not None @@ -806,17 +1281,94 @@ def test_retention_prunes_only_expired_exported_history(tmp_path): assert aggregate_versions == {"current-version", "pending-version"} +def test_retention_failure_does_not_fail_a_committed_export(tmp_path, monkeypatch): + store = SharedMetricsStore( + tmp_path / "metrics.sqlite3", + tmp_path / "outbox", + ) + store.record_model_call(_dimensions(), _resource()) + def fail_pruning(): + raise OSError("retention unavailable") + monkeypatch.setattr(store, "_prune_expired_history", fail_pruning) + [package_path] = store.create_and_export_package() + assert package_path.exists() + assert store.counter_snapshot()[0]["packaged_value"] == 1 + + +def test_file_export_failure_retries_committed_outbox_without_duplicate_delta( + tmp_path, monkeypatch +): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), _resource()) + + def fail_write(*_args, **_kwargs): + raise OSError("simulated atomic export failure") + + module_globals = SharedMetricsStore._export_pending_packages.__globals__ + original_write = module_globals["atomic_json_write"] + monkeypatch.setitem(module_globals, "atomic_json_write", fail_write) + with pytest.raises(OSError, match="simulated atomic export failure"): + store.create_and_export_package() + + with sqlite3.connect(database_path) as connection: + package_id, exported_at = connection.execute( + "SELECT package_id, exported_at FROM package_outbox" + ).fetchone() + assert exported_at is None + assert store.counter_snapshot()[0]["packaged_value"] == 1 + assert list(outbox_directory.glob("*.json")) == [] + + monkeypatch.setitem(module_globals, "atomic_json_write", original_write) + assert store.create_and_export_package() == [ + outbox_directory / f"{package_id}.json" + ] + assert len(list(outbox_directory.glob("*.json"))) == 1 + assert store.create_and_export_package() == [] + + +def test_package_export_does_not_chase_concurrent_updates(tmp_path, monkeypatch): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), _resource()) + original_create = store._create_package + create_calls = 0 + + def create_and_record_another(): + nonlocal create_calls + create_calls += 1 + package = original_create() + if create_calls == 1: + store.record_model_call(_dimensions(), _resource()) + return package + + monkeypatch.setattr(store, "_create_package", create_and_record_another) + first_paths = store.create_and_export_package() + + assert create_calls == 1 + assert len(first_paths) == 1 + [counter] = store.counter_snapshot() + assert counter["metric_name"] == MODEL_ROUTE_METRIC + assert counter["dimensions"] == _dimensions() + assert counter["value"] == 2 + assert counter["packaged_value"] == 1 + + second_paths = store.create_and_export_package() + assert len(second_paths) == 1 + assert store.counter_snapshot()[0]["packaged_value"] == 2 def test_concurrent_package_builders_commit_one_delta(tmp_path): database_path = tmp_path / "metrics.sqlite3" outbox_directory = tmp_path / "outbox" store = SharedMetricsStore(database_path, outbox_directory) - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) ready = threading.Barrier(2) def export() -> list[Path]: @@ -841,8 +1393,49 @@ def export() -> list[Path]: assert store.counter_snapshot()[0]["packaged_value"] == 1 +def test_concurrent_due_exports_create_one_daily_package(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), _resource()) + ready = threading.Barrier(8) + + def export() -> None: + worker_store = SharedMetricsStore(database_path, outbox_directory) + ready.wait(timeout=5) + worker_store.create_and_export_package_if_due() + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [executor.submit(export) for _ in range(8)] + for future in futures: + future.result() + + with sqlite3.connect(database_path) as connection: + [outbox_count] = connection.execute( + "SELECT COUNT(*) FROM package_outbox" + ).fetchone() + assert outbox_count == 1 + assert len(list(outbox_directory.glob("*.json"))) == 1 + assert store.counter_snapshot()[0]["packaged_value"] == 1 + + +def test_concurrent_model_call_updates_are_transactional(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + SharedMetricsStore(database_path, outbox_directory) + def record_calls(count: int) -> None: + store = SharedMetricsStore(database_path, outbox_directory) + for _ in range(count): + store.record_model_call(_dimensions(), _resource()) + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(record_calls, 10) for _ in range(2)] + for future in futures: + future.result() + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 20 def test_cross_process_model_call_updates_are_transactional(tmp_path): @@ -869,6 +1462,55 @@ def test_cross_process_model_call_updates_are_transactional(tmp_path): assert restarted.counter_snapshot()[0]["value"] == 20 +def test_cross_process_client_active_attempts_record_one_install(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + context = mp.get_context("spawn") + start_barrier = context.Barrier(2) + processes = [ + context.Process( + target=_record_client_active_in_process, + args=(str(database_path), str(outbox_directory), start_barrier), + ) + for _ in range(2) + ] + + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert not process.is_alive() + assert process.exitcode == 0 + + store = SharedMetricsStore(database_path, outbox_directory) + [active] = store.counter_snapshot() + assert active["metric_name"] == CLIENT_ACTIVE_METRIC + assert active["dimensions"] == {} + assert active["value"] == 1 + + +def test_schema_initialization_waits_for_an_existing_writer(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + database_path.touch() + blocker = sqlite3.connect(database_path) + blocker.execute("BEGIN IMMEDIATE") + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + SharedMetricsStore, + database_path, + outbox_directory, + ) + try: + time.sleep(0.4) + assert not future.done() + finally: + blocker.rollback() + blocker.close() + store = future.result(timeout=2) + + assert store.counter_snapshot() == [] @pytest.mark.skipif(os.name == "nt", reason="POSIX permission modes are unavailable") @@ -876,7 +1518,7 @@ def test_store_and_export_are_owner_only(tmp_path): database_path = tmp_path / "private-store" / "metrics.sqlite3" outbox_directory = tmp_path / "private-outbox" store = SharedMetricsStore(database_path, outbox_directory) - store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), _resource()) [package_path] = store.create_and_export_package() assert stat.S_IMODE(database_path.parent.stat().st_mode) == 0o700 diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py index dc8328a86a2a..b38232276b35 100644 --- a/tests/hermes_cli/test_relay_shared_metrics_runtime.py +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -359,6 +359,13 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa } assert starts[0][2] == {} assert starts[0][3]["model_name"] == "unknown" + active_marks = [ + event + for event in direct_runtime.events + if event[0] == "scope.event" and event[1] == "hermes.client.active" + ] + assert len(active_marks) == 2 + assert all(mark[2]["data"] == {} for mark in active_marks) assert ends[0][2] == { "model": "claude-sonnet", "provider": "anthropic", @@ -380,12 +387,19 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa package = json.loads(packages[0].read_text(encoding="utf-8")) metrics = {metric["name"]: metric for metric in package["metrics"]} assert set(metrics) == { + "hermes.client.active", "hermes.model_route.count", "hermes.task_run.finished", "hermes.task_run.started", "hermes.tool_approval.count", "hermes.tool_call.count", } + assert metrics["hermes.client.active"] == { + "name": "hermes.client.active", + "type": "counter", + "dimensions": {}, + "value": 1, + } assert metrics["hermes.model_route.count"]["dimensions"] == { "model": "claude-sonnet", "provider": "anthropic", @@ -605,6 +619,8 @@ def base(index: int) -> dict[str, Any]: for counter in snapshot: by_metric.setdefault(counter["metric_name"], []).append(counter) + assert by_metric["hermes.client.active"][0]["dimensions"] == {} + assert by_metric["hermes.client.active"][0]["value"] == 1 assert len(by_metric["hermes.task_run.started"]) == 1 assert by_metric["hermes.task_run.started"][0]["value"] == 3 assert len(by_metric["hermes.model_route.count"]) == 1 @@ -1214,6 +1230,7 @@ def test_disabling_shared_metrics_stops_collection_and_shutdown_export( root = profile / "telemetry" / "shared_metrics" store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") assert [row["metric_name"] for row in store.counter_snapshot()] == [ + "hermes.client.active", "hermes.task_run.started" ] assert list((root / "outbox").glob("*.json")) == [] diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index a4d5fea1e7ba..db68ee4f712e 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -6146,4 +6146,3 @@ def test_sanitize_context_strips_full_block(self): assert "memory-context" not in result.lower() assert "stale observation" not in result assert "how is the honcho working" in result - diff --git a/ui-tui/package.json b/ui-tui/package.json index f1d0bbe36b64..8965aadf1592 100644 --- a/ui-tui/package.json +++ b/ui-tui/package.json @@ -25,7 +25,7 @@ "ink-text-input": "6.0.0", "nanostores": "1.4.0", "react": "19.2.7", - "undici": "6.28.0", + "undici": "6.27.0", "unicode-animations": "1.0.3" }, "overrides": { diff --git a/uv.lock b/uv.lock index 0cef7c2e75d5..dae87d27395b 100644 --- a/uv.lock +++ b/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-20T16:38:42.729819205Z" +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P14D" [options.exclude-newer-package] @@ -284,7 +284,6 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alibabacloud-credentials" }, { name = "alibabacloud-gateway-spi" }, - { name = "alibabacloud-openapi-util" }, { name = "alibabacloud-tea-util" }, { name = "cryptography" }, { name = "darabonba-core" }, @@ -293,7 +292,6 @@ sdist = { url = "https://files.pythonhosted.org/packages/3b/73/fb0c4d44759791ecd wheels = [ { url = "https://files.pythonhosted.org/packages/8d/ec/6b368a10e9c2e8b1b394c69b96ac213ae66e8c4895e0baa1ffaf7178fd32/alibabacloud_tea_openapi-0.4.5-py3-none-any.whl", hash = "sha256:338979095c7beda80a5b413c31262892cafdc12069dde4ce4fc2e4f7ce0fc609", size = 33333, upload-time = "2026-07-14T13:15:38.365Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/be/f594e79625e5ccfcfe7f12d7d70709a3c59e920878469c998886211c850d/alibabacloud_tea_openapi-0.3.16.tar.gz", hash = "sha256:6bffed8278597592e67860156f424bde4173a6599d7b6039fb640a3612bae292", size = 13087, upload-time = "2025-07-04T09:30:10.689Z" } [[package]] name = "alibabacloud-tea-util" @@ -307,15 +305,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/9e/c394b4e2104766fb28a1e44e3ed36e4c7773b4d05c868e482be99d5635c9/alibabacloud_tea_util-0.3.14-py3-none-any.whl", hash = "sha256:10d3e5c340d8f7ec69dd27345eb2fc5a1dab07875742525edf07bbe86db93bfe", size = 6697, upload-time = "2025-11-19T06:01:07.355Z" }, ] -[[package]] -name = "alibabacloud-tea-xml" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "alibabacloud-tea" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/eb/5e82e419c3061823f3feae9b5681588762929dc4da0176667297c2784c1a/alibabacloud_tea_xml-0.0.3.tar.gz", hash = "sha256:979cb51fadf43de77f41c69fc69c12529728919f849723eb0cd24eb7b048a90c", size = 3466, upload-time = "2025-07-01T08:04:55.144Z" } - [[package]] name = "annotated-doc" version = "0.0.4" @@ -1105,6 +1094,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/a7/17208c3b3f92319e7fad259f1c6d5a5baf8fd0654c54846ced329f83c3eb/discord_py-2.7.1-py3-none-any.whl", hash = "sha256:849dca2c63b171146f3a7f3f8acc04248098e9e6203412ce3cf2745f284f7439", size = 1227550, upload-time = "2026-03-03T18:40:44.492Z" }, ] +[package.optional-dependencies] +voice = [ + { name = "davey" }, + { name = "pynacl" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -1706,7 +1701,7 @@ mem0 = [ messaging = [ { name = "aiohttp" }, { name = "brotlicffi" }, - { name = "discord-py" }, + { name = "discord-py", extra = ["voice"] }, { name = "python-telegram-bot", extra = ["webhooks"] }, { name = "qrcode" }, { name = "slack-bolt" }, @@ -1718,6 +1713,9 @@ mistral = [ modal = [ { name = "modal" }, ] +onepassword = [ + { name = "onepassword-sdk" }, +] otlp = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, @@ -1834,7 +1832,7 @@ requires-dist = [ { name = "debugpy", marker = "extra == 'dev'", specifier = "==1.8.20" }, { name = "defusedxml", marker = "extra == 'wecom'", specifier = "==0.7.1" }, { name = "dingtalk-stream", marker = "extra == 'dingtalk'", specifier = "==0.24.3" }, - { name = "discord-py", marker = "extra == 'messaging'", specifier = "==2.7.1" }, + { name = "discord-py", extras = ["voice"], marker = "extra == 'messaging'", specifier = "==2.7.1" }, { name = "edge-tts", marker = "extra == 'edge-tts'", specifier = "==7.2.7" }, { name = "elevenlabs", marker = "extra == 'tts-premium'", specifier = "==1.59.0" }, { name = "exa-py", marker = "extra == 'exa'", specifier = "==2.10.2" }, @@ -1888,6 +1886,7 @@ requires-dist = [ { name = "nemo-relay", marker = "(platform_machine == 'aarch64' and 'android' not in platform_release and sys_platform == 'linux') or (platform_machine == 'x86_64' and 'android' not in platform_release and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')", specifier = ">=0.6.0,<0.7" }, { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "numpy", marker = "extra == 'wake'", specifier = "==2.4.3" }, + { name = "onepassword-sdk", marker = "extra == 'onepassword'", specifier = ">=0.1.0,<0.2.0" }, { name = "onnxruntime", marker = "extra == 'wake'", specifier = "==1.27.0" }, { name = "openai", specifier = "==2.24.0" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp'", specifier = "==1.39.1" }, @@ -1946,7 +1945,10 @@ requires-dist = [ { name = "websockets", specifier = "==15.0.1" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "vercel", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "wake", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "vercel", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "wake", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "onepassword", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] + +[package.metadata.requires-dev] +dev = [{ name = "pytest-timeout", specifier = ">=2.4.0,<3" }] [[package]] name = "hf-xet" @@ -2550,16 +2552,16 @@ wheels = [ [[package]] name = "msal" -version = "1.37.0" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "pyjwt", extra = ["crypto"] }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/99/d840198ecf6e8057bbc937f129ae940404485d736cda73253bbff9537f01/msal-1.37.0.tar.gz", hash = "sha256:1b1672a33ee467c1d70b341bb16cafd51bb3c817147a95b93263794b03971bec", size = 182444, upload-time = "2026-05-29T19:49:05.561Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cb/b02b0f748ac668922364ccb3c3bff5b71628a05f5adfec2ba2a5c3031483/msal-1.36.0.tar.gz", hash = "sha256:3f6a4af2b036b476a4215111c4297b4e6e236ed186cd804faefba23e4990978b", size = 174217, upload-time = "2026-04-09T10:20:33.525Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/b0/d807279f4b55d16d1f120d5ac4344c6e39b56732e2a224d40bded7fd67ad/msal-1.37.0-py3-none-any.whl", hash = "sha256:dd17e95a7c71bce75e8108113438ba7c4a086b3bcad4f57a8c09b7af3d753c2d", size = 123725, upload-time = "2026-05-29T19:49:04.335Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d3/414d1f0a5f6f4fe5313c2b002c54e78a3332970feb3f5fed14237aa17064/msal-1.36.0-py3-none-any.whl", hash = "sha256:36ecac30e2ff4322d956029aabce3c82301c29f0acb1ad89b94edcabb0e58ec4", size = 121547, upload-time = "2026-04-09T10:20:32.336Z" }, ] [[package]] @@ -3580,6 +3582,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -4034,7 +4048,7 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.12'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -4089,7 +4103,7 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -4728,11 +4742,11 @@ name = "vercel-workers" version = "0.0.25" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "python_full_version >= '3.12'" }, - { name = "httpx", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "python-dotenv", marker = "python_full_version >= '3.12'" }, - { name = "vercel", marker = "python_full_version >= '3.12'" }, + { name = "anyio" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "vercel" }, ] sdist = { url = "https://files.pythonhosted.org/packages/30/df/04d37021ad7ca53b7599c313e411d91623c7a005c741f491d1eefb7a9f0c/vercel_workers-0.0.25.tar.gz", hash = "sha256:212ded01400b524be51d251df49f801caf115ad7d48cca7eb168cbeceda3def3", size = 64149, upload-time = "2026-06-20T19:26:27.177Z" } wheels = [