feat(observability): add Relay setup and first-use metrics

docs/observability/relay-shared-metrics.md

@@ -126,9 +126,9 @@ dependency does not change the collection or privacy policy.
 
 ## Current Slices
 
-The current vertical slices record pseudonymous profile activity, logical
-model calls, top-level task runs, tool and approval outcomes, and skill
-lifecycle and reuse:
+The current vertical slices record pseudonymous profile activity, setup and
+first-use milestones, logical model calls, top-level task runs, tool and
+approval outcomes, and skill lifecycle and reuse:
 
 ```text
 Hermes turn, API, tool, and approval hooks
@@ -160,6 +160,25 @@ resources. Concurrent Hermes processes share the SQLite latch, so simultaneous
 starts cannot double-count one install. A later session or task can attempt the
 mark again, but the subscriber suppresses it until the rolling window expires.
 
+An opted-in `hermes setup` run, including the equivalent `hermes portal`
+onboarding entry point, emits `hermes.setup.started` and
+`hermes.setup.finished` marks through a short-lived Relay scope. The marks
+contain only a bounded setup mode, outcome, and failure stage; provider names,
+credentials, answers, and error text are never included. An unflagged setup
+invocation is classified as `interactive` because a new user chooses quick,
+full, or blank-slate setup after the lifecycle begins. Setup that begins before
+shared-metrics consent is available is not recorded retroactively. This
+preserves the rule that no telemetry identity or local state exists before the
+profile has explicitly enabled collection.
+
+The first consented session or task that reaches Hermes's normal runtime
+boundary records `hermes.client.first_usable` once. The first accepted task
+terminal with the bounded `success` outcome records
+`hermes.client.first_successful_task` once. Both use transactional SQLite
+latches, survive process restarts, and remain single-counted when concurrent
+Hermes processes reach the milestone together. The successful-task latch is
+committed in the same transaction as its terminal task counter.
+
 Each task run is a Relay `Function` scope named `hermes.task_run`, parented to
 the owning Hermes session. The start counter contains only bounded execution
 surface and entrypoint values. The terminal counter contains bounded outcome,
@@ -250,10 +269,24 @@ The script uses the installed `nemo-relay` dependency by default. Pass
 `--relay-python ../nemo-relay/python` only when testing a locally built Relay
 binding.
 
-The smoke has the local model request a real `read_file` tool call before its
-final response, then drives create, load, reuse, patch, edit, stale, archive,
-restore, and install skill transitions through the installed Relay binding. It
-verifies model, provider, task, tool, and skill counters in SQLite, validates
-all exported delta packages against the closed schema, verifies the
-pseudonymous client-active counter, and checks that prompt, response, tool-call
-ID, tool-result, and skill-name canaries are absent from the packages.
+To repeat the complete scenario and add a real NVIDIA NIM turn, set
+`NVIDIA_API_KEY` and run:
+
+```bash
+./.venv/bin/python scripts/smoke_nemo_relay_shared_metrics_nvidia.py
+```
+
+The live wrapper keeps the deterministic tool and skill assertions, restarts
+Hermes against NVIDIA NIM, and verifies that exactly one additional model call
+and task reach SQLite and a new schema-valid delta package. The API key remains
+in the subprocess environment and is checked alongside the prompt as
+prohibited persisted data.
+
+The smoke first emits an opted-in setup lifecycle, then has the local model
+request a real `read_file` tool call before its final response. It also drives
+create, load, reuse, patch, edit, stale, archive, restore, and install skill
+transitions through the installed Relay binding. It verifies setup, first-use,
+model, provider, task, tool, and skill counters in SQLite, validates all
+exported delta packages against the closed schema, verifies the pseudonymous
+client-active counter, and checks that prompt, response, tool-call ID,
+tool-result, and skill-name canaries are absent from the packages.

hermes_cli/main.py

@@ -3556,9 +3556,20 @@ def cmd_whatsapp_cloud(args):
 
 def cmd_setup(args):
     """Interactive setup wizard."""
-    from hermes_cli.setup import run_setup_wizard
+    from hermes_cli.setup import run_setup_wizard, run_setup_with_metrics
+
+    if bool(getattr(args, "portal", False)):
+        mode = "portal"
+    elif getattr(args, "section", None):
+        mode = "section"
+    elif bool(getattr(args, "reset", False)):
+        mode = "reset"
+    elif bool(getattr(args, "quick", False)):
+        mode = "quick"
+    else:
+        mode = "interactive"
 
-    run_setup_wizard(args)
+    run_setup_with_metrics(mode, lambda: run_setup_wizard(args))
 
 
 def cmd_model(args):
@@ -3595,7 +3606,7 @@ def _is_profile_api_key_provider(provider_id: str) -> bool:
         return False
 
 
-def select_provider_and_model(args=None):
+def select_provider_and_model(args=None) -> bool:
     """Core provider selection + model picking logic.
 
     Shared by ``cmd_model`` (``hermes model``) and the setup wizard
@@ -3931,7 +3942,7 @@ def _active_custom_key_from_base_url() -> str:
     )
     if provider_idx is None or ordered[provider_idx][0] == "cancel":
         print("No change.")
-        return
+        return False
 
     selected_key = ordered[provider_idx][0]
     selected_members = ordered[provider_idx][2]
@@ -3954,14 +3965,14 @@ def _active_custom_key_from_base_url() -> str:
         )
         if member_idx is None:
             print("No change.")
-            return
+            return False
         selected_provider = selected_members[member_idx]
     else:
         selected_provider = selected_key
 
     if selected_provider == "aux-config":
         _aux_config_menu()
-        return
+        return True
 
     # Step 2: Provider-specific setup + model selection
     if selected_provider == "openrouter":
@@ -3996,7 +4007,7 @@ def _active_custom_key_from_base_url() -> str:
                 "Warning: the selected saved custom provider is no longer available. "
                 "It may have been removed from config.yaml. No change."
             )
-            return
+            return False
         _model_flow_named_custom(config, provider_info)
     elif selected_provider == "remove-custom":
         _remove_custom_provider(config)
@@ -4047,6 +4058,7 @@ def _active_custom_key_from_base_url() -> str:
         "remove-custom",
     } and not selected_provider.startswith("custom:"):
         _clear_stale_openai_base_url()
+    return True
 
 
 def _clear_stale_openai_base_url():

hermes_cli/observability/relay_shared_metrics.py

@@ -7,6 +7,7 @@
 import logging
 import threading
 from collections import deque
+import uuid
 from dataclasses import dataclass, field
 from time import monotonic_ns
 from typing import Any, Callable
@@ -17,10 +18,16 @@
 from .shared_metrics import SharedMetricsStore
 from .shared_metrics_contract import (
     CLIENT_ACTIVE_MARK,
+    CLIENT_FIRST_USABLE_MARK,
     MODEL_CALL_PROFILE_MODEL,
     MODEL_CALL_SCOPE,
     SCHEMA_KEY,
     SCHEMA_VERSION,
+    SETUP_FAILURE_STAGES,
+    SETUP_FINISHED_MARK,
+    SETUP_MODES,
+    SETUP_OUTCOMES,
+    SETUP_STARTED_MARK,
     SKILL_LIFECYCLE_MARK,
     SKILL_LOAD_MARK,
     SUBSCRIBER_NAME,
@@ -102,6 +109,15 @@ class _TaskRun:
     retry_count: int = 0
 
 
+@dataclass(frozen=True)
+class SetupMetricsAttempt:
+    """Opaque handle for one consented setup lifecycle."""
+
+    session_id: str
+    mode: str
+    _runtime: _Runtime = field(repr=False, compare=False)
+
+
 @dataclass
 class _MetricsSession:
     session_id: str
@@ -130,6 +146,7 @@ def __init__(self, host: relay_runtime.RelayRuntime | None = None) -> None:
         self._sessions_lock = threading.RLock()
         self._active = True
         self._sessions: dict[str, _MetricsSession] = {}
+        self._owned_session_ids: set[str] = set()
         self._task_creation_lock = threading.RLock()
         self._task_sessions_lock = threading.RLock()
         self._task_sessions: dict[tuple[str, str], _MetricsSession] = {}
@@ -175,18 +192,76 @@ def record_client_active(self, event: dict[str, Any]) -> None:
         self._emit_client_active(session)
 
     def _emit_client_active(self, session: _MetricsSession) -> None:
+        self._emit_client_mark(session, CLIENT_ACTIVE_MARK, {})
+        self._emit_client_mark(session, CLIENT_FIRST_USABLE_MARK, {})
+
+    def _emit_client_mark(
+        self,
+        session: _MetricsSession,
+        name: str,
+        data: dict[str, str],
+    ) -> None:
         with session.lock:
             if session.closing:
                 return
             self._run_in_session(
                 session,
                 self.relay.scope.event,
-                CLIENT_ACTIVE_MARK,
+                name,
                 handle=session.relay_session.handle,
-                data={},
+                data=data,
                 metadata=self._event_metadata(),
             )
 
+    def record_setup_started(self, attempt: SetupMetricsAttempt) -> bool:
+        with self._sessions_lock:
+            if not self._active:
+                return False
+            self._owned_session_ids.add(attempt.session_id)
+        session = self.ensure_session({
+            "session_id": attempt.session_id,
+            "platform": "cli",
+        })
+        if session is None:
+            return False
+        self._emit_client_mark(session, SETUP_STARTED_MARK, {"mode": attempt.mode})
+        return True
+
+    def record_setup_finished(
+        self,
+        attempt: SetupMetricsAttempt,
+        *,
+        outcome: str,
+        failure_stage: str,
+    ) -> None:
+        try:
+            session = self.ensure_session({
+                "session_id": attempt.session_id,
+                "platform": "cli",
+            })
+            if session is None:
+                return
+            self._emit_client_mark(
+                session,
+                SETUP_FINISHED_MARK,
+                {
+                    "failure_stage": failure_stage,
+                    "mode": attempt.mode,
+                    "outcome": outcome,
+                },
+            )
+        finally:
+            self.close_owned_session(attempt.session_id)
+
+    def close_owned_session(self, session_id: str) -> None:
+        """Close a synthetic metrics session in both runtime ownership layers."""
+        try:
+            self.close_session({"session_id": session_id})
+        finally:
+            with self._sessions_lock:
+                self._owned_session_ids.discard(session_id)
+            self.host.close_session({"session_id": session_id})
+
     def _run_in_session(
         self,
         session: _MetricsSession,
@@ -652,8 +727,13 @@ def shutdown(self) -> None:
         with self._sessions_lock:
             self._active = False
             session_ids = list(self._sessions)
+            owned_session_ids = set(self._owned_session_ids)
         for session_id in session_ids:
             self._safe(self.close_session, {"session_id": session_id})
+        for session_id in owned_session_ids:
+            self._safe(self.host.close_session, {"session_id": session_id})
+        with self._sessions_lock:
+            self._owned_session_ids.clear()
         if not self._registered:
             return
         try:
@@ -684,6 +764,7 @@ def deactivate(self) -> None:
             self._registered = False
         with self._sessions_lock:
             sessions = list(self._sessions.values())
+            owned_session_ids = set(self._owned_session_ids)
         for session in sessions:
             with session.lock:
                 if session.closing:
@@ -703,6 +784,9 @@ def deactivate(self) -> None:
                 self._end_pending_model_calls(session, {})
         with self._sessions_lock:
             self._sessions.clear()
+            self._owned_session_ids.clear()
+        for session_id in owned_session_ids:
+            self._safe(self.host.close_session, {"session_id": session_id})
         with self._task_sessions_lock:
             self._task_sessions.clear()
             self._turn_sessions.clear()
@@ -1165,6 +1249,52 @@ def prepare_session_start() -> None:
         _get_runtime(retry_failed=True)
 
 
+def start_setup_lifecycle(mode: str) -> SetupMetricsAttempt | None:
+    """Start one setup lifecycle only when collection is already allowed."""
+    if not enabled():
+        return None
+    runtime = _get_runtime(retry_failed=True)
+    if runtime is None:
+        return None
+    normalized_mode = mode if mode in SETUP_MODES else "unknown"
+    attempt = SetupMetricsAttempt(
+        session_id=f"hermes-shared-metrics-setup-{uuid.uuid4()}",
+        mode=normalized_mode,
+        _runtime=runtime,
+    )
+    if runtime._safe(runtime.record_setup_started, attempt) is not True:
+        runtime._safe(runtime.close_owned_session, attempt.session_id)
+        return None
+    return attempt
+
+
+def finish_setup_lifecycle(
+    attempt: SetupMetricsAttempt | None,
+    *,
+    outcome: str,
+    failure_stage: str = "none",
+) -> None:
+    """Finish and export one consented setup lifecycle without raw error data."""
+    if attempt is None:
+        return
+    runtime = attempt._runtime
+    if not enabled():
+        runtime._safe(runtime.close_owned_session, attempt.session_id)
+        return
+    normalized_outcome = outcome if outcome in SETUP_OUTCOMES else "failed"
+    normalized_stage = (
+        failure_stage if failure_stage in SETUP_FAILURE_STAGES else "unknown"
+    )
+    if normalized_outcome == "success":
+        normalized_stage = "none"
+    runtime._safe(
+        runtime.record_setup_finished,
+        attempt,
+        outcome=normalized_outcome,
+        failure_stage=normalized_stage,
+    )
+
+
 def _prepare_core_session(
     host: relay_runtime.RelayRuntime,
     context: dict[str, Any],