feat(harness): trace correlation via OTel baggage + iii.* span attrs

.gitignore

@@ -29,6 +29,8 @@ __pycache__/
 
 # Local runtime state
 .omx/
+.serena/
+docs/
 data/
 *.pid
 *.zip

harness/ARCHITECTURE.md

@@ -94,23 +94,21 @@ The harness owns *no* logic from any of these — it only knows their names. Eac
 
 `EXPECTED_WORKERS` is generated from `iii.worker.yaml` at build time by `build.rs`, so the two cannot drift — there is no separate sync test to maintain.
 
-### 4. `scripts/demo.sh` — local orchestration
+### 4. `Makefile` — local orchestration
 
-For registry-based installs, `iii worker add harness` fetches the harness binary and its declared dependencies automatically (see `registry/index.json`). `scripts/demo.sh` is the alternative path for local development from a source checkout:
+For registry-based installs, `iii worker add harness` fetches the harness binary and its declared dependencies automatically (see `registry/index.json`). The `Makefile` is the alternative path for local development from a source checkout:
 
 ```
-demo.sh build    # cargo build --release for harness + dep workers
-demo.sh engine   # start `iii --use-default-config` in background
-demo.sh start    # spawn all workers + harness as nohup processes
-demo.sh verify   # call harness::status, models::list, provider::cli::list_models
-demo.sh web      # npm install + vite in a tmux session
-demo.sh stop     # kill every PID in $DEMO_DIR/pids/ + engine + tmux
-demo.sh all      # build + engine + start + verify
+make config         # generate config.yaml + iii.lock via `iii worker add .`
+make observability  # add iii-observability to config.yaml (powers TRACES tab)
+make engine         # start `iii --config config.yaml` in background
+make verify         # call harness::status + models::list
+make web            # vite dev server on :5173
+make stop           # kill engine + web
+make all            # config + observability + engine + verify
 ```
 
-PIDs and logs live under `$DEMO_DIR` (default `~/iii-harness-demo`). One PID file per worker, one log file per worker — no shared logger, no daemon supervisor.
-
-`scripts/real-usage.sh` exercises the running stack end-to-end: `auth::set_token` → `run::start_and_wait` → `state::get` for both messages and turn record → `state::list` to enumerate sessions.
+PIDs and logs live under `$DEMO_DIR` (default `~/iii-harness-demo`). The engine spawns each worker via its `iii.worker.yaml` `scripts.start`.
 
 ## Runtime data flow
 

harness/Cargo.toml

@@ -20,7 +20,7 @@ name = "harness"
 path = "src/main.rs"
 
 [dependencies]
-iii-sdk = "=0.11.3"
+iii-sdk = "=0.11.7-next.3"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
@@ -33,6 +33,7 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
 
 [dev-dependencies]
 harness-types = { path = "crates/harness-types" }
+uuid = { version = "1", features = ["v4"] }
 serial_test = "3"
 which = "8"
 

harness/Makefile

@@ -13,8 +13,10 @@
 #
 # Usage:
 #   make                # help
-#   make all            # config + engine + verify
+#   make all            # build + config + observability + engine + verify
+#   make build          # cargo build --release + symlink into ~/.iii/workers/
 #   make config         # (re)generate config.yaml + iii.lock via `iii worker add .`
+#   make observability  # iii worker add iii-observability (powers TRACES tab)
 #   make engine         # start `iii` in background reading harness/config.yaml
 #   make verify         # call harness::status + models::list
 #   make web            # background vite dev server on :5173 (no tmux)
@@ -31,22 +33,48 @@ SHELL := bash
 MAKEFLAGS += --no-print-directory
 
 HARNESS_DIR    := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+WORKERS_REPO   := $(abspath $(HARNESS_DIR)/..)
+WORKERS_BIN    := $(HOME)/.iii/workers
 DEMO_DIR       ?= $(HOME)/iii-harness-demo
 DEMO_ENGINE_WS ?= $(or $(III_DEMO_ENGINE_URL),ws://127.0.0.1:49134)
 PIDS_DIR       := $(DEMO_DIR)/pids
 LOGS_DIR       := $(DEMO_DIR)/logs
 
 CONFIG_FILE := $(HARNESS_DIR)/config.yaml
 
-.PHONY: help all config engine verify web stop restart logs clean ensure-dirs
+# Local worker crates that need to be cargo-built and symlinked into
+# ~/.iii/workers/ so the engine spawns them on host instead of trying
+# `cargo run` inside libkrun (where cargo isn't installed).
+LOCAL_WORKERS := approval-gate auth-credentials hook-fanout iii-directory \
+                 llm-budget models-catalog policy-denylist provider-anthropic \
+                 provider-openai provider-router session shell turn-orchestrator \
+                 harness
+
+.PHONY: help all build config observability engine verify web stop restart logs clean ensure-dirs
 
 help:
 	@awk '/^[^#]/ && !/^$$/ {exit} /^#/ {sub(/^# ?/, ""); print}' $(firstword $(MAKEFILE_LIST))
 
-all: config engine verify
+all: build config observability engine verify
 
 ensure-dirs:
-	@mkdir -p $(PIDS_DIR) $(LOGS_DIR) $(HARNESS_DIR)/data/skills
+	@mkdir -p $(PIDS_DIR) $(LOGS_DIR) $(HARNESS_DIR)/data/skills $(WORKERS_BIN)
+
+# ─── build ───────────────────────────────────────────────────────────────────
+
+# Cargo-builds every local worker in release mode and symlinks each binary
+# into ~/.iii/workers/<name>. The engine looks up workers by that path;
+# symlinks let us iterate on source without re-running `iii worker add`.
+# Each worker is its own cargo workspace (no shared top-level workspace),
+# so we loop and `cargo build --release` per crate.
+build: ensure-dirs
+	@for w in $(LOCAL_WORKERS); do \
+	  echo "==> cargo build --release: $$w"; \
+	  ( cd "$(WORKERS_REPO)/$$w" && cargo build --release --quiet ) \
+	    || { echo "    [error] $$w: cargo build failed"; exit 1; }; \
+	  ln -sf "$(WORKERS_REPO)/$$w/target/release/$$w" "$(WORKERS_BIN)/$$w"; \
+	done
+	@echo "==> all $(words $(LOCAL_WORKERS)) workers built and symlinked into $(WORKERS_BIN)"
 
 # ─── config ──────────────────────────────────────────────────────────────────
 
@@ -62,6 +90,18 @@ $(CONFIG_FILE): $(HARNESS_DIR)/iii.worker.yaml
 	@echo "==> generating $@ via iii worker add"
 	@cd "$(HARNESS_DIR)" && iii worker add . --no-wait
 
+# ─── observability ───────────────────────────────────────────────────────────
+
+# Adds iii-observability to config.yaml. Powers the iii Developer Console
+# TRACES tab and `engine::traces::*` query path. Kept OUT of
+# iii.worker.yaml `dependencies:` (and thus EXPECTED_WORKERS) so prod stacks
+# without observability don't flag missing in `harness::status`. `iii worker
+# add` is idempotent — re-running is a no-op.
+observability: config
+	@command -v iii >/dev/null || { echo "iii CLI not found"; exit 1; }
+	@echo "==> iii worker add iii-observability (idempotent)"
+	@cd "$(HARNESS_DIR)" && iii worker add iii-observability --no-wait
+
 # ─── engine ──────────────────────────────────────────────────────────────────
 
 engine: ensure-dirs config

harness/README.md

@@ -1,31 +1,19 @@
 # harness
 
-Meta-worker that composes the modular workers behind a runnable iii chat
-surface and exposes the browser-facing HTTP bridge (`harness::call`)
-the bundled Vite/React UI talks to. The harness does
-not own chat, agent, or provider logic — it registers a small set of
-bus functions and expects peers such as
-[`turn-orchestrator`](../turn-orchestrator),
-[`provider-router`](../provider-router), shell tools, and related
-workers to be installed alongside it. `iii worker add harness` pulls
-the whole bundle in transitively.
+Meta-worker that composes fifteen modular workers into a runnable iii chat surface, exposes a browser-facing HTTP bridge (`bridge::trigger`, `bridge::events`), and ships a Vite/React UI that talks to the bus through it. The harness does not own chat, agent, or provider logic; it registers a small set of bus functions and expects peers such as `turn-orchestrator`, `provider-router`, shell tools, and related workers to be installed alongside it. Deeper layout and streams behavior are documented in [`ARCHITECTURE.md`](ARCHITECTURE.md).
 
 ## Install
 
 ```bash
 iii worker add harness
 ```
 
-`iii worker add` fetches the binary, writes a config block into
-`~/.iii/config.yaml`, resolves every transitive worker from
-`iii.worker.yaml` `dependencies:`, and the engine starts the bundle on
-the next `iii start`.
+`iii worker add` fetches the binary, writes a config block into `~/.iii/config.yaml`, and the engine starts the worker on the next `iii start`.
 
-To back chat history with durable SQL storage instead of the bundled
-in-memory `iii-state`, add the [`iii-database`](../iii-database) worker:
+To register the harness skill bundle metadata with the bus (the worker does this automatically at boot when `skills` is available), ensure the [skills](../skills) worker is part of your stack:
 
 ```bash
-iii worker add iii-database
+iii worker add skills
 ```
 
 ## Quickstart
@@ -54,12 +42,13 @@ async fn main() -> anyhow::Result<()> {
 }
 ```
 
-Forward an arbitrary bus call through the HTTP-oriented bridge:
+Forward an arbitrary bus call through the HTTP-oriented bridge (same shape as `bridge::trigger` on the engine):
 
 ```rust
+// function_id / payload match iii.trigger(...)
 let result = iii
     .trigger(TriggerRequest {
-        function_id: "harness::call".into(),
+        function_id: "bridge::trigger".into(),
         payload: json!({
             "function_id": "models::list",
             "payload": {},
@@ -70,23 +59,138 @@ let result = iii
     .await?;
 ```
 
-Registered functions:
+Registered functions (use `::` ids on the bus):
 
 | Function | Role |
 |---|---|
 | `harness::status` | Bundle name, version, and expected worker list (cheap liveness probe). |
-| `harness::call` | Forwards `{ function_id, payload }` to `iii.trigger`. HTTP: `POST harness/call`. |
+| `bridge::trigger` | Forwards `{ function_id, payload }` to `iii.trigger`. HTTP: `POST` `bridge/trigger`. |
+| `bridge::events` | SSE-style tail of `agent::events` for a session. HTTP: `GET` `bridge/events`. |
 
-`harness::call` is the browser's call-anything escape hatch — not
-meant as an LLM tool.
+`bridge::trigger` is not meant as an LLM tool — it is the browser’s call-anything escape hatch.
 
 ## Configuration
 
 ```yaml
-engine_url: "ws://127.0.0.1:49134"   # WebSocket URL when III_URL / --url are unset
+# Default engine WebSocket URL when III_URL / --url are unset
+engine_url: "ws://127.0.0.1:49134"
 ```
 
-Runtime flags:
+Other runtime flags:
 
 - `--config` — path to this file (default `./config.yaml`; override with `III_HARNESS_CONFIG`).
-- `--url` / `III_URL` — engine WebSocket URL; wins over `engine_url` in the file.
+- `--url` or `III_URL` — engine WebSocket URL; wins over `engine_url` in the file.
+
+Registry-facing defaults also appear in `iii-harness --manifest` under `default_config`.
+
+## Expected workers
+
+`EXPECTED_WORKERS` (in [`src/lib.rs`](src/lib.rs)) is generated at build time
+from the `dependencies:` block of [`iii.worker.yaml`](iii.worker.yaml) by
+[`build.rs`](build.rs). Add or remove a worker by editing `iii.worker.yaml`
+only — the Rust constant rebuilds automatically.
+
+## Trace correlation
+
+Every harness-registered function wraps its body in an OTel span tagged with
+`iii.session.id`, `iii.message.id`, and (for `bridge::trigger` only)
+`iii.function.id`. The HTTP response carries two new headers when
+observability is active:
+
+- `traceparent: 00-<trace_id>-<span_id>-01` — W3C trace context for the span
+  that wrapped this call.
+- `x-iii-message-id: <id>` — the `message_id` you sent on the request, or
+  the upstream value propagated via OTel baggage. **Omitted entirely** when
+  neither source supplied one. This keeps plumbing calls (UI subscribes,
+  status polls, engine-internal traffic) out of `Group by message` in the
+  console — only real chat-turn IDs land there.
+
+Discover harness traces in the iii Developer Console TRACES tab, or via:
+
+```bash
+# By span name (any harness function):
+iii trigger --function-id engine::traces::list \
+  --payload '{"name":"harness.status","search_all_spans":true}'
+
+# By message_id directly (engine v0.11.7+ — needs the search_all_spans
+# attribute-filter widening + iii-sdk BaggageSpanProcessor; works on
+# every span in the trace, not just the harness-wrapped one):
+iii trigger --function-id engine::traces::list \
+  --payload '{"attributes":[["iii.message.id","<msg-id>"]],"search_all_spans":true}'
+
+# Server-side aggregation (engine v0.11.7+):
+iii trigger --function-id engine::traces::group_by \
+  --payload '{"attribute":"iii.message.id"}'
+```
+
+Both headers are absent when the iii-observability worker is not running
+(see `harness/config.yaml`). Web clients should treat them as optional —
+"`traceparent` absent" means "observability is off," not "the call failed."
+
+#### Operator observability of the wrapper itself
+
+The wrapper emits a `tracing::trace!` event per span entry with `fn_name`,
+`recording` (whether OTel is active), `session_id`, and `message_id_minted`
+(always `false` since the harness no longer mints; kept for log-format
+stability).
+Tail with `RUST_LOG=harness::otel=trace` to detect the
+"observability worker went silent" failure mode (rising `recording=false`
+rate without an OTel runtime change).
+
+#### Baggage propagation (and why TRACES doesn't group by message_id yet)
+
+The wrapper also writes `iii.session.id`, `iii.message.id`, and (for
+`bridge::trigger` only) `iii.function.id` into the OTel **baggage** of the
+context attached around the handler. Every downstream `iii.trigger(...)`
+call ships the baggage on the wire automatically (iii-sdk's `inject_baggage`
+is wired into the invocation message at `iii-sdk/src/iii.rs:312`). Receiving
+workers extract it via `extract_context(traceparent, baggage)` and the
+entries live in their task-local OTel context for the duration of the
+handler.
+
+What this does NOT do yet: **baggage entries are not automatically copied
+onto span attributes** of downstream worker spans. The OTel SDK requires an
+explicit `SpanProcessor` that reads baggage on `on_start` and writes it to
+the span as attributes; none exists in `iii-observability` today. So in the
+iii Developer Console TRACES tab, downstream spans (e.g. `state::set`,
+`approval::list_pending`) still appear without `iii.message.id` even though
+the baggage *is* travelling alongside them.
+
+Required engine-side follow-up to make TRACES group by message:
+
+1. Add a span processor in `iii-observability` that copies a configurable
+   allowlist of baggage keys onto each span at start time (allowlist defaults
+   to `iii.session.id`, `iii.message.id`, `iii.function.id`).
+2. Optionally extend `engine::traces::list` so `search_all_spans: true` also
+   applies the attribute filter (currently root-only — documented above).
+3. Optionally, surface a "group by attribute" affordance in the TRACES tab.
+
+Once (1) lands, every span in the trace inherits the ids automatically. The
+harness side is forward-compatible: the baggage is already flowing.
+
+### Direct-bus return shapes
+
+Two functions return the HTTP-trigger envelope `{status_code, headers, body}`:
+`bridge::trigger` and `bridge::events`. The other five — `harness::status`,
+`bridge::info`, `ui::subscribe`, `ui::unsubscribe`, `harness::fs::read_inline`
+— return their raw payloads so direct-WebSocket callers (the web `StatusPill`,
+`fetchBridgeInfo`, `ui::subscribe` registration, FilesystemPanel reads) can
+read fields off the top level. The OTel span still fires with `iii.*`
+attributes for all seven; only the HTTP `traceparent` / `x-iii-message-id`
+header echo is skipped for the five raw-shape functions (their wrapper sees
+no `status_code` in the return and leaves headers untouched).
+
+> **Contract reminder.** Any change to a wrapped function's return shape
+> (envelope ↔ raw) is a breaking change for direct-WS consumers in
+> `harness/web/`. Commit `767c83d` reverted four functions from envelope back
+> to raw after the unified-envelope rollout broke `StatusPill.tsx` at
+> runtime. The wrapper variants document the contract at the call site —
+> `with_envelope_span` for `bridge::trigger`/`bridge::events`, `with_raw_span`
+> for the other five. Keep them aligned with their consumers.
+
+### Wildcard subscriptions
+
+`ui::subscribe` / `ui::unsubscribe` accept `session_id: null` to mean "all
+sessions." In TRACES those calls show up with `iii.session.id = "*"`.
+
+Contributor commands (fmt, clippy, tests) for this crate live in [`binary-worker.md`](../binary-worker.md) §11; source layout notes are in [`ARCHITECTURE.md`](ARCHITECTURE.md).