feat(#2862): Level 2 distributed tracing — OTLP export to a backend

.github/workflows/reusable-dispatch.yml

@@ -65,6 +65,13 @@ on:
         required: false
       FULLSEND_GCP_PROJECT_ID:
         required: true
+      OTEL_EXPORTER_OTLP_TRACES_HEADERS:
+        description: >-
+          OTLP headers for ADR 0050 Level 2 trace export, forwarded to the
+          triage stage (baggage-style k=v,k=v; may carry auth, hence a
+          secret). Optional — unset leaves export disabled unless the
+          endpoint variable alone suffices.
+        required: false
 
 jobs:
   route:
@@ -453,6 +460,7 @@ jobs:
     secrets:
       FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }}
       FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }}
+      OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
 
   code:
     name: Code

.github/workflows/reusable-triage.yml

@@ -47,6 +47,12 @@ on:
         required: true
       FULLSEND_GCP_PROJECT_ID:
         required: true
+      OTEL_EXPORTER_OTLP_TRACES_HEADERS:
+        description: >-
+          OTLP headers for ADR 0050 Level 2 trace export (baggage-style
+          k=v,k=v; may carry auth, hence a secret). Optional — unset leaves
+          export disabled unless the endpoint variable alone suffices.
+        required: false
 
 concurrency:
   group: fullsend-triage-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }}
@@ -171,6 +177,12 @@ jobs:
         env:
           GITHUB_ISSUE_URL: ${{ fromJSON(inputs.event_payload).issue.html_url }}
           REPO_FULL_NAME: ${{ inputs.source_repo }}
+          # ADR 0050 Level 2: orgs opt into OTLP trace export by defining the
+          # endpoint as an Actions variable (and, when the backend needs auth
+          # or routing headers, the headers secret). Unset = export inert.
+          OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
+          OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
+          OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
         with:
           agent: triage
           version: ${{ inputs.fullsend_version }}

docs/guides/infrastructure/distributed-tracing.md

@@ -19,6 +19,21 @@ configuration required:
 These files are always written, even when no OTLP backend is configured. They
 contain metadata only — no prompts, completions, or source code content.
 
+## Prerequisites
+
+Level 1 requires nothing. To enable OTLP export (Level 2 and Level 3) you need:
+
+- An **OTLP/HTTP-capable backend** and its endpoint URL — e.g. Jaeger, Tempo,
+  Grafana, MLflow ≥ 3.6, or any OpenTelemetry Collector.
+- Any **backend authentication** (bearer token or basic auth) for the
+  `OTEL_EXPORTER_OTLP_TRACES_HEADERS` variable.
+- **Network reachability** from where runs execute (your machine or CI runners)
+  to the backend endpoint.
+- For a backend behind a **private CA** (e.g. an internal MLflow): the CA
+  certificate bundle, pointed to by `OTEL_EXPORTER_OTLP_CERTIFICATE`. Local
+  and bring-your-own-workflow runs only — the managed workflows do not yet
+  pass a CA bundle through.
+
 ## Enabling OTLP export (Level 2)
 
 To send metadata spans to an OpenTelemetry-compatible backend, set one of the
@@ -45,8 +60,59 @@ Langfuse, SigNoz, Honeycomb, Datadog, etc.
 If the endpoint is unreachable, the CLI continues normally — local files are
 still produced and the run is not affected.
 
+Operational details:
+
+- **Export timing:** spans are exported once, when the run closes, inside a
+  hard wall-clock budget (5 seconds). There is no mid-run network traffic; a
+  dead endpoint costs at most the budget and one warning line.
+- **Crashed runs are not exported:** export replays the finalized artifacts,
+  so a run that never finalizes (crash, OOM, SIGKILL) writes no
+  `run-summary.json` and exports nothing. Its `run-telemetry.jsonl` remains
+  the local forensic record.
+- **Sampling:** when the run continues an inbound `TRACEPARENT` whose W3C
+  sampled flag is unset (`-00`), the upstream sampling decision is respected:
+  nothing is exported. Local files are always written regardless.
+- **Protocol:** OTLP over `http/protobuf` only. Setting
+  `OTEL_EXPORTER_OTLP_PROTOCOL` (or the traces-specific variant) to anything
+  else — e.g. `grpc` — skips export with a warning rather than posting
+  protobuf at a gRPC endpoint.
+- **Validation:** a malformed endpoint value skips export with a warning; it
+  is never silently replaced with the SDK's `localhost:4318` default.
+- **Kill switches:** `OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none`
+  are honored.
+- **Private CAs:** point `OTEL_EXPORTER_OTLP_CERTIFICATE` at a PEM bundle for
+  backends with certificates outside the system trust store. There is no
+  skip-verify option.
+
+### MLflow example
+
+MLflow ≥ 3.6 ingests OTLP/HTTP natively at `{server}/v1/traces` and routes
+traces to an experiment via a required header:
+
+```bash
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://mlflow.example.com/v1/traces"
+export OTEL_EXPORTER_OTLP_TRACES_HEADERS="x-mlflow-experiment-id=42"
+```
+
+Header values are URL-decoded, so spaces are percent-encoded — for a
+Basic-auth-fronted instance:
+
+```bash
+export OTEL_EXPORTER_OTLP_TRACES_HEADERS="authorization=Basic%20${CREDS_B64},x-mlflow-experiment-id=42"
+```
+
+> **Cost columns:** MLflow's per-trace cost is its own estimate — extracted
+> input/output token counts priced against MLflow's internal model table. It
+> excludes cache-creation/cache-read tokens, which dominate agent-run cost.
+> The authoritative figure is the runtime-reported `fullsend.cost_usd` on
+> `agent` spans (also in `run-summary.json`).
+
 ## Enabling content capture (Level 3)
 
+> **Planned:** Level 3 content capture is not yet implemented. This section
+> documents the contract decided in
+> [ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md).
+
 By default, spans contain metadata only (timing, token counts, tool names,
 errors). To include full prompt/completion content in spans:
 
@@ -92,91 +158,140 @@ consumers (scripts, other agents) can continue the trace chain.
 
 ## Span structure
 
-A typical agent run produces this span hierarchy:
+A run produces this span hierarchy (span names match the `name` field in
+`run-telemetry.jsonl` — the exported spans and the local file are two views
+of the same trace, with identical span ids):
 
 ```
-fullsend-run (root, SpanKind=Consumer if dispatched)
-├── load-harness
-├── setup-sandbox
-│   └── create-sandbox (gen_ai.operation.name=create_agent)
-├── agent-execution.iteration-0
-│   └── (gen_ai.operation.name=invoke_agent)
-├── agent-execution.iteration-1
-├── collect-artifacts
-├── security-scan
-└── validation
+run (root; Consumer when dispatched with TRACEPARENT, else Internal)
+├── sandbox_create (gen_ai.operation.name=create_agent)
+└── agent           (one per iteration; gen_ai.operation.name=invoke_agent)
 ```
 
 ### GenAI semantic conventions
 
-Root and iteration spans carry [OTEL GenAI semantic convention](https://opentelemetry.io/docs/specs/semconv/gen-ai/) attributes:
+Spans carry [OTEL GenAI semantic convention](https://opentelemetry.io/docs/specs/semconv/gen-ai/) attributes:
 
-| Attribute | Example | Description |
-|-----------|---------|-------------|
-| `gen_ai.operation.name` | `invoke_agent` | The GenAI operation type |
-| `gen_ai.agent.name` | `triage` | The agent being executed |
-| `gen_ai.request.model` | `claude-sonnet-4-20250514` | The model configured in the harness |
-| `gen_ai.system` | `anthropic` | The LLM provider |
+| Attribute | Example | On |
+|-----------|---------|-----|
+| `gen_ai.operation.name` | `invoke_agent` | `run` and `agent` spans (`create_agent` on `sandbox_create`) |
+| `gen_ai.agent.name` | `triage` | `run` and `agent` spans |
+| `gen_ai.request.model` | `claude-opus-4-6` | `agent` spans (resolved model) |
+| `gen_ai.system` | `anthropic` | `agent` spans (the model vendor, from the runtime) |
+| `gen_ai.usage.input_tokens` / `output_tokens` / `cache_*_input_tokens` | `109938` | `agent` spans |
 
 These attributes enable LLM-aware backends to recognize fullsend spans as
 agent operations and surface them in GenAI-specific dashboards.
 
 ### SpanKind
 
-- **Consumer**: The root span when `TRACEPARENT` is set (the run was
-  dispatched by an external system).
-- **Internal**: The root span for local/manual invocations.
+- **Consumer**: The root span when a valid inbound `TRACEPARENT` was adopted
+  (the run was dispatched by an instrumented system).
+- **Internal**: The root span for local/manual invocations, and all child
+  spans.
 
 ## Custom attributes
 
-Every span also carries fullsend-specific attributes:
+Fullsend-specific attributes:
+
+| Attribute | On | Description |
+|-----------|-----|-------------|
+| `fullsend.work_item_id` | every span | Work item identity (e.g. `owner/repo#123`) — the primary cross-run correlation key |
+| `fullsend.cost_usd` | `agent` spans | Iteration cost in USD, rounded to cents |
+| `fullsend.tool_calls` | `agent` spans | Tool invocations in the iteration |
+| `agent` | `run` span | Agent name (predates `gen_ai.agent.name`; kept for Level 1 consumers) |
+
+## GHA workflow configuration
 
-| Attribute | Description |
-|-----------|-------------|
-| `fullsend.agent` | Agent name from the harness |
-| `fullsend.harness` | Path to the harness YAML |
-| `fullsend.model` | Model identifier |
-| `fullsend.image` | Container image used |
-| `fullsend.work_item_id` | Issue/PR number being addressed |
+### Managed workflows
 
-## GHA workflow configuration example
+Only the **triage** stage forwards OTEL configuration in this release; the
+other agents (code, fix, review, retro, prioritize) do not export yet.
 
-Add these environment variables to workflow jobs that run `fullsend run`:
+To enable export for triage runs, set on the org (or repo) that hosts the
+fullsend caller workflows:
+
+1. Actions **variable** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — the backend's
+   full traces URL (e.g. `https://mlflow.example.com/v1/traces`).
+2. Actions **secret** `OTEL_EXPORTER_OTLP_TRACES_HEADERS` — the complete
+   header string, auth and routing included (e.g.
+   `Authorization=Bearer%20<token>,x-mlflow-experiment-id=42`).
+3. Optional: Actions **variable** `OTEL_RESOURCE_ATTRIBUTES` — static
+   `k=v,k=v` trace tags. The value is used verbatim: `${{ github.* }}`
+   expressions evaluate only in workflow YAML, not in variables.
+
+Installations scaffolded before this release must also forward the secret
+(add `OTEL_EXPORTER_OTLP_TRACES_HEADERS` under `secrets:`) until the scaffold
+is re-synced: in the `.fullsend` repo's `triage.yml` (per-org), or in the
+fullsend shim workflow's dispatch job (per-repo).
+
+### Bring your own workflow
+
+Add the environment variables to any job that runs `fullsend run`:
 
 ```yaml
 env:
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "${{ secrets.OTLP_ENDPOINT }}"
-  OTEL_EXPORTER_OTLP_TRACES_HEADERS: "Authorization=Bearer ${{ secrets.OTLP_TOKEN }}"
+  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}"
+  OTEL_EXPORTER_OTLP_TRACES_HEADERS: "${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}"
 ```
 
-The secret names and values depend on your chosen backend. Consult your
-backend's documentation for the endpoint URL and authentication mechanism.
+Any variable and secret names work here — the values reach the exporter
+as-is. Consult your backend's documentation for the endpoint URL and
+authentication mechanism.
+
+### Organizing traces for an org
+
+Two conventions keep a shared backend navigable as repos onboard:
+
+1. **One backend bucket per org.** On MLflow, create one experiment per org
+   (e.g. `fullsend-<org>`) and point the org's header secret at its id. The
+   backend's per-bucket access controls then align with org boundaries.
+2. **Slice inside the bucket with resource attributes.** Standard OTel
+   resource env is honored, so workflows can tag every trace with repo,
+   agent, and environment:
+
+   ```yaml
+   env:
+     OTEL_RESOURCE_ATTRIBUTES: "fullsend.repo=${{ github.repository }},fullsend.agent=triage,deployment.environment=prod"
+   ```
+
+   The example is inline workflow `env:`, where `${{ github.* }}` evaluates.
+   On the managed path, set the `OTEL_RESOURCE_ATTRIBUTES` Actions variable
+   to a static value instead — variables are not expression-expanded.
+
+   These become filterable trace tags (enable them as columns in MLflow's
+   Traces table). `fullsend.work_item_id` is already on every span, so runs
+   for the same issue correlate without configuration.
 
 ## Local development
 
 Run an agent locally with traces going to a local backend:
 
-```bash
-# Start a local Jaeger instance (OTLP-compatible)
-podman run -d --name jaeger \
-  -p 16686:16686 \
-  -p 4318:4318 \
-  jaegertracing/jaeger
+1. Start a local Jaeger instance (OTLP-compatible):
 
-# Run an agent with tracing enabled
-export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
-fullsend run triage --issue 42
+   ```bash
+   podman run -d --name jaeger \
+     -p 16686:16686 \
+     -p 4318:4318 \
+     jaegertracing/jaeger
+   ```
 
-# View traces at http://localhost:16686
-```
+2. Point the exporter at it and run an agent:
+
+   ```bash
+   export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+   fullsend run triage --issue 42
+   ```
+
+3. View the traces at <http://localhost:16686>.
 
 Other lightweight local backends:
 
 | Backend | Command | UI |
 |---------|---------|-----|
 | Jaeger | `podman run -p 16686:16686 -p 4318:4318 jaegertracing/jaeger` | `localhost:16686` |
 | Arize Phoenix | `podman run -p 6006:6006 -p 4318:4318 arizephoenix/phoenix` | `localhost:6006` |
-| MLflow | `uvx mlflow server` (with OTLP plugin) | `localhost:5000` |
+| MLflow ≥ 3.6 | `uvx "mlflow>=3.6" server --backend-store-uri sqlite:///mlflow.db` (native OTLP at `/v1/traces`; requires the `x-mlflow-experiment-id` header — see the MLflow example above) | `localhost:5000` |
 
 ## Other backends
 

go.mod

@@ -9,6 +9,11 @@ require (
 	github.com/knights-analytics/hugot v0.7.5
 	github.com/spf13/cobra v1.10.2
 	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/otel v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0
+	go.opentelemetry.io/otel/sdk v1.44.0
+	go.opentelemetry.io/otel/trace v1.44.0
+	go.opentelemetry.io/proto/otlp v1.10.0
 	golang.org/x/crypto v0.52.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/term v0.43.0
@@ -17,16 +22,27 @@ require (
 )
 
 require (
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect
 	github.com/cucumber/messages/go/v21 v21.0.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-memdb v1.3.4 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/grpc v1.81.1 // indirect
 )
 
 require (
-	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/colorprofile v0.4.3 // indirect
 	github.com/charmbracelet/x/ansi v0.11.6 // indirect
@@ -64,7 +80,7 @@ require (
 	golang.org/x/image v0.41.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
+	google.golang.org/protobuf v1.36.11
 	k8s.io/klog/v2 v2.140.0 // indirect
 )