From f303c4006fa765ac653de52d768d269082e9761d Mon Sep 17 00:00:00 2001 From: Hector Martinez Date: Thu, 23 Jul 2026 16:09:28 +0200 Subject: [PATCH] feat(telemetry): forward OTEL env vars to all managed workflow agents Setting OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_HEADERS, and OTEL_RESOURCE_ATTRIBUTES now produces traces for all agents on your system (triage, code, review, fix, retro, prioritize, harness), including the ones dispatched via triggers. Previously only triage exported traces. The env vars are added to every agent step in reusable-dispatch.yml (per-repo) and all reusable-{stage}.yml files (per-org), with the secret forwarded through scaffold caller workflows. The distributed tracing guide is updated to reflect this, and the OTEL threading test is expanded to cover all stages. Closes #5528 Co-Authored-By: Claude Opus 4.6 Signed-off-by: Hector Martinez --- .github/workflows/reusable-code.yml | 5 ++++ .github/workflows/reusable-dispatch.yml | 24 ++++++++++++++--- .github/workflows/reusable-fix.yml | 5 ++++ .github/workflows/reusable-prioritize.yml | 5 ++++ .github/workflows/reusable-retro.yml | 5 ++++ .github/workflows/reusable-review.yml | 5 ++++ .../infrastructure/distributed-tracing.md | 16 +++++------- .../fullsend-repo/.github/workflows/code.yml | 1 + .../fullsend-repo/.github/workflows/fix.yml | 1 + .../.github/workflows/prioritize.yml | 1 + .../fullsend-repo/.github/workflows/retro.yml | 1 + .../.github/workflows/review.yml | 1 + .../scaffold/workflow_call_alignment_test.go | 26 +++++++++++++------ 13 files changed, 76 insertions(+), 20 deletions(-) diff --git a/.github/workflows/reusable-code.yml b/.github/workflows/reusable-code.yml index ace4401319..de71e554d9 100644 --- a/.github/workflows/reusable-code.yml +++ b/.github/workflows/reusable-code.yml @@ -46,6 +46,8 @@ on: required: true FULLSEND_GCP_PROJECT_ID: required: true + OTEL_EXPORTER_OTLP_TRACES_HEADERS: + required: false concurrency: group: fullsend-code-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -206,6 +208,9 @@ jobs: REPO_FULL_NAME: ${{ inputs.source_repo }} CODE_ALLOWED_TARGET_BRANCHES: '' TARGET_BRANCH: main + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: code version: ${{ inputs.fullsend_version }} diff --git a/.github/workflows/reusable-dispatch.yml b/.github/workflows/reusable-dispatch.yml index e4d2c7f07e..6c2e88edfe 100644 --- a/.github/workflows/reusable-dispatch.yml +++ b/.github/workflows/reusable-dispatch.yml @@ -66,9 +66,9 @@ on: required: true OTEL_EXPORTER_OTLP_TRACES_HEADERS: description: >- - OTLP headers for ADR 0050 Level 2 trace export, forwarded to the - triage stage (baggage-style k=v,k=v; may carry auth, hence a - secret). Optional — unset leaves export disabled unless the + OTLP headers for ADR 0050 Level 2 trace export, forwarded to all + agent stages (baggage-style k=v,k=v; may carry auth, hence a + secret). Optional; unset leaves export disabled unless the endpoint variable alone suffices. required: false @@ -744,6 +744,9 @@ jobs: REPO_FULL_NAME: ${{ github.repository }} CODE_ALLOWED_TARGET_BRANCHES: '' TARGET_BRANCH: main + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: code fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} @@ -853,6 +856,9 @@ jobs: PRIOR_REVIEW_SHA: ${{ steps.prior-review.outputs.prior_sha }} PRIOR_REVIEW_FILE: ${{ steps.prior-review.outputs.prior_review_file }} PRIOR_REVIEW_PROVENANCE: ${{ steps.prior-review.outputs.prior_review_provenance }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: review fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} @@ -1143,6 +1149,9 @@ jobs: FIX_ITERATION: ${{ steps.context.outputs.iteration }} REVIEW_BODY_FILE: ${{ steps.review-body.outputs.review_file }} PRE_AGENT_HEAD: ${{ steps.pre-agent.outputs.head }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: fix fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} @@ -1235,6 +1244,9 @@ jobs: RETRO_COMMENT: ${{ fromJSON(needs.route.outputs.event_payload).comment.body || '' }} REPO_FULL_NAME: ${{ github.repository }} MINT_REPOS: ${{ steps.workspace.outputs.repo_name != '' && (inputs.install_mode == 'per-repo' && steps.workspace.outputs.repo_name || format('{0},.fullsend', steps.workspace.outputs.repo_name)) || '' }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: retro fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} @@ -1312,6 +1324,9 @@ jobs: env: GITHUB_ISSUE_URL: ${{ fromJSON(needs.route.outputs.event_payload).issue.html_url }} REPO_FULL_NAME: ${{ github.repository }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: prioritize fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} @@ -1616,6 +1631,9 @@ jobs: env: GITHUB_ISSUE_URL: ${{ steps.dispatch-env.outputs.issue_url }} REPO_FULL_NAME: ${{ matrix.source_repo }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: ${{ matrix.agent }} version: ${{ inputs.fullsend_version }} diff --git a/.github/workflows/reusable-fix.yml b/.github/workflows/reusable-fix.yml index 1f480751a9..c8e27789fd 100644 --- a/.github/workflows/reusable-fix.yml +++ b/.github/workflows/reusable-fix.yml @@ -58,6 +58,8 @@ on: required: true FULLSEND_GCP_PROJECT_ID: required: true + OTEL_EXPORTER_OTLP_TRACES_HEADERS: + required: false concurrency: group: >- @@ -407,6 +409,9 @@ jobs: FIX_ITERATION: ${{ steps.context.outputs.iteration }} REVIEW_BODY_FILE: ${{ steps.review-body.outputs.review_file }} PRE_AGENT_HEAD: ${{ steps.pre-agent.outputs.head }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: fix version: ${{ inputs.fullsend_version }} diff --git a/.github/workflows/reusable-prioritize.yml b/.github/workflows/reusable-prioritize.yml index 5c12c1e8b2..1e5c9158e5 100644 --- a/.github/workflows/reusable-prioritize.yml +++ b/.github/workflows/reusable-prioritize.yml @@ -50,6 +50,8 @@ on: required: true FULLSEND_GCP_PROJECT_ID: required: true + OTEL_EXPORTER_OTLP_TRACES_HEADERS: + required: false concurrency: group: fullsend-prioritize-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -161,6 +163,9 @@ jobs: env: GITHUB_ISSUE_URL: ${{ fromJSON(inputs.event_payload).issue.html_url }} REPO_FULL_NAME: ${{ inputs.source_repo }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: prioritize version: ${{ inputs.fullsend_version }} diff --git a/.github/workflows/reusable-retro.yml b/.github/workflows/reusable-retro.yml index 654f9c600f..c98e3fac12 100644 --- a/.github/workflows/reusable-retro.yml +++ b/.github/workflows/reusable-retro.yml @@ -46,6 +46,8 @@ on: required: true FULLSEND_GCP_PROJECT_ID: required: true + OTEL_EXPORTER_OTLP_TRACES_HEADERS: + required: false concurrency: group: fullsend-retro-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} @@ -172,6 +174,9 @@ jobs: RETRO_COMMENT: ${{ fromJSON(inputs.event_payload).comment.body || '' }} REPO_FULL_NAME: ${{ inputs.source_repo }} MINT_REPOS: ${{ steps.repo-parts.outputs.name != '' && (inputs.install_mode == 'per-repo' && steps.repo-parts.outputs.name || format('{0},.fullsend', steps.repo-parts.outputs.name)) || '' }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: retro version: ${{ inputs.fullsend_version }} diff --git a/.github/workflows/reusable-review.yml b/.github/workflows/reusable-review.yml index 8ded5e0c24..32eea5c6b8 100644 --- a/.github/workflows/reusable-review.yml +++ b/.github/workflows/reusable-review.yml @@ -46,6 +46,8 @@ on: required: true FULLSEND_GCP_PROJECT_ID: required: true + OTEL_EXPORTER_OTLP_TRACES_HEADERS: + required: false concurrency: group: fullsend-review-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} @@ -188,6 +190,9 @@ jobs: PRIOR_REVIEW_SHA: ${{ steps.prior-review.outputs.prior_sha }} PRIOR_REVIEW_FILE: ${{ steps.prior-review.outputs.prior_review_file }} PRIOR_REVIEW_PROVENANCE: ${{ steps.prior-review.outputs.prior_review_provenance }} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} + OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} with: agent: review fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 0370550519..6ad5998b37 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -202,11 +202,9 @@ Fullsend-specific attributes: ### Managed workflows -Only the **triage** stage forwards OTEL configuration in this release; the -other agents (code, fix, review, retro, prioritize) do not export yet. - -To enable export for triage runs, set on the org (or repo) that hosts the -fullsend caller workflows: +All agent stages (triage, code, review, fix, retro, prioritize, harness) +forward OTEL configuration. To enable export, set on the org (or repo) +that hosts the fullsend caller workflows: 1. Actions **variable** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — the backend's full traces URL (e.g. `https://mlflow.example.com/v1/traces`). @@ -217,10 +215,10 @@ fullsend caller workflows: `k=v,k=v` trace tags. The value is used verbatim: `${{ github.* }}` expressions evaluate only in workflow YAML, not in variables. -Installations scaffolded before this release must also forward the secret -(add `OTEL_EXPORTER_OTLP_TRACES_HEADERS` under `secrets:`) until the scaffold -is re-synced: in the `.fullsend` repo's `triage.yml` (per-org), or in the -fullsend shim workflow's dispatch job (per-repo). +Installations scaffolded before OTEL support was added must also forward the +secret (add `OTEL_EXPORTER_OTLP_TRACES_HEADERS` under `secrets:`) until the +scaffold is re-synced: in the `.fullsend` repo's stage workflows (per-org), +or in the fullsend shim workflow's dispatch job (per-repo). ### Bring your own workflow diff --git a/internal/scaffold/fullsend-repo/.github/workflows/code.yml b/internal/scaffold/fullsend-repo/.github/workflows/code.yml index 3bf7144698..17406c5d05 100644 --- a/internal/scaffold/fullsend-repo/.github/workflows/code.yml +++ b/internal/scaffold/fullsend-repo/.github/workflows/code.yml @@ -41,3 +41,4 @@ jobs: secrets: FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} diff --git a/internal/scaffold/fullsend-repo/.github/workflows/fix.yml b/internal/scaffold/fullsend-repo/.github/workflows/fix.yml index 56c50840dc..3a10c9f5b9 100644 --- a/internal/scaffold/fullsend-repo/.github/workflows/fix.yml +++ b/internal/scaffold/fullsend-repo/.github/workflows/fix.yml @@ -65,3 +65,4 @@ jobs: secrets: FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} diff --git a/internal/scaffold/fullsend-repo/.github/workflows/prioritize.yml b/internal/scaffold/fullsend-repo/.github/workflows/prioritize.yml index 6aa1b5305b..fb4a7699bd 100644 --- a/internal/scaffold/fullsend-repo/.github/workflows/prioritize.yml +++ b/internal/scaffold/fullsend-repo/.github/workflows/prioritize.yml @@ -40,3 +40,4 @@ jobs: secrets: FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} diff --git a/internal/scaffold/fullsend-repo/.github/workflows/retro.yml b/internal/scaffold/fullsend-repo/.github/workflows/retro.yml index d9b38f7b57..f31d0104ef 100644 --- a/internal/scaffold/fullsend-repo/.github/workflows/retro.yml +++ b/internal/scaffold/fullsend-repo/.github/workflows/retro.yml @@ -46,3 +46,4 @@ jobs: secrets: FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} diff --git a/internal/scaffold/fullsend-repo/.github/workflows/review.yml b/internal/scaffold/fullsend-repo/.github/workflows/review.yml index 8883fc82be..7d3e61701b 100644 --- a/internal/scaffold/fullsend-repo/.github/workflows/review.yml +++ b/internal/scaffold/fullsend-repo/.github/workflows/review.yml @@ -40,3 +40,4 @@ jobs: secrets: FULLSEND_GCP_WIF_PROVIDER: ${{ secrets.FULLSEND_GCP_WIF_PROVIDER }} FULLSEND_GCP_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }} diff --git a/internal/scaffold/workflow_call_alignment_test.go b/internal/scaffold/workflow_call_alignment_test.go index 8071aaabc8..0cdacff9e6 100644 --- a/internal/scaffold/workflow_call_alignment_test.go +++ b/internal/scaffold/workflow_call_alignment_test.go @@ -274,6 +274,7 @@ func TestReusableWorkflowsShareCommonInputs(t *testing.T) { commonSecrets := []string{ "FULLSEND_GCP_WIF_PROVIDER", "FULLSEND_GCP_PROJECT_ID", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", } stages := []string{"triage", "code", "review", "fix", "retro", "prioritize"} @@ -320,10 +321,10 @@ func TestReusableDispatchProjectNumberInput(t *testing.T) { } // TestOTELHeadersSecretThreading validates that the optional OTLP headers -// secret (#2862) is forwarded along both installation-mode chains to -// reusable-triage.yml. TestWorkflowCallInputAlignment only enforces required -// secrets — an omitted optional forward silently arrives empty, which turns -// into a 401 at authenticated backends instead of failing loudly. +// secret (#2862) is forwarded along both installation-mode chains to every +// reusable stage workflow. TestWorkflowCallInputAlignment only enforces +// required secrets; an omitted optional forward silently arrives empty, +// which turns into a 401 at authenticated backends instead of failing loudly. func TestOTELHeadersSecretThreading(t *testing.T) { const forward = "OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}" @@ -331,13 +332,22 @@ func TestOTELHeadersSecretThreading(t *testing.T) { name string content func(t *testing.T) []byte }{ - // consumer: env injection on the agent step - {"reusable-triage.yml", loadRepoFile(".github/workflows/reusable-triage.yml")}, - // per-repo chain: shim → reusable-dispatch → reusable-triage + // per-repo chain: shim → reusable-dispatch (covers all inlined stages) {"scaffold/templates/shim-per-repo.yaml", loadScaffoldFile("templates/shim-per-repo.yaml")}, {"reusable-dispatch.yml", loadRepoFile(".github/workflows/reusable-dispatch.yml")}, - // per-org chain: thin caller → reusable-triage + // per-org chain: thin caller → reusable-{stage} + {"reusable-triage.yml", loadRepoFile(".github/workflows/reusable-triage.yml")}, {"scaffold/triage.yml", loadScaffoldFile(".github/workflows/triage.yml")}, + {"reusable-code.yml", loadRepoFile(".github/workflows/reusable-code.yml")}, + {"scaffold/code.yml", loadScaffoldFile(".github/workflows/code.yml")}, + {"reusable-review.yml", loadRepoFile(".github/workflows/reusable-review.yml")}, + {"scaffold/review.yml", loadScaffoldFile(".github/workflows/review.yml")}, + {"reusable-fix.yml", loadRepoFile(".github/workflows/reusable-fix.yml")}, + {"scaffold/fix.yml", loadScaffoldFile(".github/workflows/fix.yml")}, + {"reusable-retro.yml", loadRepoFile(".github/workflows/reusable-retro.yml")}, + {"scaffold/retro.yml", loadScaffoldFile(".github/workflows/retro.yml")}, + {"reusable-prioritize.yml", loadRepoFile(".github/workflows/reusable-prioritize.yml")}, + {"scaffold/prioritize.yml", loadScaffoldFile(".github/workflows/prioritize.yml")}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) {