From 600f798a2a3d97d65771ac97e8be6a12cdfe37f3 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Wed, 29 Jul 2026 14:11:10 +0200 Subject: [PATCH 1/2] feat(ci): agentic VCR capture mode for the demo corpus workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agentic=vcr layers the demo 3.0+ agent/MCP/chatbot services (compose.agent.yaml, prebuilt ghcr images) in VCR replay mode — the demo's .env defaults already point at the shipped azure/gpt-5.5 cassette with no API key — gates readiness on the agent, and drives the three cassette-backed prompts through the capture window. The LLM calls replay from cassettes; the Traceloop/OTel instrumentation around them is live, so the corpus gains real gen_ai.* semconv telemetry at zero cost with no secrets. The manifest gains a gen_ai.* key census so a capture proves (or disproves) the GenAI content at a glance. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Qtny6z6cA74xPZa4qRhk4F Signed-off-by: Jens Holdgaard Pedersen --- .../workflows/capture-otel-demo-corpus.yml | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/.github/workflows/capture-otel-demo-corpus.yml b/.github/workflows/capture-otel-demo-corpus.yml index d0a9435d8..623bace2e 100644 --- a/.github/workflows/capture-otel-demo-corpus.yml +++ b/.github/workflows/capture-otel-demo-corpus.yml @@ -55,6 +55,15 @@ on: description: Space/comma-separated flagd failure flags to force on (e.g. adFailure paymentFailure) — empty leaves the demo's defaults default: '' required: false + agentic: + description: >- + 'vcr' layers the demo's agent/MCP/chatbot services (3.0+, + compose.agent.yaml) in VCR replay mode — no LLM key, recorded + responses, REAL gen_ai.* telemetry — and drives the three + cassette-backed prompts through the capture window. Empty + skips the agent layer (the pre-3.0 behaviour). + default: '' + required: false release_tag: description: If set, publish the corpus as assets on a GitHub release at this tag (e.g. corpus/otel-demo-v1) default: '' @@ -135,6 +144,8 @@ jobs: # (resolved from the `demo` working-dir); the override is # absolute so it's project-dir-independent. - name: Resolve the demo's base compose file + env: + AGENTIC: ${{ inputs.agentic }} run: | set -euxo pipefail if [ -f demo/compose.yaml ]; then @@ -145,7 +156,20 @@ jobs: echo "::error::neither compose.yaml nor docker-compose.yml in the demo clone" exit 1 fi - echo "COMPOSE_FILE=${base}:${GITHUB_WORKSPACE}/.github/otel-demo-capture-compose-override.yml" \ + layers="${base}" + # The agent layer (3.0+) rides between the base and our + # override so the override's collector mount still wins. + if [ "$AGENTIC" = "vcr" ]; then + if [ ! -f demo/compose.agent.yaml ]; then + echo "::error::agentic=vcr but this demo ref has no compose.agent.yaml (needs 3.0+)" + exit 1 + fi + layers="${layers}:compose.agent.yaml" + elif [ -n "$AGENTIC" ]; then + echo "::error::unknown agentic mode '$AGENTIC' (only 'vcr' is supported)" + exit 1 + fi + echo "COMPOSE_FILE=${layers}:${GITHUB_WORKSPACE}/.github/otel-demo-capture-compose-override.yml" \ >> "$GITHUB_ENV" - name: Inject the file/corpus collector overlay @@ -240,6 +264,8 @@ jobs: - name: Wait for the traffic path to be ready working-directory: demo + env: + AGENTIC: ${{ inputs.agentic }} run: | set -euxo pipefail # We bypass frontend-proxy (Envoy) — its baked image @@ -266,6 +292,12 @@ jobs: } want_running otel-collector want_running frontend + # The agent layer, when present: VCR replay needs no LLM + # backend, but the service itself must be up before the + # prompt driver starts. + if [ "$AGENTIC" = "vcr" ]; then + want_running agent + fi # frontend-proxy is bypassed and crash-loops on its bad # baked config; stop it so it stops burning runner CPU # and restart-spamming. Nothing else depends on it (it @@ -282,6 +314,7 @@ jobs: env: WARMUP: ${{ inputs.warmup_seconds }} DURATION: ${{ inputs.duration_seconds }} + AGENTIC: ${{ inputs.agentic }} run: | set -euxo pipefail raw="$OURIOS_CAPTURE_DIR/logs.raw.jsonl" @@ -297,7 +330,39 @@ jobs: start_line="$(wc -l < "$raw" 2>/dev/null || echo 0)" echo "CAPTURE_START_LINE=${start_line}" >> "$GITHUB_ENV" echo "steady-state window starts at raw line ${start_line}; capturing ${DURATION}s" + driver_pid="" + if [ "$AGENTIC" = "vcr" ]; then + # Drive the agent with the three cassette-backed prompts + # (src/agent/README.md "Default Requests") for the whole + # window: VCR replays the recorded LLM responses, the + # Traceloop/OTel instrumentation around them is live, so + # every loop emits real gen_ai.* telemetry. The host port + # is ephemeral (compose publishes "${AGENT_PORT}"). + agent_addr="$(docker compose port agent 8010)" + ( + set +x + prompts=( + 'Show all available products in the store.' + 'What currencies are supported by the Astronomy Shop?' + 'What current promotions are available on binoculars?' + ) + i=0 + while :; do + p="${prompts[$((i % 3))]}" + curl -s -m 60 -X POST "http://${agent_addr}/prompt" \ + -H 'content-type: application/json' \ + -d "{\"message\": \"${p}\", \"history\": []}" \ + -o /dev/null -w "prompt $((i % 3)): %{http_code}\n" || true + i=$((i + 1)) + sleep 10 + done + ) & + driver_pid=$! + fi sleep "$DURATION" + if [ -n "$driver_pid" ]; then + kill "$driver_pid" 2>/dev/null || true + fi - name: Slice steady-state corpus + verify working-directory: demo @@ -354,6 +419,11 @@ jobs: # emitting services landed in the corpus. grep -o '"service.name"[^}]*"stringValue":"[^"]*"' logs.jsonl \ | grep -o '"stringValue":"[^"]*"$' | sort | uniq -c | sort -rn | head -40 || true + echo + echo "## gen_ai.* attribute keys seen" + # The agentic layer's whole point: the census proves (or + # disproves) that GenAI semconv telemetry landed. + grep -o '"key":"gen_ai[^"]*"' logs.jsonl | sort | uniq -c | sort -rn | head -20 || true } > manifest.md echo '--- manifest ---' cat manifest.md From 8a42afd4d39c1e69e5c9adf8797cf108513edc48 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Wed, 29 Jul 2026 14:18:54 +0200 Subject: [PATCH 2/2] fix(ci): probe /prompt readiness, gate on prompt success, group-kill the driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Container running-state is not /prompt readiness: the capture now probes until the first cassette-backed prompt answers 2xx and fails loudly otherwise (a cassette mismatch 500s here too) — an apparently successful corpus with no gen_ai content is the failure mode this prevents. The driver logs every HTTP code; zero successes across the window fails the capture. setsid gives the driver its own process group so teardown kills in-flight curls, with a wait before slicing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Qtny6z6cA74xPZa4qRhk4F Signed-off-by: Jens Holdgaard Pedersen --- .../workflows/capture-otel-demo-corpus.yml | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/.github/workflows/capture-otel-demo-corpus.yml b/.github/workflows/capture-otel-demo-corpus.yml index 623bace2e..b6e893eea 100644 --- a/.github/workflows/capture-otel-demo-corpus.yml +++ b/.github/workflows/capture-otel-demo-corpus.yml @@ -331,6 +331,7 @@ jobs: echo "CAPTURE_START_LINE=${start_line}" >> "$GITHUB_ENV" echo "steady-state window starts at raw line ${start_line}; capturing ${DURATION}s" driver_pid="" + driver_log="$OURIOS_CAPTURE_DIR/prompt-driver.log" if [ "$AGENTIC" = "vcr" ]; then # Drive the agent with the three cassette-backed prompts # (src/agent/README.md "Default Requests") for the whole @@ -339,29 +340,57 @@ jobs: # every loop emits real gen_ai.* telemetry. The host port # is ephemeral (compose publishes "${AGENT_PORT}"). agent_addr="$(docker compose port agent 8010)" - ( - set +x + # Container `running` is not `/prompt` ready: probe until the + # first cassette-backed prompt answers 2xx, and fail the + # capture loudly if it never does — an apparently successful + # corpus with no gen_ai.* content is the failure mode this + # exists to prevent (a cassette mismatch would 500 here too). + ready="" + for _ in $(seq 1 18); do + code="$(curl -s -m 60 -X POST "http://${agent_addr}/prompt" \ + -H 'content-type: application/json' \ + -d '{"message": "Show all available products in the store.", "history": []}' \ + -o /dev/null -w '%{http_code}' || echo 000)" + if [ "${code:0:1}" = "2" ]; then ready=yes; break; fi + echo "agent /prompt not ready yet (HTTP ${code})" + sleep 10 + done + if [ -z "$ready" ]; then + echo "::error::agent /prompt never answered 2xx — VCR replay is not serving; refusing to capture a gen_ai-less agentic corpus" + docker compose logs agent | tail -50 || true + exit 1 + fi + # `setsid` gives the driver its own process group so + # teardown can kill the loop AND any in-flight curl. + setsid bash -c ' prompts=( - 'Show all available products in the store.' - 'What currencies are supported by the Astronomy Shop?' - 'What current promotions are available on binoculars?' + "Show all available products in the store." + "What currencies are supported by the Astronomy Shop?" + "What current promotions are available on binoculars?" ) i=0 while :; do p="${prompts[$((i % 3))]}" - curl -s -m 60 -X POST "http://${agent_addr}/prompt" \ - -H 'content-type: application/json' \ + curl -s -m 60 -X POST "http://'"${agent_addr}"'/prompt" \ + -H "content-type: application/json" \ -d "{\"message\": \"${p}\", \"history\": []}" \ - -o /dev/null -w "prompt $((i % 3)): %{http_code}\n" || true + -o /dev/null -w "%{http_code}\n" >> "'"$driver_log"'" || echo 000 >> "'"$driver_log"'" i=$((i + 1)) sleep 10 done - ) & + ' & driver_pid=$! fi sleep "$DURATION" if [ -n "$driver_pid" ]; then - kill "$driver_pid" 2>/dev/null || true + kill -- -"$driver_pid" 2>/dev/null || kill "$driver_pid" 2>/dev/null || true + wait "$driver_pid" 2>/dev/null || true + ok="$(grep -c '^2' "$driver_log" 2>/dev/null || echo 0)" + echo "prompt driver: ${ok} successful prompts (log: $(wc -l < "$driver_log" 2>/dev/null || echo 0) attempts)" + if [ "$ok" -eq 0 ]; then + echo "::error::no prompt succeeded during the capture window — the corpus would carry no agent-driven gen_ai telemetry" + exit 1 + fi fi - name: Slice steady-state corpus + verify