From 8de2f3b7369dbc5a845e986db43fad32255c0a81 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Sun, 31 May 2026 17:54:15 +0200 Subject: [PATCH] fix(bench): bypass frontend-proxy, drive frontend directly (PR-N3.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published frontend-proxy image (demo 2.2.0) carries a baked Envoy bootstrap with a 10th cluster whose address env var the demo doesn't set, so Envoy fails proto validation and crash-loops in CI. That flapping container — not a startup race — is what gave the first capture its ERR_NAME_NOT_RESOLVED flood (intermittent DNS for a restarting container). The source tag's envoy.tmpl.yaml has only 9 clusters all with set vars, and the image is distroless (no shell to inspect the baked template), so the empty var can't be identified from outside. Rather than reverse-engineer the image, remove Envoy from the path: the compose override sets LOCUST_BROWSER_TRAFFIC_ENABLED= false + LOCUST_HOST=http://frontend: so the load generator drives the frontend's HTTP API directly. The frontend still fans out to cart / checkout / currency / product-catalog / …, so business-service log diversity still flows. Readiness now gates on otel-collector + frontend running (frontend has no healthcheck and only an ephemeral host port, so the post-restart settle + steady-state slice absorb early ramp errors), and the crash-looping frontend-proxy is stopped to free runner CPU. Co-Authored-By: Claude Opus 4.8 --- .../otel-demo-capture-compose-override.yml | 18 +++++ .../workflows/capture-otel-demo-corpus.yml | 69 ++++++++----------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/.github/otel-demo-capture-compose-override.yml b/.github/otel-demo-capture-compose-override.yml index 8219e6e68..ef26b9c78 100644 --- a/.github/otel-demo-capture-compose-override.yml +++ b/.github/otel-demo-capture-compose-override.yml @@ -17,8 +17,26 @@ # doesn't depend on Compose's project-directory resolution when # the override is supplied from a different directory than the # base compose file. +# +# The `load-generator` overrides bypass `frontend-proxy` (Envoy): +# the published frontend-proxy image carries a baked Envoy +# bootstrap with a 10th cluster whose address env var the demo +# doesn't set, so Envoy fails proto validation and crash-loops in +# CI — which is what flooded the first capture with the load +# generator's `ERR_NAME_NOT_RESOLVED` (intermittent DNS for a +# flapping container). Driving the `frontend` HTTP API directly +# (browser traffic off) removes Envoy from the path entirely; the +# frontend still fans out to cart / checkout / currency / +# product-catalog / …, so the business-service log diversity we +# want still flows. `environment` entries merge by key, so these +# override the base compose's `LOCUST_BROWSER_TRAFFIC_ENABLED=true` +# and the `.env` `LOCUST_HOST`. services: otel-collector: volumes: - ${OURIOS_CAPTURE_DIR}:/capture + load-generator: + environment: + - LOCUST_BROWSER_TRAFFIC_ENABLED=false + - LOCUST_HOST=http://frontend:${FRONTEND_PORT} diff --git a/.github/workflows/capture-otel-demo-corpus.yml b/.github/workflows/capture-otel-demo-corpus.yml index 1f7b18ad2..07e7e8b60 100644 --- a/.github/workflows/capture-otel-demo-corpus.yml +++ b/.github/workflows/capture-otel-demo-corpus.yml @@ -63,12 +63,6 @@ jobs: # writes `logs.jsonl` here, so the corpus lands straight on # the runner filesystem. OURIOS_CAPTURE_DIR: ${{ github.workspace }}/captured - # Envoy admin port the demo publishes (demo .env default). - # Used for the frontend-proxy /ready readiness probe — the - # demo's .env isn't sourced into the workflow shell, so the - # value is pinned here (bump alongside demo_ref if it ever - # changes upstream). - ENVOY_ADMIN_PORT: '10000' # `COMPOSE_FILE` (colon-separated on Linux) layers our # override onto the demo's base compose for every # `docker compose` call without repeating `-f`. The base @@ -126,47 +120,38 @@ jobs: working-directory: demo run: | set -euxo pipefail - # Gate on the services that actually matter, not just the - # collector. The first capture (run 26715857483) showed - # ~34% of the corpus was the load generator's - # ERR_NAME_NOT_RESOLVED against frontend-proxy: the - # load-generator only `depends_on` frontend (started, not - # healthy) and frontend-proxy has no healthcheck, so its - # Playwright browser tasks hammered Envoy before it was - # serving and flooded the corpus with one error template. - # - # Envoy publishes an admin port with a /ready endpoint - # that returns 200 "LIVE" once initialization completes — - # the real "frontend-proxy is serving" signal. - wait_for() { - local name="$1" url="$2" tries="$3" - for _ in $(seq 1 "$tries"); do - if curl -fsS -o /dev/null "$url" 2>/dev/null; then - echo "$name ready" - return 0 - fi + # We bypass frontend-proxy (Envoy) — its baked image + # config crash-loops in CI (see the compose override) — + # and drive the `frontend` HTTP API directly. So gate on + # otel-collector (the capture sink) and frontend (the + # load target) reaching running state. Neither defines a + # healthcheck and frontend only publishes an ephemeral + # host port, so "running" + the post-restart settle + + # the steady-state slice (which discards early ramp + # errors) is the readiness story rather than an HTTP + # probe. + want_running() { + local svc="$1" + for _ in $(seq 1 60); do + docker compose ps --status running --services 2>/dev/null \ + | grep -qx "$svc" && { echo "$svc running"; return 0; } sleep 5 done - echo "::error::$name not ready after $((tries * 5))s ($url)" + echo "::error::$svc did not reach running state" docker compose ps - docker compose logs frontend-proxy | tail -50 || true + docker compose logs "$svc" | tail -50 || true return 1 } - # otel-collector first (it's the capture sink); then Envoy - # readiness on the published admin port. - for _ in $(seq 1 60); do - docker compose ps --status running --services 2>/dev/null \ - | grep -qx otel-collector && break - sleep 5 - done - docker compose ps --status running --services | grep -qx otel-collector \ - || { echo "::error::otel-collector not running"; docker compose ps; docker compose logs otel-collector | tail -50; exit 1; } - wait_for frontend-proxy "http://localhost:${ENVOY_ADMIN_PORT}/ready" 60 - # Restart the load generator now that Envoy is serving, so - # its browser tasks start fresh against a resolvable, - # ready proxy instead of replaying the startup-race - # failures. Pre-restart noise is discarded by the - # steady-state slice in the capture step. + want_running otel-collector + want_running frontend + # frontend-proxy is bypassed and crash-loops on its bad + # baked config; stop it so it stops burning runner CPU + # and restart-spamming. Nothing else depends on it (it + # fronts the frontend, not the reverse). + docker compose stop frontend-proxy 2>/dev/null || true + # Restart the load generator once frontend is up so it + # ramps cleanly against a live target; any residual early + # errors are discarded by the steady-state slice. docker compose restart load-generator docker compose ps