From 9e0528a691c908012935c7b5dbfb22de65e76995 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Tue, 25 Aug 2026 21:31:34 +0900 Subject: [PATCH] fix(perf): preserve valid k6 lifecycle evidence --- docs/operability/http-concurrency-evidence.md | 9 ++++ docs/product-requirements.md | 8 ++- scripts/k6_http_e2e.js | 54 +++++++++++++------ tests/test_k6_http_e2e_contract.py | 14 +++++ 4 files changed, 63 insertions(+), 22 deletions(-) create mode 100644 tests/test_k6_http_e2e_contract.py diff --git a/docs/operability/http-concurrency-evidence.md b/docs/operability/http-concurrency-evidence.md index 9e846f7ba..b3435fe1b 100644 --- a/docs/operability/http-concurrency-evidence.md +++ b/docs/operability/http-concurrency-evidence.md @@ -37,6 +37,15 @@ The custom metrics separate: - `lineageweave_ask_enqueue_duration`: time to persist and acknowledge the job; - `lineageweave_read_duration{endpoint:posts|lineage}`: ordinary reader paths; - `lineageweave_ask_poll_duration`: owner-scoped status polling. +- `lineageweave_ask_state_observations{job_status:...}`: how many observations + occurred while the one queued job was queued, running, or settled. + +The harness observes one Ask job's real lifecycle; it does not keep provider +work running artificially. Report reader distributions with the state counts +so a long settled tail is not misrepresented as contended capacity. Per-VU +authentication is renewed after an HTTP 401 and the failed batch is retried +once, so observation windows longer than the realm access-token lifetime do +not silently become rejection measurements. There are deliberately no pass/fail thresholds. A latency or concurrency SLO requires a named deployment, representative workload, capacity evidence, and diff --git a/docs/product-requirements.md b/docs/product-requirements.md index 96f66733b..75cba0410 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -126,13 +126,11 @@ vendor selector, duplicate identity store, or psychometric substitute appears. during provider execution. Authenticated concurrent HTTP behavior is measured end to end against synthetic Compose data; latency and concurrency become release thresholds only after a named deployment and representative - workload establish an approved capacity/SLO contract. + workload establish an approved capacity/SLO contract. Release evidence + includes observed Ask-job state counts, measured bottlenecks, and a capacity + envelope rather than an unmeasured concurrency claim. - Public APIs have bounded inputs, stable typed responses, and provenance- preserving failure states. -- Long-running web work executes asynchronously with observable job state; - release evidence includes an end-to-end k6 concurrency test, measured - bottlenecks, and a capacity envelope rather than an unmeasured concurrency - claim. - WCAG 2.2 AA, keyboard/touch parity, responsive layouts, reduced motion, design tokens, Storybook edge states, and screenshot review apply to every customer-facing surface. diff --git a/scripts/k6_http_e2e.js b/scripts/k6_http_e2e.js index d9b79bf25..737b2e09f 100644 --- a/scripts/k6_http_e2e.js +++ b/scripts/k6_http_e2e.js @@ -8,7 +8,7 @@ import http from "k6/http"; import { check, fail } from "k6"; -import { Trend } from "k6/metrics"; +import { Counter, Trend } from "k6/metrics"; const backendUrl = (__ENV.BACKEND_URL || "http://localhost:18420").replace(/\/$/, ""); const keycloakUrl = (__ENV.KEYCLOAK_URL || "http://localhost:18080").replace(/\/$/, ""); @@ -20,9 +20,12 @@ const password = __ENV.K6_PASSWORD || "lineageweave-demo-only"; const askEnqueueDuration = new Trend("lineageweave_ask_enqueue_duration", true); const readDuration = new Trend("lineageweave_read_duration", true); const askPollDuration = new Trend("lineageweave_ask_poll_duration", true); +const askStateObservations = new Counter("lineageweave_ask_state_observations"); -export function setup() { - const tokenResponse = http.post( +let vuToken; + +function authenticate() { + const response = http.post( `${keycloakUrl}/realms/${realm}/protocol/openid-connect/token`, { grant_type: "password", @@ -32,11 +35,28 @@ export function setup() { }, { tags: { endpoint: "oidc_token" } }, ); - if (tokenResponse.status !== 200) { - fail(`synthetic OIDC login failed with HTTP ${tokenResponse.status}`); + if (response.status !== 200) { + fail(`synthetic OIDC login failed with HTTP ${response.status}`); } + return response.json("access_token"); +} + +function readBatch(token, askJobId) { + const params = { headers: { Authorization: `Bearer ${token}` } }; + return http.batch([ + ["GET", `${backendUrl}/api/posts`, null, { ...params, tags: { endpoint: "posts" } }], + ["GET", `${backendUrl}/api/lineage`, null, { ...params, tags: { endpoint: "lineage" } }], + [ + "GET", + `${backendUrl}/api/ask/jobs/${askJobId}`, + null, + { ...params, tags: { endpoint: "ask_poll" } }, + ], + ]); +} - const token = tokenResponse.json("access_token"); +export function setup() { + const token = authenticate(); const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }; const submitted = http.post( `${backendUrl}/api/ask`, @@ -51,21 +71,21 @@ export function setup() { } export default function (data) { - const params = { headers: { Authorization: `Bearer ${data.token}` } }; - const responses = http.batch([ - ["GET", `${backendUrl}/api/posts`, null, { ...params, tags: { endpoint: "posts" } }], - ["GET", `${backendUrl}/api/lineage`, null, { ...params, tags: { endpoint: "lineage" } }], - [ - "GET", - `${backendUrl}/api/ask/jobs/${data.askJobId}`, - null, - { ...params, tags: { endpoint: "ask_poll" } }, - ], - ]); + vuToken ||= data.token; + let responses = readBatch(vuToken, data.askJobId); + if (responses.some((response) => response.status === 401)) { + vuToken = authenticate(); + responses = readBatch(vuToken, data.askJobId); + } readDuration.add(responses[0].timings.duration, { endpoint: "posts" }); readDuration.add(responses[1].timings.duration, { endpoint: "lineage" }); askPollDuration.add(responses[2].timings.duration); + if (responses[2].status === 200) { + askStateObservations.add(1, { + job_status: String(responses[2].json("job_status_code") || "unknown"), + }); + } check(responses[0], { "posts read succeeds": (response) => response.status === 200 }); check(responses[1], { "lineage read succeeds": (response) => response.status === 200 }); check(responses[2], { "Ask poll succeeds": (response) => response.status === 200 }); diff --git a/tests/test_k6_http_e2e_contract.py b/tests/test_k6_http_e2e_contract.py new file mode 100644 index 000000000..f77d8f596 --- /dev/null +++ b/tests/test_k6_http_e2e_contract.py @@ -0,0 +1,14 @@ +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "k6_http_e2e.js" + + +def test_k6_harness_renews_expired_auth_and_discloses_job_state() -> None: + """Long observations retry expired authentication and separate job states.""" + source = SCRIPT.read_text(encoding="utf-8") + + assert "responses.some((response) => response.status === 401)" in source + assert source.count("responses = readBatch(vuToken, data.askJobId)") == 2 + assert "lineageweave_ask_state_observations" in source + assert 'job_status: String(responses[2].json("job_status_code")' in source