diff --git a/.github/workflows/chat-coverage.yml b/.github/workflows/chat-coverage.yml new file mode 100644 index 000000000..8ead24f2d --- /dev/null +++ b/.github/workflows/chat-coverage.yml @@ -0,0 +1,145 @@ +name: Chat interaction coverage + +# Two jobs, deliberately gated differently. +# +# self-check Runs everywhere, on every pull request that touches the gate. +# Needs no deployment, no session and no secrets: it serves its +# own fixture from an intercepted origin and asserts the prover +# still calls a dead button dead. This is the part that keeps +# the gate from rotting into something that cannot go red. +# +# live-sweep The real sweep, against the deployed chat surface. It needs a +# running deployment and a minted session, so it never fires on +# a plain pull request: run it by hand, or label a pull request +# run-chat-coverage. The same reason owui-nightly is not a +# per-pull-request gate. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + push: + branches: [main] + workflow_dispatch: + inputs: + surfaces: + description: 'Surface filter (a regular expression, blank for a full sweep)' + required: false + default: '' + +concurrency: + group: chat-coverage-${{ github.ref }} + cancel-in-progress: true + +# Read only, and narrowed to what is actually read: the self-check job asks the +# issues API whether a surface exclusion's blocking issue has closed. Nothing +# here writes to the repository, and a job that walks a live OAuth flow should +# not be holding a token that could. +permissions: + contents: read + issues: read + +jobs: + self-check: + name: Chat coverage self-check (no deployment needed) + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: apps/web-console + steps: + - uses: actions/checkout@v4 + with: + # Leave no usable git credential in the runner's config for the rest + # of the job, which runs a browser against a live deployment. + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: apps/web-console/package-lock.json + - name: Install dependencies + run: npm ci + - name: Install Chromium + run: npx playwright install --with-deps chromium + - name: Prover break-proof, registry and exclusion expiry + # GITHUB_TOKEN is what the exclusion-expiry check reads issue state + # with. It fails rather than skips without one, on purpose: an + # exclusion whose blocking issue has closed has to become a failure + # the day it closes, and a check that skips itself cannot do that. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: npm run e2e:chat-coverage:self-check + + live-sweep: + name: Chat coverage live sweep + runs-on: ubuntu-latest + timeout-minutes: 60 + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'run-chat-coverage')) + defaults: + run: + working-directory: apps/web-console + env: + CHAT_URL: ${{ vars.CHAT_URL || 'https://chat-hive.scubed.co' }} + CONSOLE_URL: ${{ vars.CONSOLE_URL || 'https://console-hive.scubed.co' }} + # An existing account to mint a session for. live-auth.mjs mints through + # the admin one-time-token flow, so no password is read and none is + # changed; HIVE_QA_TESTER_PASSWORD is deliberately not passed in here. + HIVE_QA_AGENT_EMAIL: ${{ secrets.HIVE_QA_AGENT_EMAIL || secrets.HIVE_QA_TESTER_EMAIL }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + COV_SURFACES: ${{ inputs.surfaces }} + steps: + - uses: actions/checkout@v4 + with: + # Leave no usable git credential in the runner's config for the rest + # of the job, which runs a browser against a live deployment. + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: apps/web-console/package-lock.json + - name: Check the inputs the sweep cannot run without + # Loud, not quiet. A missing secret here used to leave the whole + # config matching zero files, which reports a green run of nothing. + run: | + set -euo pipefail + missing="" + for v in CHAT_URL CONSOLE_URL HIVE_QA_AGENT_EMAIL SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY; do + [ -n "${!v:-}" ] || missing="$missing $v" + done + if [ -n "$missing" ]; then + echo "::error::the live sweep needs:$missing" + exit 1 + fi + - name: Install dependencies + run: npm ci + - name: Install Chromium + run: npx playwright install --with-deps chromium + - name: Sweep the chat surface + run: npm run e2e:chat-coverage + - name: Upload the coverage ledger + if: always() + uses: actions/upload-artifact@v4 + with: + name: chat-coverage-ledger + # Two named files, never a directory. Everything this suite writes + # itself goes through redactUrl, but a directory upload also sweeps + # up whatever the framework wrote beside them: Playwright's JSON and + # HTML reports carry raw error text, and a sign-in that outruns its + # ninety second wait quotes the OAuth callback with a live code and + # state in the timeout message. Those reports are configured to land + # outside this directory, and naming the files here means a future + # reporter added to the config cannot quietly rejoin the artifact. + # Traces and videos are off for the same reason and cannot be + # inspected by the text lint at all. + path: | + apps/web-console/chat-coverage-report/coverage.run.json + apps/web-console/chat-coverage-report/coverage.md + if-no-files-found: error + retention-days: 14 diff --git a/.gitignore b/.gitignore index 39ed47e4e..85efbbe7e 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,9 @@ __pycache__/ # named anything other than test-results/ was landing traces and videos in # `git status` where an `add -A` could commit them. /apps/web-console/test-results*/ +/apps/web-console/playwright-report-chat-coverage/ +/apps/web-console/chat-coverage-report/ +/apps/web-console/e2e/chat-coverage/.auth/ # Added by code-review-graph .code-review-graph/ diff --git a/apps/web-console/e2e/chat-coverage/auth.setup.ts b/apps/web-console/e2e/chat-coverage/auth.setup.ts new file mode 100644 index 000000000..526f9f019 --- /dev/null +++ b/apps/web-console/e2e/chat-coverage/auth.setup.ts @@ -0,0 +1,36 @@ +// Signs the coverage sweep in and saves storageState. +// +// signInToChat lives in tests/e2e/support/live-auth.ts, and is the only +// implementation of the chat hop: the mint itself stays in live-auth.mjs, +// which this shells out to, so the service-role key never enters this worker's +// module graph. It mints a session the way a magic link does, so no password +// is read, written or rotated, then carries it through the real "Continue with +// Hive" OIDC hop, which is the only thing that gets Open WebUI's own session +// cookie set. See docs/live-test-auth.md. +// +// This project records no trace, video or screenshot (see the config): the +// callback URL it walks through carries a live `code` and `state`, and no text +// lint can read inside a trace or a video to catch them. +import path from "node:path"; + +import { test as setup } from "@playwright/test"; + +import { signInToChat } from "../../tests/e2e/support/live-auth"; + +const STATE = path.join(__dirname, ".auth", "state.json"); +const CHAT = process.env.CHAT_URL ?? process.env.OWUI_URL ?? ""; +const CONSOLE = process.env.CONSOLE_URL ?? ""; +const EMAIL = process.env.OWUI_E2E_EMAIL ?? process.env.HIVE_QA_AGENT_EMAIL ?? ""; + +setup("authenticate against the chat surface", async ({ browser }) => { + setup.setTimeout(6 * 60_000); + if (!CHAT || !CONSOLE || !EMAIL) { + setup.skip(true, "CHAT_URL, CONSOLE_URL and an account are all required"); + return; + } + + const context = await browser.newContext({ viewport: { width: 1440, height: 950 } }); + await signInToChat(context, { email: EMAIL, consoleUrl: CONSOLE, chatUrl: CHAT }); + await context.storageState({ path: STATE }); + await context.close(); +}); diff --git a/apps/web-console/e2e/chat-coverage/break-proof.spec.ts b/apps/web-console/e2e/chat-coverage/break-proof.spec.ts new file mode 100644 index 000000000..846a75b1e --- /dev/null +++ b/apps/web-console/e2e/chat-coverage/break-proof.spec.ts @@ -0,0 +1,239 @@ +// The checks that keep the coverage gate honest, none of which need a live +// deployment or a session. +// +// They are in their own file, and their own Playwright project, because they +// used to sit inside the live sweep behind the same credential gate: with no +// CHAT_URL and no Supabase keys the config matched zero files, so the one test +// that proves the detector can tell a working control from a broken one never +// ran anywhere except on a machine already able to run the full sweep. A gate +// nobody has watched fail is worth nothing. +import { test, expect } from "@playwright/test"; + +declare global { + interface Window { + /** Set by the fixture's destructive button if anything ever clicks it. */ + __destructiveFired?: boolean; + } +} + +import { EXCLUSIONS, REGISTRY } from "./data"; +import { + checkWithoutFiring, + enumerate, + exclusionFailures, + expiredExclusions, + isDeferred, + proveByClick, + sampleChatter, +} from "./lib"; + +const SELF_TEST_HTML = ` +
+
idle
+ + + + + + + + + + +
+ +`; + +test.describe("chat coverage gate self-checks", () => { + test("the prover tells a working control from a broken one", async ({ page }) => { + test.setTimeout(120_000); + // Five buttons that look alike: wired does something and must be proven; + // dead has no handler and must be unproven; failing calls an endpoint that + // answers 500 and shows a toast and must be unproven, which is the case + // that used to come back proven twice over; toasty only shows an error and + // must be unproven; pollster does nothing at all while the page polls + // underneath it, and must be unproven despite the traffic. + // + // An intercepted origin rather than setContent, so the failing button has a + // same-origin endpoint whose status the prover can actually read. + await page.route("https://cov.selftest/**", async (route) => { + const url = route.request().url(); + if (url.endsWith("/index")) { + await route.fulfill({ contentType: "text/html", body: SELF_TEST_HTML }); + return; + } + if (url.endsWith("/api/poll")) { + await route.fulfill({ status: 200, contentType: "application/json", body: "{}" }); + return; + } + if (url.endsWith("/api/hang")) { + // Never fulfilled and never aborted: the request stays open past the + // settle window, exactly like an endpoint that accepts and hangs. + return; + } + if (url.endsWith("/api/cut")) { + await route.abort("connectionreset"); + return; + } + await route.fulfill({ + status: 500, + contentType: "application/json", + body: JSON.stringify({ detail: "boom" }), + }); + }); + await page.goto("https://cov.selftest/index"); + + const controls = await enumerate(page, "self-test"); + const byName = (name: string) => { + const found = controls.find((c) => c.name === name); + expect(found, "the fixture must render " + name).toBeTruthy(); + if (!found) throw new Error("unreachable: the assertion above already failed"); + return found; + }; + + // What the page does with nobody touching it, the same sample the sweep + // takes per surface. + const chatter = await sampleChatter(page, 1500); + expect( + [...chatter].some((signature) => signature.endsWith("/api/poll")), + "the idle sample must see the fixture's own polling, or it is not sampling anything", + ).toBe(true); + + const wired = await proveByClick(page, byName("Wired"), { ignoreRequests: chatter }); + const dead = await proveByClick(page, byName("Dead"), { ignoreRequests: chatter }); + const failing = await proveByClick(page, byName("Failing"), { ignoreRequests: chatter }); + const toasty = await proveByClick(page, byName("Toasty"), { ignoreRequests: chatter }); + const pollster = await proveByClick(page, byName("Pollster"), { ignoreRequests: chatter }); + + expect(wired.proven, "a working control was reported as broken: " + wired.detail).toBe(true); + expect(dead.proven, "a control with no handler was reported as working: " + dead.detail).toBe( + false, + ); + expect( + failing.proven, + "a control whose endpoint answered 500 was reported as working: " + failing.detail, + ).toBe(false); + expect(failing.detail).toContain("500"); + expect( + toasty.proven, + "a control that only raised an error was reported as working: " + toasty.detail, + ).toBe(false); + expect(toasty.detail).toContain("raised an error"); + expect( + pollster.proven, + "a dead control was proven by the page's own background polling: " + pollster.detail, + ).toBe(false); + + const hanging = await proveByClick(page, byName("Hanging"), { ignoreRequests: chatter }); + expect( + hanging.proven, + "a control whose request nothing answered was reported as working: " + hanging.detail, + ).toBe(false); + expect(hanging.detail).toContain("nothing answered"); + + const cutOff = await proveByClick(page, byName("Cut Off"), { ignoreRequests: chatter }); + expect( + cutOff.proven, + "a control whose request died in transport was reported as working: " + cutOff.detail, + ).toBe(false); + expect(cutOff.detail).toContain("never reached a server"); + + // Destructive and disabled controls are checked, not fired, and the + // verdict is never proof. Both used to count as coverage: the disabled one + // outright, which meant a broken control could be "fixed" by disabling it. + const destructive = await checkWithoutFiring(page, byName("Delete Everything")); + expect(destructive.proven, "a destructive control was reported as proven").toBe(false); + expect(destructive.proof).toBe("not-fired"); + expect(isDeferred(destructive)).toBe(true); + // The verdict string is not the claim being tested. The claim is that the + // control was never activated, and only the page can answer that. + expect( + await page.evaluate(() => window.__destructiveFired === true), + "the harness fired a destructive control it reported as not fired", + ).toBe(false); + + const locked = await proveByClick(page, byName("Locked"), { ignoreRequests: chatter }); + expect(locked.proven, "a disabled control was reported as proven: " + locked.detail).toBe( + false, + ); + expect(locked.proof).toBe("not-fired"); + expect(locked.detail).toContain("finish the form first"); + }); + + test("every inert-registry entry carries a justification", async () => { + const bad = REGISTRY.filter((e) => !e.key || !e.justification.trim()); + expect( + bad.map((e) => e.key || "(no key)"), + "registry entries without a key or a justification", + ).toEqual([]); + expect(exclusionFailures(EXCLUSIONS), "malformed surface exclusions").toEqual([]); + }); + + // An exclusion ends when the thing blocking it is fixed, and the only + // authority on that is the issue tracker. Every path through this test that + // cannot read that state now fails. It used to skip on a missing token and + // `continue` past any non-ok response, which meant the expiry could never + // fire: an exclusion would outlive its own issue forever and the check would + // report green the whole time. + test("no excluded surface waits on an issue that already closed", async ({ request }) => { + const pending = EXCLUSIONS.filter((entry) => entry.issue !== undefined); + if (pending.length === 0) return; + + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN ?? ""; + expect( + token, + "GITHUB_TOKEN (or GH_TOKEN) is required: this check reads whether the issues that " + + `justify ${pending.length} surface exclusion(s) are still open, and it fails rather ` + + "than passes when it cannot find out", + ).not.toBe(""); + + const repo = process.env.GITHUB_REPOSITORY ?? "sakibsadmanshajib/hive"; + const closed = new Set(); + const unreadable: string[] = []; + for (const entry of pending) { + const url = "https://api.github.com/repos/" + repo + "/issues/" + String(entry.issue); + const response = await request.get(url, { + headers: { authorization: "Bearer " + token, accept: "application/vnd.github+json" }, + }); + if (!response.ok()) { + unreadable.push(`#${String(entry.issue)}: HTTP ${response.status()}`); + continue; + } + const body: unknown = await response.json(); + const state = + typeof body === "object" && body !== null + ? new Map(Object.entries(body)).get("state") + : undefined; + if (state === "closed" && entry.issue !== undefined) closed.add(entry.issue); + } + + expect( + unreadable, + "issue state could not be read, so an exclusion whose reason has expired would go unnoticed", + ).toEqual([]); + expect(expiredExclusions(EXCLUSIONS, closed)).toEqual([]); + }); +}); diff --git a/apps/web-console/e2e/chat-coverage/chat-coverage.spec.ts b/apps/web-console/e2e/chat-coverage/chat-coverage.spec.ts new file mode 100644 index 000000000..3990f3dee --- /dev/null +++ b/apps/web-console/e2e/chat-coverage/chat-coverage.spec.ts @@ -0,0 +1,811 @@ +// Live interaction coverage for the Hive chat surface. +// +// Owner mandate: "Every control surface, anything that can be clicked, dropped +// down, changed, or swapped, should be tested live." This suite enumerates the +// chat app's controls from the rendered DOM and requires each one to produce an +// observable effect against a running deployment. It reports a ratio, writes a +// machine-readable file so the number can be tracked between runs, and fails +// loudly on any control that does nothing. +// +// Everything here needs a live deployment and a session. The checks that need +// neither live in break-proof.spec.ts. +import fs from "node:fs"; +import path from "node:path"; + +import { test, expect, type Page } from "@playwright/test"; + +import { EXCLUSIONS, FLOORS, REGISTRY, REMOVED } from "./data"; +import { + checkWithoutFiring, + enumerate, + exclusionFailures, + floorFailures, + flip, + isDeferred, + isDestructive, + identityOf, + isStateful, + locate, + proveByClick, + readState, + redactText, + redactUrl, + sampleChatter, + summarise, + type Control, + type Result, + type Swept, +} from "./lib"; +import { + STATIC_SURFACES, + discoverSettingsTabs, + discoverWorkspaceTabs, + enumerateSurface, + relocate, + gotoHome, + openSettings, + type Surface, +} from "./surfaces"; + +const REPORT_DIR = path.join(__dirname, "../../chat-coverage-report"); + +/** + * Safety net, and deliberately not a source of proof. + * + * Destructive controls are recognised by their label and never clicked (see + * checkWithoutFiring). This second guard exists for the control whose label + * says nothing about what it does: its write is aborted at the network layer + * so the demo account's data survives, and the control it came from is then + * recorded as not-fired rather than as proven, because an aborted request + * never reaches the server and so has no verdict to read. + */ +function isDestructiveRequest(method: string, url: string): boolean { + if (method === "DELETE") return true; + return /\/(delete|archive|clear|reset|unshare)/i.test(url); +} + +async function saveIfPresent(page: Page): Promise { + const save = page.getByRole("button", { name: /^save$/i }).first(); + if (await save.isVisible().catch(() => false)) { + await save.click().catch(() => {}); + await page.waitForTimeout(1500); + } +} + +/** + * Failures that say nothing about the control. + * + * The demo box has gone unreachable for minutes at a time mid-run (#815), and + * a click that lands in that window is indistinguishable from a dead control. + * Only these retry. A clean verdict of "this control does nothing" is final on + * the first attempt: retrying it until it passes is how an in-test loop turns + * itself into the retries the config sets to zero on purpose. + */ +function isInfrastructural(result: Result): boolean { + return /surface would not re-open|was not there when the surface was re-opened|no verdict within/.test( + result.detail, + ); +} + +type PassContext = { + /** Request signatures the page emits on its own, sampled while idle. */ + chatter: ReadonlySet; + /** Writes the safety net aborted, appended to for the whole run. */ + blockedWrites: string[]; +}; + +/** + * Click pass: one control at a time, re-opening the surface when an action + * moved the app. Results are appended to the caller's array as they are + * produced: an earlier version returned them at the end, and one timeout in + * the middle of a surface threw away every proof collected before it. + */ +async function clickPass( + page: Page, + surface: Surface, + controls: Control[], + results: Result[], + ctx: PassContext, +): Promise { + for (const ctl of controls) { + let res: Result = { + key: ctl.key, + surface: surface.id, + name: ctl.label || ctl.name, + role: ctl.role || ctl.tag, + proven: false, + proof: "none", + detail: "the pass produced no verdict for this control", + }; + + for (let attempt = 0; attempt < 2; attempt++) { + let target: Control | undefined; + try { + target = await relocate(page, surface, ctl.key); + } catch (err) { + res = { + ...res, + proven: false, + proof: "none", + detail: `surface would not re-open: ${String(err).split("\n")[0].slice(0, 120)}`, + }; + continue; + } + if (!target) { + res = { + ...res, + proven: false, + proof: "none", + detail: "control was not there when the surface was re-opened", + }; + continue; + } + + if (isDestructive(target)) { + res = await checkWithoutFiring(page, target); + } else { + const before = ctx.blockedWrites.length; + res = await proveByClick(page, target, { ignoreRequests: ctx.chatter }); + if (ctx.blockedWrites.length > before) { + // It fired a write the safety net stopped. Whatever the positive + // channels saw, the server never answered, so there is no verdict + // to record and this is not proof. + res = { + ...res, + proven: false, + proof: "not-fired", + detail: `fired a write that was intercepted (${ctx.blockedWrites[before]}), so the server never answered it`, + }; + } + } + if (res.proven || isDeferred(res) || !isInfrastructural(res)) break; + } + + results.push(res); + // eslint-disable-next-line no-console + console.log(`[control] ${res.proven ? res.proof : res.proof === "not-fired" ? "NOT-FIRED" : "UNPROVEN"} :: ${res.key}`); + } +} + +/** + * Persistence pass. Open WebUI keeps user settings in a persisted config blob, + * so a toggle that flips visually and persists nothing is invisible until the + * next reload. Every stateful control on a persisting surface is flipped, saved, + * and re-read after a full reload, then put back the way it was found. + */ +async function persistencePass( + page: Page, + surface: Surface, + statefuls: Control[], + results: Result[], +): Promise { + if (statefuls.length === 0) return; + + let live = await enumerateSurface(page, surface); + const baselines = new Map(); + + for (const ctl of statefuls) { + const target = live.find((c) => c.key === ctl.key); + if (!target) continue; + baselines.set(ctl.key, target.state); + await flip(page, target).catch(() => {}); + await page.waitForTimeout(200); + } + await saveIfPresent(page); + + await page.reload({ waitUntil: "domcontentloaded" }); + await page.waitForTimeout(2000); + live = await enumerateSurface(page, surface); + + const missing: Control[] = []; + const reverted: Control[] = []; + for (const ctl of statefuls) { + if (!baselines.has(ctl.key)) { + results.push(unprovable(surface, ctl, "control was not present when the pass started")); + continue; + } + const after = live.find((c) => c.key === ctl.key); + if (!after) { + missing.push(ctl); + continue; + } + const before = baselines.get(ctl.key) ?? null; + if (after.state !== before) { + results.push(proven(surface, ctl, "persisted", `${before} -> ${after.state} survived a reload`)); + } else { + // Do not report a batch result as a defect. Open WebUI saves some + // sections on their own Save button and some immediately, and flipping a + // whole tab at once can leave one of them behind for reasons that have + // nothing to do with this control. Re-check it on its own before saying + // it saves nothing. + reverted.push(ctl); + } + } + + // Put the surface back the way it was found before doing anything else. + for (const ctl of statefuls) { + if (!baselines.has(ctl.key)) continue; + const target = live.find((c) => c.key === ctl.key); + if (!target) continue; + const before = baselines.get(ctl.key) ?? null; + if (target.state === before) continue; + await restore(page, target, before).catch(() => {}); + } + await saveIfPresent(page); + + // Controls the batch could not re-locate (a flipped toggle can hide its own + // dependants) get an individual pass rather than a false negative. + for (const ctl of [...missing, ...reverted]) { + results.push(await persistOne(page, surface, ctl)); + } +} + +async function restore(page: Page, ctl: Control, before: string | null): Promise { + const el = locate(page, ctl); + if (ctl.role === "switch" || ctl.role === "checkbox" || ctl.type === "checkbox") { + await el.click({ timeout: 6000 }); + return; + } + if (ctl.tag === "select") { + await el.selectOption(before ?? "", { timeout: 6000 }); + return; + } + await el.fill(before ?? "", { timeout: 6000 }); +} + +async function persistOne(page: Page, surface: Surface, ctl: Control): Promise { + const live = await enumerateSurface(page, surface); + const target = live.find((c) => c.key === ctl.key); + if (!target) return unprovable(surface, ctl, "not present on a clean re-open"); + const before = target.state; + await flip(page, target).catch(() => {}); + await saveIfPresent(page); + await page.reload({ waitUntil: "domcontentloaded" }); + await page.waitForTimeout(2000); + const after = await enumerateSurface(page, surface); + const back = after.find((c) => c.key === ctl.key); + if (!back) return unprovable(surface, ctl, "disappeared after its own change"); + const ok = back.state !== before; + if (ok) { + await restore(page, back, before).catch(() => {}); + await saveIfPresent(page); + } + return ok + ? proven(surface, ctl, "persisted", `${before} -> ${back.state} survived a reload`) + : { + key: ctl.key, + surface: surface.id, + name: ctl.label || ctl.name, + role: ctl.role || ctl.tag, + proven: false, + proof: "none", + detail: `changed in the UI but reverted to ${before} after reload`, + }; +} + +function proven(surface: Surface, ctl: Control, proof: Result["proof"], detail: string): Result { + return { + key: ctl.key, + surface: surface.id, + name: ctl.label || ctl.name, + role: ctl.role || ctl.tag, + proven: true, + proof, + detail, + }; +} + +function unprovable(surface: Surface, ctl: Control, detail: string): Result { + return { + key: ctl.key, + surface: surface.id, + name: ctl.label || ctl.name, + role: ctl.role || ctl.tag, + proven: false, + proof: "none", + detail, + }; +} + +/** Non-persisting surfaces still have to show that typing lands somewhere. */ +async function valuePass( + page: Page, + surface: Surface, + statefuls: Control[], + results: Result[], +): Promise { + if (statefuls.length === 0) return; + const live = await enumerateSurface(page, surface); + for (const ctl of statefuls) { + const target = live.find((c) => c.key === ctl.key); + if (!target) { + results.push(unprovable(surface, ctl, "not present on re-open")); + continue; + } + const before = target.state; + const written = await flip(page, target).catch(() => null); + await page.waitForTimeout(400); + const after = await readState(page, target.covId); + results.push( + after !== before + ? proven(surface, ctl, "value", `${before} -> ${after ?? written}`) + : unprovable(surface, ctl, `value did not change (still ${before})`), + ); + await restore(page, target, before).catch(() => {}); + } +} + +test.describe("chat interaction coverage", () => { + test("every rendered chat control has a proven live effect", async ({ page }) => { + test.setTimeout(45 * 60_000); + + const sabotage = process.env.COV_SABOTAGE ?? ""; + if (sabotage) { + // Self-test hook: neuter one named control so the gate can be watched + // going red on a control that works in reality. + await page.addInitScript((needle: string) => { + // Every event a handler could be bound to, stopped in the capture phase + // so it never reaches Svelte's delegated listeners at the document root. + // Blocking "click" alone left pointerdown handlers running. + for (const type of ["pointerdown", "mousedown", "mouseup", "click", "keydown"]) { + window.addEventListener( + type, + (e) => { + const el = + e.target instanceof HTMLElement + ? e.target.closest("button, a, [role=button], [role=tab], [role=switch]") + : null; + if (!el) return; + const text = (el.getAttribute("aria-label") || el.textContent || "").trim(); + if (text.toLowerCase() === needle.toLowerCase()) { + e.stopImmediatePropagation(); + e.preventDefault(); + window.__covBlocked = (window.__covBlocked ?? 0) + 1; + } + }, + true, + ); + } + }, sabotage); + } + + const stage = (msg: string) => { + // eslint-disable-next-line no-console + console.log(`[stage ${new Date().toISOString().slice(11, 19)}] ${msg}`); + }; + + // Safety net for the whole run. Nothing that reaches this is proof of + // anything; see isDestructiveRequest. + const blockedWrites: string[] = []; + await page.route("**/*", async (route) => { + const req = route.request(); + if (isDestructiveRequest(req.method(), req.url())) { + // Redacted, because this list is written into the ledger and the + // ledger is uploaded as a CI artifact. + blockedWrites.push(`${req.method()} ${redactUrl(req.url())}`); + await route.abort(); + return; + } + await route.continue(); + }); + + stage("opening the chat surface"); + await gotoHome(page); + + // Everything pushed here goes through redactText on the way out (see the + // ledger write below): these strings are mostly Playwright's own error + // messages, which quote the URLs a failed navigation walked through, and + // this suite signs in through an OAuth callback. + const errors: string[] = []; + errors.push(...exclusionFailures(EXCLUSIONS)); + + // Surface list is assembled live. Settings tabs and Workspace sections come + // from the DOM, so this run covers whatever the deployment actually renders. + // A discovery that finds nothing throws rather than returning an empty + // list; it is recorded and the sweep continues, so the run still produces a + // ledger, and the floor guard below fails on every surface it lost. + stage("discovering settings tabs"); + let settings: Surface[] = []; + try { + settings = await discoverSettingsTabs(page); + } catch (err) { + errors.push(`settings discovery failed: ${String(err).split("\n")[0]}`); + } + await page.keyboard.press("Escape").catch(() => {}); + stage(`discovering workspace sections (settings tabs: ${settings.length})`); + let workspace: Surface[] = []; + try { + workspace = await discoverWorkspaceTabs(page); + } catch (err) { + errors.push(`workspace discovery failed: ${String(err).split("\n")[0]}`); + } + stage(`workspace sections: ${workspace.length}`); + const allSurfaces: Surface[] = [ + ...STATIC_SURFACES.filter((s) => s.id !== "user-menu"), + ...settings, + ...workspace, + // Last: it holds Sign Out, which ends the session. + ...STATIC_SURFACES.filter((s) => s.id === "user-menu"), + ]; + // #782 kills a chat session about 55 minutes after sign-in, so a full sweep + // can outlive its own credentials. COV_SURFACES runs one slice per + // invocation; the tracker file below merges slices back into one number. + const filter = process.env.COV_SURFACES ? new RegExp(process.env.COV_SURFACES) : null; + // Exclusions are enumerated, owned, and tied to an open issue, and they + // leave the denominator entirely rather than counting as covered. The + // expiry test in break-proof.spec.ts fails once the blocking issue closes, + // so an exclusion cannot quietly outlive its reason. + const excludedIds = new Set(EXCLUSIONS.map((e) => e.id)); + const inScope = (id: string) => !excludedIds.has(id) && (filter === null || filter.test(id)); + const surfaces = allSurfaces.filter((s) => inScope(s.id)); + expect(surfaces.length, "surface filter matched nothing").toBeGreaterThan(0); + // A sliced run measures a slice. It still writes a ledger and still fails + // on any control that does nothing, but it must never print a percentage + // of the whole, because its denominator is not the whole. + const partial = filter !== null; + + const results: Result[] = []; + const swept: Swept[] = []; + + for (const surface of surfaces) { + let controls: Control[] = []; + stage(`enumerating ${surface.id}`); + try { + controls = await enumerateSurface(page, surface); + swept.push({ surface: surface.id, enumerated: controls.length }); + stage(`${surface.id}: ${controls.length} controls`); + } catch (err) { + errors.push(`${surface.id}: could not open (${String(err).split("\n")[0]})`); + continue; + } + if (controls.length === 0) { + errors.push(`${surface.id}: enumerated zero controls`); + continue; + } + // Destructive first, whatever kind of control it is. flip() fills every + // text input and then clicks Save, so a stateful control named "Clear + // memories" or a "Reset to default" select was being activated for real + // while only the click pass consulted isDestructive. + const destructive = controls.filter(isDestructive); + const rest = controls.filter((c) => !isDestructive(c)); + const stateful = rest.filter(isStateful); + const clickable = rest.filter((c) => !isStateful(c)); + for (const ctl of destructive) { + const target = await relocate(page, surface, ctl.key).catch(() => undefined); + results.push( + target + ? await checkWithoutFiring(page, target) + : { + key: ctl.key, + surface: surface.id, + name: ctl.label || ctl.name, + role: ctl.role || ctl.tag, + proven: false, + proof: "none", + detail: "destructive control was not there when the surface was re-opened", + }, + ); + } + + // What this surface does while nobody is touching it. Anything in here + // cannot be counted as a control's network proof. + const chatter = await sampleChatter(page, 1500); + if (chatter.size > 0) { + stage(`${surface.id}: ignoring ${chatter.size} background request signature(s)`); + } + + try { + await clickPass(page, surface, clickable, results, { chatter, blockedWrites }); + } catch (err) { + errors.push(`${surface.id}: click pass aborted (${String(err).split("\n")[0]})`); + } + try { + // A search or filter box on a settings screen is a view control, not a + // setting, and nothing is wrong with it forgetting what was typed. It + // still has to prove that typing lands somewhere. + const transient = (c: Control) => /search|filter/i.test(c.label || c.name); + if (surface.persists) { + await persistencePass(page, surface, stateful.filter((c) => !transient(c)), results); + await valuePass(page, surface, stateful.filter(transient), results); + } else { + await valuePass(page, surface, stateful, results); + } + } catch (err) { + errors.push(`${surface.id}: state pass aborted (${String(err).split("\n")[0]})`); + } + // eslint-disable-next-line no-console + console.log( + `[coverage] ${surface.id}: ${results.filter((r) => r.surface === surface.id && r.proven).length}/${ + results.filter((r) => r.surface === surface.id).length + } proven`, + ); + } + + if (sabotage) { + const blocked = await page + .evaluate(() => window.__covBlocked ?? 0) + .catch(() => 0); + stage(`sabotage "${sabotage}" intercepted ${blocked} events on the last page`); + } + + // Denominator guard, before anything is said about the ratio. This run only + // ever reads the floors. Raising or lowering them is a separate, deliberate + // operation: scripts/update-chat-coverage-floors.mjs, run against a + // recorded ledger, in its own commit, with a reason. A run that could + // rewrite its own bar in the same pass that checks it has no bar at all, + // which is how a real degradation was once ratified as the new baseline. + errors.push(...floorFailures(FLOORS, swept, inScope)); + + const summary = summarise(results); + const registryHits = new Map(); + for (const entry of REGISTRY) { + if (!entry.key) continue; + registryHits.set(entry.key, entry.justification.trim()); + } + // A control nobody fired is not coverage, and it is not a free pass + // either. It fails the gate exactly like a dead control unless + // inert-registry.json carries a justification for its key, which is the + // same escape hatch every other unfireable control uses and which someone + // has to write down and sign. An earlier version dropped every not-fired + // result from the failure list, which would have turned two controls this + // repository's own ledger proves dead into a green gate. + // Exact key, or the identity the key is an instance of. Substring matching + // used to be the rule, and it excuses far more than whoever wrote the entry + // intended: "settings::button::Save" would also excuse + // "settings::button::Save and close". An excuse is a decision about one + // control, so it names one control. + const excused = results.filter((r) => { + if (r.proven) return false; + const justification = + registryHits.get(r.key) ?? registryHits.get(identityOf(r.key)) ?? ""; + return justification.length > 0; + }); + const deferred = results.filter(isDeferred); + const failures = results.filter( + (r) => !r.proven && !excused.some((e) => e.key === r.key), + ); + + fs.mkdirSync(REPORT_DIR, { recursive: true }); + const generatedAt = new Date().toISOString(); + const report = { + generatedAt, + target: process.env.CHAT_URL ?? process.env.OWUI_URL ?? "", + surfaceFilter: process.env.COV_SURFACES ?? null, + partial, + sabotage: sabotage || null, + summary, + swept, + excused: excused.map((r) => r.key), + deferred: deferred.map((r) => r.key), + blockedWrites, + surfaceErrors: errors.map(redactText), + results, + }; + fs.writeFileSync( + path.join(REPORT_DIR, "coverage.run.json"), + JSON.stringify(report, null, 2), + ); + + // Tracker. Assertions below judge this run only; this file carries the + // running total across sliced runs. + // + // It is written from a full sweep only. A slice used to merge into it and + // then print summarise() over the merged set as though it were a total, + // mixing this run's verdicts with whatever a previous run happened to + // leave behind and labelling the result with no marker at all. A number + // built half from now and half from an unknown then is worse than no + // number. + const trackerPath = path.join(REPORT_DIR, "coverage.json"); + const tracker = partial ? {} : readTracker(trackerPath); + if (partial) { + // eslint-disable-next-line no-console + console.log("[coverage] partial run: coverage.json left untouched"); + } + // Drop stale keys before merging. A surface swept in this run is fully + // described by this run, so anything it used to contain and no longer does + // has to go, or a control that was removed (or an enumeration bug that has + // since been fixed) keeps dragging the ratio down forever. + const sweptSurfaces = new Set(results.map((r) => r.surface)); + const seenKeys = new Set(results.map((r) => r.key)); + for (const key of Object.keys(tracker)) { + if (sweptSurfaces.has(tracker[key].surface) && !seenKeys.has(key)) { + delete tracker[key]; + } + } + if (!partial) { + for (const r of results) tracker[r.key] = { ...r, lastSeen: generatedAt }; + fs.writeFileSync( + trackerPath, + JSON.stringify( + { + generatedAt, + target: report.target, + summary: summarise(Object.values(tracker)), + controls: tracker, + }, + null, + 2, + ), + ); + } + const headline = partial + ? [ + `Partial run: surfaces matching ${String(process.env.COV_SURFACES)} only.`, + `Proven ${summary.identitiesProven} of the ${summary.identities} distinct control identities on those surfaces. No percentage is reported for a slice, because its denominator is not the app.`, + ] + : [ + `Proven ${summary.identitiesProven}/${summary.identities} distinct control identities (${(summary.identityRatio * 100).toFixed(1)}%)`, + ``, + `Secondary, not comparable between runs: ${summary.proven}/${summary.total} raw instances (${(summary.ratio * 100).toFixed(1)}%). Instance counts move with account data, for example the number of chat rows, so only the identity figure above should be compared over time.`, + ]; + const lines = [ + `# Chat interaction coverage`, + ``, + ...headline, + ``, + `Checked but deliberately not activated: ${summary.identitiesDeferred} identities (${summary.deferred} instances). A control whose label says it destroys data is asserted present, enabled and named, and never clicked. That is not coverage and is never counted as proof.`, + ``, + `| surface | proven | not fired | total |`, + `| --- | ---: | ---: | ---: |`, + ...Object.entries(summary.surfaces).map( + ([s, v]) => `| ${s} | ${v.proven} | ${v.deferred} | ${v.total} |`, + ), + ``, + `## Unproven`, + ...(failures.length === 0 + ? ["none"] + : failures.map((f) => `- \`${f.key}\`: ${f.detail}`)), + ]; + fs.writeFileSync(path.join(REPORT_DIR, "coverage.md"), lines.join("\n")); + + // eslint-disable-next-line no-console + console.log( + partial + ? `[coverage] PARTIAL slice ${String(process.env.COV_SURFACES)}: ${summary.identitiesProven}/${summary.identities} identities on the swept surfaces, ${summary.deferred} instances not fired. No total for a slice.` + : `[coverage] TOTAL ${summary.identitiesProven}/${summary.identities} identities (${(summary.identityRatio * 100).toFixed(1)}%), ${summary.proven}/${summary.total} instances, ${summary.deferred} not fired`, + ); + + // Redacted here too: a failing assertion's message is the one string that + // reaches the CI log, the JSON report and the HTML report at once. + const reportedErrors = errors.map(redactText); + expect( + reportedErrors, + `surfaces that could not be swept: ${reportedErrors.join(" | ")}`, + ).toEqual([]); + expect( + failures.map((f) => `${f.key} :: ${f.detail}`), + "controls with no proven effect and no registry justification", + ).toEqual([]); + }); + + test("surfaces removed from upstream Open WebUI stay absent", async ({ page }) => { + test.setTimeout(10 * 60_000); + await gotoHome(page); + + const found: string[] = []; + const seen: Array<{ surface: string; label: string }> = []; + const enumerated = new Map(); + + const collect = async (surface: string) => { + const controls = await enumerate(page, surface); + enumerated.set(surface, controls.length); + for (const c of controls) { + seen.push({ surface, label: (c.name || c.label).trim() }); + } + }; + + await openSettings(page); + await collect("settings:"); + await page.keyboard.press("Escape").catch(() => {}); + + await gotoHome(page); + await page.getByRole("button", { name: /user menu/i }).first().click(); + await page.waitForTimeout(1200); + await collect("user-menu"); + await page.keyboard.press("Escape").catch(() => {}); + + await page.goto("/workspace", { waitUntil: "domcontentloaded" }); + await page.waitForTimeout(3000); + await collect("workspace"); + + // Absence only means something if the enumeration happened. A settings + // modal that never opened produces the same empty list as a settings modal + // with nothing forbidden in it, and this test used to call that a pass. + for (const [surface, count] of enumerated) { + expect( + count, + `${surface} enumerated no controls at all, so it proves nothing about what was removed from it`, + ).toBeGreaterThan(0); + } + + for (const rule of REMOVED.forbiddenControls) { + const surfaceRe = new RegExp(rule.surface); + const labelRe = new RegExp(rule.label); + for (const item of seen) { + if (surfaceRe.test(item.surface) && labelRe.test(item.label)) { + found.push(`${item.surface} still renders "${item.label}" (${rule.issue})`); + } + } + } + + for (const route of REMOVED.forbiddenRoutes) { + await page.goto(route.path, { waitUntil: "domcontentloaded" }); + await page.waitForTimeout(2500); + const landed = new URL(page.url()).pathname; + // A bounce to the sign-in page is a dead session, not a removed route. + // Treating it as removal is how this test would pass against an app + // nobody is signed in to. + expect( + landed.startsWith("/auth"), + `${route.path} redirected to the sign-in page, so the session died and absence proves nothing`, + ).toBe(false); + const body = (await page.locator("body").innerText()).toLowerCase(); + expect( + body.trim().length, + `${route.path} rendered an empty document, so absence proves nothing`, + ).toBeGreaterThan(0); + const gone = + body.includes("not found") || body.includes("404") || !landed.startsWith(route.path); + if (!gone) found.push(`${route.path} still renders a page (${route.issue})`); + } + + expect(found, "removed upstream surfaces are still reachable").toEqual([]); + }); +}); + +type TrackedResult = Result & { lastSeen: string }; + +/** + * The running tracker from a previous slice, if there is one. Read defensively: + * a truncated or hand-edited file is not worth failing a 45 minute sweep over, + * and starting from empty only costs this run its cross-slice history. + */ +function readTracker(file: string): Record { + if (!fs.existsSync(file)) return {}; + const out: Record = {}; + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(file, "utf8")); + } catch { + return out; + } + if (typeof parsed !== "object" || parsed === null) return out; + const controls = Object.entries(parsed).find(([name]) => name === "controls")?.[1]; + if (typeof controls !== "object" || controls === null) return out; + for (const [key, value] of Object.entries(controls)) { + if (typeof value !== "object" || value === null) continue; + const entry = new Map(Object.entries(value)); + const surface = entry.get("surface"); + const proofValue = entry.get("proof"); + if (typeof surface !== "string" || typeof proofValue !== "string") continue; + out[key] = { + key, + surface, + name: String(entry.get("name") ?? ""), + role: String(entry.get("role") ?? ""), + proven: entry.get("proven") === true, + proof: proofOf(proofValue), + detail: String(entry.get("detail") ?? ""), + lastSeen: String(entry.get("lastSeen") ?? ""), + }; + } + return out; +} + +const PROOF_NAMES = [ + "navigate", + "network", + "dom", + "overlay", + "download", + "filechooser", + "value", + "persisted", + "not-fired", + "none", +] as const; + +function proofOf(value: string): Result["proof"] { + const match = PROOF_NAMES.find((name) => name === value); + return match ?? "none"; +} diff --git a/apps/web-console/e2e/chat-coverage/data.ts b/apps/web-console/e2e/chat-coverage/data.ts new file mode 100644 index 000000000..437e3dbe9 --- /dev/null +++ b/apps/web-console/e2e/chat-coverage/data.ts @@ -0,0 +1,33 @@ +// The committed data files this gate is configured by, read once and +// validated on the way in. +// +// Separate from the specs so the live sweep and the offline self-checks read +// exactly the same objects through exactly the same validators. A malformed +// file fails loudly here rather than turning into an undefined floor or an +// empty exclusion list at the moment the gate is deciding whether to fail. +import fs from "node:fs"; +import path from "node:path"; + +import { + parseDataDriven, + parseExclusions, + parseFloors, + parseRegistry, + parseRemoved, + type Floors, + type InertEntry, + type RemovedSurfaces, + type SurfaceExclusion, +} from "./lib"; + +export const FLOOR_FILE = path.join(__dirname, "surface-floors.json"); + +function read(file: string): unknown { + return JSON.parse(fs.readFileSync(path.join(__dirname, file), "utf8")); +} + +export const FLOORS: Floors = parseFloors(read("surface-floors.json")); +export const DATA_DRIVEN: Set = parseDataDriven(read("surface-floors.json")); +export const EXCLUSIONS: SurfaceExclusion[] = parseExclusions(read("surface-exclusions.json")); +export const REGISTRY: InertEntry[] = parseRegistry(read("inert-registry.json")); +export const REMOVED: RemovedSurfaces = parseRemoved(read("removed-surfaces.json")); diff --git a/apps/web-console/e2e/chat-coverage/inert-registry.json b/apps/web-console/e2e/chat-coverage/inert-registry.json new file mode 100644 index 000000000..2479b1dbc --- /dev/null +++ b/apps/web-console/e2e/chat-coverage/inert-registry.json @@ -0,0 +1,4 @@ +{ + "$comment": "Controls allowed to have no observable effect. Every entry needs a real justification; an entry with an empty or missing justification fails the gate on its own. Keys are matched as substrings against the control key emitted in coverage.json (surface::role::label).", + "allowed": [] +} diff --git a/apps/web-console/e2e/chat-coverage/lib.ts b/apps/web-console/e2e/chat-coverage/lib.ts new file mode 100644 index 000000000..7a060b746 --- /dev/null +++ b/apps/web-console/e2e/chat-coverage/lib.ts @@ -0,0 +1,1117 @@ +// Live interaction-coverage engine for the Hive chat surface (Open WebUI). +// +// The rule this file exists to enforce: every control the chat app renders has +// to be shown to *do something*, live, against a running deployment. Nothing +// here carries a hand-written list of controls. The control set is read out of +// the rendered DOM on every run, so a control added upstream (or by a Hive +// patch) is enumerated the first time it renders and has to earn a proof or +// fail the gate. A hand-written inventory would go stale silently, which is +// the exact failure mode the owner named. +import type { Page, Locator } from "@playwright/test"; + +import { redactText, redactUrl } from "../../tests/e2e/support/redact"; + +export { redactText, redactUrl }; + +declare global { + interface Window { + /** Mutation records counted by the watcher below. */ + __covMut?: number; + __covObs?: MutationObserver; + /** Events the sabotage hook intercepted, for the self-test. */ + __covBlocked?: number; + } +} + +// Everything a user can reach with a mouse or a keyboard. Roles are included +// alongside tags because Open WebUI builds switches, tabs and menu items out of +// plain