test: live interaction coverage gate for the chat surface

.github/workflows/chat-coverage.yml

@@ -0,0 +1,145 @@
+name: Chat interaction coverage
+
+# Two jobs, deliberately gated differently.
+#
+#   self-check   Runs everywhere, on every pull request that touches the gate.
+#                Needs no deployment, no session and no secrets: it serves its
+#                own fixture from an intercepted origin and asserts the prover
+#                still calls a dead button dead. This is the part that keeps
+#                the gate from rotting into something that cannot go red.
+#
+#   live-sweep   The real sweep, against the deployed chat surface. It needs a
+#                running deployment and a minted session, so it never fires on
+#                a plain pull request: run it by hand, or label a pull request
+#                run-chat-coverage. The same reason owui-nightly is not a
+#                per-pull-request gate.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+  push:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      surfaces:
+        description: 'Surface filter (a regular expression, blank for a full sweep)'
+        required: false
+        default: ''
+
+concurrency:
+  group: chat-coverage-${{ github.ref }}
+  cancel-in-progress: true
+
+# Read only, and narrowed to what is actually read: the self-check job asks the
+# issues API whether a surface exclusion's blocking issue has closed. Nothing
+# here writes to the repository, and a job that walks a live OAuth flow should
+# not be holding a token that could.
+permissions:
+  contents: read
+  issues: read
+
+jobs:
+  self-check:
+    name: Chat coverage self-check (no deployment needed)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: apps/web-console
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Leave no usable git credential in the runner's config for the rest
+          # of the job, which runs a browser against a live deployment.
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: apps/web-console/package-lock.json
+      - name: Install dependencies
+        run: npm ci
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+      - name: Prover break-proof, registry and exclusion expiry
+        # GITHUB_TOKEN is what the exclusion-expiry check reads issue state
+        # with. It fails rather than skips without one, on purpose: an
+        # exclusion whose blocking issue has closed has to become a failure
+        # the day it closes, and a check that skips itself cannot do that.
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+        run: npm run e2e:chat-coverage:self-check
+
+  live-sweep:
+    name: Chat coverage live sweep
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       contains(github.event.pull_request.labels.*.name, 'run-chat-coverage'))
+    defaults:
+      run:
+        working-directory: apps/web-console
+    env:
+      CHAT_URL: ${{ vars.CHAT_URL || 'https://chat-hive.scubed.co' }}
+      CONSOLE_URL: ${{ vars.CONSOLE_URL || 'https://console-hive.scubed.co' }}
+      # An existing account to mint a session for. live-auth.mjs mints through
+      # the admin one-time-token flow, so no password is read and none is
+      # changed; HIVE_QA_TESTER_PASSWORD is deliberately not passed in here.
+      HIVE_QA_AGENT_EMAIL: ${{ secrets.HIVE_QA_AGENT_EMAIL || secrets.HIVE_QA_TESTER_EMAIL }}
+      SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+      SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }}
+      SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+      COV_SURFACES: ${{ inputs.surfaces }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Leave no usable git credential in the runner's config for the rest
+          # of the job, which runs a browser against a live deployment.
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: apps/web-console/package-lock.json
+      - name: Check the inputs the sweep cannot run without
+        # Loud, not quiet. A missing secret here used to leave the whole
+        # config matching zero files, which reports a green run of nothing.
+        run: |
+          set -euo pipefail
+          missing=""
+          for v in CHAT_URL CONSOLE_URL HIVE_QA_AGENT_EMAIL SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY; do
+            [ -n "${!v:-}" ] || missing="$missing $v"
+          done
+          if [ -n "$missing" ]; then
+            echo "::error::the live sweep needs:$missing"
+            exit 1
+          fi
+      - name: Install dependencies
+        run: npm ci
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+      - name: Sweep the chat surface
+        run: npm run e2e:chat-coverage
+      - name: Upload the coverage ledger
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: chat-coverage-ledger
+          # Two named files, never a directory. Everything this suite writes
+          # itself goes through redactUrl, but a directory upload also sweeps
+          # up whatever the framework wrote beside them: Playwright's JSON and
+          # HTML reports carry raw error text, and a sign-in that outruns its
+          # ninety second wait quotes the OAuth callback with a live code and
+          # state in the timeout message. Those reports are configured to land
+          # outside this directory, and naming the files here means a future
+          # reporter added to the config cannot quietly rejoin the artifact.
+          # Traces and videos are off for the same reason and cannot be
+          # inspected by the text lint at all.
+          path: |
+            apps/web-console/chat-coverage-report/coverage.run.json
+            apps/web-console/chat-coverage-report/coverage.md
+          if-no-files-found: error
+          retention-days: 14

.gitignore

@@ -67,6 +67,9 @@ __pycache__/
 # named anything other than test-results/ was landing traces and videos in
 # `git status` where an `add -A` could commit them.
 /apps/web-console/test-results*/
+/apps/web-console/playwright-report-chat-coverage/
+/apps/web-console/chat-coverage-report/
+/apps/web-console/e2e/chat-coverage/.auth/
 # Added by code-review-graph
 .code-review-graph/
 

apps/web-console/e2e/chat-coverage/auth.setup.ts

@@ -0,0 +1,36 @@
+// Signs the coverage sweep in and saves storageState.
+//
+// signInToChat lives in tests/e2e/support/live-auth.ts, and is the only
+// implementation of the chat hop: the mint itself stays in live-auth.mjs,
+// which this shells out to, so the service-role key never enters this worker's
+// module graph. It mints a session the way a magic link does, so no password
+// is read, written or rotated, then carries it through the real "Continue with
+// Hive" OIDC hop, which is the only thing that gets Open WebUI's own session
+// cookie set. See docs/live-test-auth.md.
+//
+// This project records no trace, video or screenshot (see the config): the
+// callback URL it walks through carries a live `code` and `state`, and no text
+// lint can read inside a trace or a video to catch them.
+import path from "node:path";
+
+import { test as setup } from "@playwright/test";
+
+import { signInToChat } from "../../tests/e2e/support/live-auth";
+
+const STATE = path.join(__dirname, ".auth", "state.json");
+const CHAT = process.env.CHAT_URL ?? process.env.OWUI_URL ?? "";
+const CONSOLE = process.env.CONSOLE_URL ?? "";
+const EMAIL = process.env.OWUI_E2E_EMAIL ?? process.env.HIVE_QA_AGENT_EMAIL ?? "";
+
+setup("authenticate against the chat surface", async ({ browser }) => {
+  setup.setTimeout(6 * 60_000);
+  if (!CHAT || !CONSOLE || !EMAIL) {
+    setup.skip(true, "CHAT_URL, CONSOLE_URL and an account are all required");
+    return;
+  }
+
+  const context = await browser.newContext({ viewport: { width: 1440, height: 950 } });
+  await signInToChat(context, { email: EMAIL, consoleUrl: CONSOLE, chatUrl: CHAT });
+  await context.storageState({ path: STATE });
+  await context.close();
+});