sakibsadmanshajib · sakibsadmanshajib · Sep 2, 2026 · Sep 2, 2026 · Sep 2, 2026 · Sep 2, 2026
@@ -0,0 +1,216 @@
+// Visual proof of the signed-in Hive chat surface.
+//
+// Runs AFTER the `owui-setup` Playwright project, which is what mints the
+// session: it walks the real "Continue with Hive" journey through GoTrue's
+// authorize endpoint, the console's consent screen and back, installs the
+// hive_jwt_forward Functions filter, and saves the result as storage state.
+// This file loads that state into a fresh browser and photographs the surface.
+//
+// WHY THIS IS A PLAIN SCRIPT AND NOT ANOTHER SPEC
+//
+// Nothing here asserts product behaviour, so nothing here wants a test
+// reporter. It wants stamp.mjs, which is the one implementation of the proof
+// footer this repository stamps into a capture, and which is an .mjs module in
+// apps/agent-console. A spec would have to reach across apps into an untyped
+// module from TypeScript; a script imports it in one line. The captures also
+// have to land in docs/proof/, and a Playwright HTML reporter CLEARS its own
+// output directory before writing, which silently deleted a whole capture pass
+// on PR #951.
+//
+// WHAT IT REFUSES TO DO
+//
+// It never passes without an image. A missing chat input, a sign-in page where
+// a signed-in surface was expected, or an assistant turn that never arrives all
+// exit non-zero, because the point of a proof job is proof, and a green run
+// with an empty capture directory is worse than a red one: it carries
+// authority it did not earn.
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { commitStamp, loadChromium, shoot } from "../../../../agent-console/proof/harness/stamp.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+
+const OWUI_URL = (process.env.OWUI_URL ?? "http://localhost:3003").replace(/\/$/, "");
+const OUT_DIR = process.env.PROOF_OUT;
+const NOTE = process.env.PROOF_NOTE ?? "";
+const STATE = process.env.PROOF_STORAGE_STATE ?? join(HERE, ".auth", "owui-user.json");
+const SCENARIO = "chat";
+// The prompt is deliberately one whose right answer is one word, so "the model
+// replied" is checkable rather than eyeballed. Same reasoning, and the same
+// question, as 01-chat-send-stream.spec.ts.
+const PROMPT = "What colour is a banana? Reply with only the single colour word.";
+const REPLY_PATTERN = /yellow/i;
+// Which alias the captured turn is allowed to run on.
+//
+// This is a money guard, not a cosmetic one. The workflow runs on every pull
+// request touching a chat path, and the composer's model is whatever Open WebUI
+// resolved, which is not necessarily what the deployment asked for: on run
+// 33677630521 the run tenant could not see any free alias, Open WebUI silently
+// fell back to the first one it could, and the captured turn ran on a paid
+// model. Nothing failed, nothing said so, and the only evidence was a model
+// name in a screenshot. The owner's 2026-08-30 directive is that CI does not
+// spend on paid aliases, so this asserts the name rather than trusting the
+// configuration to have taken.
+const MODEL_PATTERN = new RegExp(process.env.PROOF_MODEL_PATTERN ?? "free", "i");
+
+if (!OUT_DIR) {
+  console.error("::error::PROOF_OUT is not set, so there is nowhere to write the capture");
+  process.exit(2);
+}
+
+const log = [];
+function record(line) {
+  const stamped = `${new Date().toISOString()}  ${line}`;
+  log.push(stamped);
+  console.log(stamped);
+}
+
+/**
+ * A URL with its query string and fragment removed.
+ *
+ * The journey this harness photographs passes through an OAuth authorize
+ * round trip, and those URLs carry `code`, `authorization_id` and an
+ * access_token fragment. `npm run lint:proof-tokens` would catch a bare one
+ * in this log and fail the build, which is the right backstop and the wrong
+ * place to discover it. Dropping everything after the path means the log
+ * still says where the browser was and can never say with what.
+ */
+function safeUrl(raw) {
+  try {
+    const url = new URL(raw);
+    return `${url.origin}${url.pathname}`;
+  } catch {
+    return "(unparseable url)";
+  }
+}
+
+async function main() {
+  mkdirSync(OUT_DIR, { recursive: true });
+  const chromium = loadChromium();
+  const browser = await chromium.launch();
+  const context = await browser.newContext({
+    storageState: STATE,
+    viewport: { width: 1440, height: 900 },
+    // The stack under proof serves plain http on the runner; this is here for
+    // the JWKS front's local authority, which the browser may meet on a
+    // redirect. It narrows nothing about what is being proven: the surface is
+    // the chat page, not the transport.
+    ignoreHTTPSErrors: true,
+  });
+  const page = await context.newPage();
+  const shots = [];
+
+  try {
+    record(`commit under proof: ${commitStamp()}`);
+    record(`chat origin: ${OWUI_URL}`);
+    record(`session: storage state minted by owui.setup.ts through the real Continue with Hive journey`);
+
+    await page.goto(`${OWUI_URL}/`, { waitUntil: "domcontentloaded" });
+    record(`landed on ${safeUrl(page.url())}`);
+
+    // The negative half matters as much as the positive one, and it has two
+    // shapes. A signed-out browser either gets Open WebUI's own sign-in page on
+    // the same path, or, because this deployment sets OAUTH_AUTO_REDIRECT, is
+    // bounced straight off the chat origin into the authorize chain. A
+    // screenshot alone cannot tell either from a signed-in surface, and a
+    // capture of a login page would be published as proof of chat.
+    const landedOrigin = new URL(page.url()).origin;
+    if (landedOrigin !== new URL(OWUI_URL).origin) {
+      throw new Error(
+        `the browser was redirected off the chat origin to ${landedOrigin}, which is the sign-in chain, so the storage state carries no live session`,
+      );
+    }
+    const signInButton = page.getByRole("button", { name: /continue with hive/i });
+    if (await signInButton.isVisible().catch(() => false)) {
+      throw new Error(
+        "the chat origin served the sign-in page, so the storage state carries no live session",
+      );
+    }
+    const composer = page.locator("#chat-input");
+    await composer.waitFor({ state: "visible", timeout: 60_000 });
+    record("composer is present, so the session is live");
+
+    shots.push(await shoot(page, { outDir: OUT_DIR, scenario: SCENARIO, name: "01-signed-in", note: NOTE }));
+
+    // The model picker, because "which models does this tenant actually see"
+    // is the question a catalog or visibility change is answered with, and it
+    // is invisible in a shot of an empty composer.
+    const picker = page.getByRole("button", { name: /^select(ed)? .*model/i });
+    if (await picker.isVisible().catch(() => false)) {
+      // What the composer will actually send to, read before anything is
+      // opened, because opening the picker changes what the control reads.
+      const selectedModel = (await picker.innerText()).replace(/\s+/g, " ").trim();
+      record(`model in the composer: ${selectedModel}`);
+      if (!MODEL_PATTERN.test(selectedModel)) {
+        throw new Error(
+          `the composer is set to "${selectedModel}", which does not match ${MODEL_PATTERN}. ` +
+            "Capturing a turn on an unexpected alias would spend on it once per pull request; " +
+            "check the run tenant's tenant_model_visibility grant and OWUI_DEFAULT_MODEL.",
+        );
+      }
+      await picker.click();
+      await page.getByRole("option").first().waitFor({ state: "visible", timeout: 15_000 });
+      const models = await page.getByRole("option").allInnerTexts();
+      record(`model picker lists ${models.length} model(s): ${models.join(", ").replace(/\s+/g, " ").trim()}`);
+      shots.push(await shoot(page, { outDir: OUT_DIR, scenario: SCENARIO, name: "02-model-picker", note: NOTE }));
+      await page.keyboard.press("Escape");
+    } else {
+      // Fatal, unlike the shot itself. Without the picker there is no way to
+      // read which alias the turn is about to be billed to, and sending one
+      // blind is the thing the guard above exists to prevent.
+      throw new Error(
+        "the model picker control was not visible, so the alias this turn would run on cannot be read",
+      );
+    }
+
+    await composer.fill(PROMPT);
+    const chatRequest = page.waitForRequest(
+      (request) =>
+        request.method() === "POST" && request.url().includes("/api/chat/completions"),
+      { timeout: 30_000 },
+    );
+    await page.keyboard.press("Enter");
+    await chatRequest;
+    record("chat completion request left the browser");
+
+    // The Copy button appears on an assistant turn only once the stream has
+    // finished, which is what makes it the signal that a reply arrived rather
+    // than that a spinner did.
+    const assistantTurn = page.getByRole("listitem").last();
+    await assistantTurn
+      .getByRole("button", { name: "Copy" })
+      .waitFor({ state: "visible", timeout: 180_000 });
+    const replyText = (await assistantTurn.innerText()).replace(/\s+/g, " ").trim();
+    record(`assistant turn settled: ${replyText.slice(0, 200)}`);
+    if (!REPLY_PATTERN.test(replyText)) {
+      throw new Error(
+        `the assistant turn settled but does not answer the question (expected ${REPLY_PATTERN}), so the surface rendered something other than a working completion`,
+      );
+    }
+
+    shots.push(await shoot(page, { outDir: OUT_DIR, scenario: SCENARIO, name: "03-streamed-reply", note: NOTE }));
+    record(`captured ${shots.length} screenshot(s)`);
+  } catch (error) {
+    record(`::error::${error instanceof Error ? error.message : String(error)}`);
+    // A failure shot is evidence too, and it is the only evidence of what the
+    // page actually looked like when the run went red.
+    await shoot(page, { outDir: OUT_DIR, scenario: SCENARIO, name: "99-failure", note: NOTE }).catch(() => {});
+    throw error;
+  } finally {
+    writeFileSync(join(OUT_DIR, "proof-log.txt"), `${log.join("\n")}\n`);
+    await context.close();
+    await browser.close();
+  }
+
+  if (shots.length === 0) {
+    throw new Error("no screenshots were captured");
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});
@@ -262,10 +262,31 @@ setup("OWUI OIDC sign-in via Hive consent", async ({ page, browser }) => {
   const bootstrapPage = await bootstrapContext.newPage();
   await bootstrapPage.goto(OWUI_URL);
   await signInWithHive(bootstrapPage, bootstrapEmail, bootstrapPassword, owuiOrigin);
+  // Either outcome, because on a FRESH instance this account is deliberately
+  // stuck at the activation screen and cannot reach chat at all.
+  //
+  // utils/oauth.py's get_user_role returns DEFAULT_USER_ROLE early when
+  // `user_count == 0`, above the point where
+  // deploy/docker/owui-patches/tenant_role_from_db.py splices Hive's own
+  // membership lookup in, and issue #748 deleted the post-insert promotion that
+  // upstream used to repair the first account with. On this deployment
+  // DEFAULT_USER_ROLE is "pending", so the very first account ever seen by a
+  // container gets "Account Activation Pending" no matter what tenant
+  // membership it holds. That is the intended posture for an instance shared by
+  // every tenant: nothing self-promotes.
+  //
+  // Which outcome appears therefore depends only on whether the container's
+  // volume is fresh, which is not something this fixture controls and not
+  // something it is testing. What it IS testing is unchanged and asserted by
+  // both arms: a full OAuth round trip completed, and the instance now holds an
+  // account that is not the fixture user. Demanding chat here made this step
+  // fail on every freshly created container, which is every CI run
+  // (run 33673391630).
   await expect(
     bootstrapPage
       .getByRole("button", { name: /new chat/i })
       .or(bootstrapPage.getByRole("link", { name: /new chat/i }))
+      .or(bootstrapPage.getByText(/activation pending/i))
       .first(),
   ).toBeVisible({ timeout: 60_000 });
   await bootstrapContext.close();

@@ -50,6 +50,28 @@ export default defineConfig({
       dependencies: ["owui-setup"],
       use: { ...devices["Desktop Chrome"] },
     },
+    {
+      // The visual proof capture (.github/workflows/chat-visual-proof.yml).
+      // Depends on owui-setup for its session the same way the two projects
+      // above do, so the proof job never has to invoke a setup project on its
+      // own: tools/verify-spec-wiring.mjs excludes setup files from its spec
+      // universe and correctly rejects an invocation that selects only one.
+      //
+      // Gated on hasCreds like the others, so a fork or a local run missing the
+      // seeded values skips it rather than failing on a blank credential.
+      name: "owui-proof",
+      // Anchored on the parent directory, not a bare `proof/`. Playwright
+      // matches testMatch against the ABSOLUTE path, so `/proof\//` also
+      // matches any checkout whose own path contains that segment: measured in
+      // a worktree named ci-chat-visual-proof, where this project collected
+      // every owui spec in the tree.
+      testMatch: hasCreds ? /owui\/proof\/[^/]+\.spec\.ts$/ : [],
+      dependencies: ["owui-setup"],
+      // The capture waits on a real provider stream. The spec sets its own
+      // per-test timeout on top of this.
+      timeout: 420_000,
+      use: { ...devices["Desktop Chrome"] },
+    },
     {
       // Logs in from scratch against a deployed OWUI_URL, so it takes no
       // storageState and depends on no setup project. The spec skips itself

@@ -0,0 +1,52 @@
+import { test } from "@playwright/test";
+import { spawn } from "node:child_process";
+import path from "node:path";
+
+// Visual proof of the signed-in chat surface, run as a spec so that the session
+// it needs arrives the way every other Open WebUI run gets one: the `owui-proof`
+// project depends on `owui-setup`, which walks the real "Continue with Hive"
+// journey and saves the storage state.
+//
+// WHY THIS IS A SPEC WRAPPING A SCRIPT, RATHER THAN EITHER ONE ALONE
+//
+// The capture itself stays a standalone Node script. It imports stamp.mjs, the
+// one implementation of the proof footer this repository stamps into an image,
+// which lives in apps/agent-console and is untyped; and it writes into
+// docs/proof/, because a Playwright HTML reporter CLEARS its own output
+// directory before writing and silently deleted a whole capture pass on PR #951.
+//
+// But a workflow step that ran the script directly had to invoke the setup
+// project on its own to get a session first, and tools/verify-spec-wiring.mjs
+// rejects that by design: setup files are deliberately excluded from its spec
+// universe, so an invocation selecting only `owui-setup` selects zero specs and
+// reads as a broken selector. It is right to reject it. This shape gives the
+// guard a real spec to see, gives the capture its session through the ordinary
+// dependency mechanism, and leaves the script runnable by hand.
+//
+// A spawned child rather than an import, for the same reason owui.setup.ts
+// spawns its two in-container installers: the child is an .mjs module in
+// another app, and running it needs no declaration file and no allowJs.
+//
+// Awaited spawn, never the sync form. A synchronous child blocks the Playwright
+// worker's event loop, which stops the timeout below from firing at all while
+// the capture runs, and PR #838 found a test recorded as passed at 30194 ms
+// under a 30000 ms timeout for exactly that reason
+// (tools/lint-no-sync-child-process-in-tests.mjs). stdio is inherited so the
+// capture log reaches the run log live, which is where a failure is read.
+test("signed-in chat surface, captured", async () => {
+  // The capture waits up to 180 seconds for a provider to finish streaming, on
+  // top of a page load and three full-page screenshots. The default 60 second
+  // test timeout would cut it off mid-stream and report a Playwright timeout
+  // instead of the provider's own failure.
+  test.setTimeout(360_000);
+
+  const script = path.resolve(__dirname, "..", "capture-chat-proof.mjs");
+  await new Promise<void>((resolve, reject) => {
+    const child = spawn("node", [script], { stdio: "inherit" });
+    child.on("error", reject);
+    child.on("close", (code, signal) => {
+      if (code === 0) resolve();
+      else reject(new Error(`capture-chat-proof.mjs exited with code ${code} signal ${signal}`));
+    });
+  });
+});
@@ -22,14 +22,37 @@ export async function signInWithHive(
   password: string,
   owuiOrigin: string,
 ): Promise<void> {
-  // ponytail: OWUI login page has a continuously animating element, so
-  // Playwright's click-stability check never settles, and its force-click still
-  // requires the element in the viewport, which fails during the same
-  // animation. dispatchEvent fires the DOM click handler directly, regardless of
-  // geometry, stability, or overlays.
   const hiveButton = page.getByRole("button", { name: /continue with hive/i });
-  await expect(hiveButton).toBeVisible({ timeout: 30_000 });
-  await hiveButton.dispatchEvent("click");
+
+  // Click the button if it is there, and do not require it to be.
+  //
+  // This deployment sets OAUTH_AUTO_REDIRECT (deploy/docker/docker-compose.yml),
+  // so Open WebUI's landing page starts the authorize chain by itself. Whether a
+  // given load paints the button first or bounces before it can be clicked is a
+  // race with no bearing on what this helper proves, and losing it left the
+  // browser sitting on the console's own sign-in form while a 30 second
+  // assertion waited for a button on an origin the page had already left.
+  // Measured on run 33676212820, where the same run won that race on one login
+  // and lost it on the next.
+  //
+  // Nothing is weakened by accepting either. The assertion that actually
+  // matters is the one below, which requires the browser to have reached the
+  // consent origin, and it is unchanged: a page that neither offers the button
+  // nor redirects still fails there, by name.
+  const stillOnOwui = () => new URL(page.url()).origin === owuiOrigin;
+  const buttonDeadline = Date.now() + 30_000;
+  while (Date.now() < buttonDeadline && stillOnOwui()) {
+    if (await hiveButton.isVisible().catch(() => false)) {
+      // ponytail: OWUI login page has a continuously animating element, so
+      // Playwright's click-stability check never settles, and its force-click
+      // still requires the element in the viewport, which fails during the same
+      // animation. dispatchEvent fires the DOM click handler directly,
+      // regardless of geometry, stability, or overlays.
+      await hiveButton.dispatchEvent("click").catch(() => {});
+      break;
+    }
+    await page.waitForTimeout(250);
+  }
 
   // The OAuth click starts a real full-page redirect chain: OWUI -> Supabase
   // authorize -> /oauth/consent (web-console origin, unauthenticated) ->

@@ -18,6 +18,7 @@
     "e2e:verify-collection": "node scripts/verify-spec-collection.mjs",
     "e2e:owui": "playwright test --config=e2e/phase-19/owui/playwright.owui.config.ts --project=owui",
     "e2e:owui:perf": "playwright test --config=e2e/phase-19/owui/playwright.owui.config.ts --project=owui-perf",
+    "e2e:owui:proof": "playwright test --config=e2e/phase-19/owui/playwright.owui.config.ts --project=owui-proof",
     "e2e:chat-coverage": "playwright test --config=e2e/chat-coverage/playwright.chat-coverage.config.ts --project=chat-coverage",
     "e2e:chat-coverage:self-check": "playwright test --config=e2e/chat-coverage/playwright.chat-coverage.config.ts --project=chat-coverage-break-proof",
     "chat-coverage:floors": "node scripts/update-chat-coverage-floors.mjs"