diff --git a/admin-ui/__specs__/health-tile-llm-cause.spec.tsx b/admin-ui/__specs__/health-tile-llm-cause.spec.tsx index 82687109..06692e4c 100644 --- a/admin-ui/__specs__/health-tile-llm-cause.spec.tsx +++ b/admin-ui/__specs__/health-tile-llm-cause.spec.tsx @@ -1,17 +1,177 @@ +// @jest-environment jsdom // STORY-353 — A red LLM tile names its cause (SPEC F139.2, the admin-ui half · PLAN T334) // -// BDD specification — Jest (jsdom). PENDING via it.todo until T334 builds the surface; -// the backend half rides tests/GenWave.Host.Tests/Specs/Story353_LlmCauseTaxonomy.cs. +// BDD specification — Jest (jsdom). The backend half rides +// tests/GenWave.Host.Tests/Specs/Story353_LlmCauseTaxonomy.cs (the /api/llm-calls surface) and +// Story125_LlmStatus.cs (the /api/status llm.dominantCause* fields this file's own tile reads). +// +// StatusTiles is a pure, prop-driven presentational component (`{ status, error, timeZone }` — no +// fetch of its own; DashboardView owns polling via usePoll, Q5/STORY-087) — these specs render it +// directly with a built `status` prop, mirroring safe-scope-tile.spec.tsx's own idiom, rather than +// standing up DashboardView's three-endpoint fetch mock: the tile rides the EXISTING /api/status +// poll (no new poller, the gh-#558 lesson), so there is nothing fetch-shaped left to prove here — +// dashboard-llm-tile.spec.tsx already covers that this data arrives via that one poll. + +import { describe, it, expect, jest, afterEach } from "@jest/globals"; +import { render, screen, cleanup } from "@testing-library/react"; +import "@testing-library/jest-dom/jest-globals"; +import { StatusTiles } from "../app/(authed)/dashboard/StatusTiles"; +import type { StatusResponse } from "@/lib/broadcast-api"; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +interface LlmOverrides { + enabled?: boolean; + model?: string | null; + lastOutcome?: "ok" | "failed" | null; + dominantCause?: string | null; + dominantCauseCount?: number | null; + dominantCauseModel?: string | null; +} + +/** Catalog/SafeScope/Voice are fixed benign values — these specs exercise the LLM tile's own + * dominant-cause line only. */ +function makeStatus(llm: LlmOverrides = {}): StatusResponse { + return { + startedAt: "2026-01-01T08:00:00.000Z", + catalog: { ready: 10, enriching: 0, failed: 0, unavailable: 0 }, + safeScope: { libraryIds: [1], playable: 5 }, + llm: { + enabled: true, + model: "gemma3:12b", + activePersona: null, + lastOutcome: "ok", + lastAttemptAt: "2026-01-01T07:59:00.000Z", + dominantCause: null, + dominantCauseCount: null, + dominantCauseModel: null, + ...llm, + }, + voice: { engine: "kokoro", degraded: false, reason: null, checkedAt: null }, + }; +} -import { describe, it } from "@jest/globals"; +afterEach(() => { + cleanup(); + jest.restoreAllMocks(); +}); + +// --------------------------------------------------------------------------- +// Feature: a red LLM tile names its cause +// --------------------------------------------------------------------------- describe("Feature: a red LLM tile names its cause", () => { describe("Scenario: the tile explains a red verdict", () => { - it.todo("names the dominant recent cause when the LLM verdict is red (T334)"); - it.todo("names the model alongside the cause (T334)"); + it("names the dominant recent cause when the LLM verdict is red (T334)", () => { + render( + + ); + + // SPEC F139.2's own worked example shape ("red: 6 timeouts…"), sentence-cased per house + // copy rule — pluralized since the count is 6, not 1. + expect(screen.getByText(/Red: 6 timeouts/)).toBeInTheDocument(); + }); + + it("names the model alongside the cause (T334)", () => { + render( + + ); + + const line = screen.getByText(/Red: 6 timeouts/); + expect(line).toHaveTextContent("gemma3:12b"); + }); + + it("singularizes the noun for a count of exactly one", () => { + render( + + ); + + expect(screen.getByText("Red: 1 connection failure in the last 24h, llama3.1:8b")).toBeInTheDocument(); + }); + + // T334 review round 1, advisory c — the same "never drop an unknown kind" discipline + // LlmCallsFeed.tsx's CAUSE_LABELS already follows: a cause value shipped by the api ahead of an + // admin-ui label update still renders, unstyled, as its raw wire token — never `undefined`, + // never a thrown render. + it("renders the raw wire token for a cause this tile has no specific label for", () => { + render( + + ); + + expect(screen.getByText("Red: 2 somenewcause in the last 24h, gemma3:12b")).toBeInTheDocument(); + }); }); describe("Scenario: quiet states stay quiet", () => { - it.todo("shows no cause line when the LLM verdict is green (T334)"); + it("shows no cause line when the LLM verdict is green (T334)", () => { + render( + + ); + + expect(screen.queryByText(/Red:/)).not.toBeInTheDocument(); + }); + + it("shows no cause line when the LLM is disabled", () => { + render(); + + expect(screen.queryByText(/Red:/)).not.toBeInTheDocument(); + }); + + it("shows no cause line for a failed verdict with nothing yet to explain it", () => { + // The three dominantCause* fields travel together (broadcast-api.ts's own remarks) — a + // failed verdict with none of them set renders the existing failure line alone, never an + // empty or malformed "why" line. + render(); + + expect(screen.getByText(/Last completion failed/)).toBeInTheDocument(); + expect(screen.queryByText(/Red:/)).not.toBeInTheDocument(); + }); }); }); diff --git a/admin-ui/__specs__/llm-calls-page.spec.tsx b/admin-ui/__specs__/llm-calls-page.spec.tsx index c465dd2c..1b6557df 100644 --- a/admin-ui/__specs__/llm-calls-page.spec.tsx +++ b/admin-ui/__specs__/llm-calls-page.spec.tsx @@ -35,6 +35,10 @@ interface EntryOverrides { promptChars?: number; responseChars?: number; kind?: string; + /** SPEC F139.1, STORY-353, PLAN T334 */ + cause?: string; + /** SPEC F139.2, STORY-353, PLAN T334 */ + model?: string; } function makeEntry(overrides: EntryOverrides = {}) { @@ -52,14 +56,20 @@ function makeEntry(overrides: EntryOverrides = {}) { promptChars: 100, responseChars: 48, kind: "copy", + cause: "success", + model: "test-model", ...overrides, }; } type MockResult = { kind: "ok"; body: unknown } | { kind: "network-error" }; -function ok(body: unknown): MockResult { - return { kind: "ok", body }; +/** Wraps `calls` in the SPEC F139.2/PLAN T334 response shape (`{ calls, causeSummary }` — + * `GenWave.Host.Api.LlmCallsResponseDto`) — every call site below still just hands this the flat + * array of entries it always did; `causeSummary` is empty since none of these facts exercise it + * (this page doesn't render it — see lib/llm-calls-api.ts's own remarks). */ +function ok(calls: unknown[]): MockResult { + return { kind: "ok", body: { calls, causeSummary: [] } }; } function networkError(): MockResult { @@ -209,6 +219,32 @@ describe("Feature: LLM call inspector", () => { }); }); + // SPEC F139.1, STORY-353, PLAN T334 — Cause is Status's own finer-grained sibling: an operator + // can tell a timeout apart from a connection failure without expanding into the raw prompt/ + // response text. + describe("Scenario: rows show why the call resolved the way it did", () => { + it("renders a Timeout cause chip", async () => { + installFetchMock(ok([makeEntry({ status: "failed", cause: "timeout" })])); + + render(); + await flush(); + + expect(screen.getByText("Timeout")).toBeInTheDocument(); + }); + + it("renders the raw wire value for a cause this UI has no specific label for", async () => { + // The "never drop an unknown kind" discipline STATUS_LABELS/KIND_LABELS already follow — a + // taxonomy value shipped by the api ahead of an admin-ui label update still renders, + // unstyled, rather than vanishing (SPEC F139.1's own history: eight values as of T330). + installFetchMock(ok([makeEntry({ cause: "somenewcause" })])); + + render(); + await flush(); + + expect(screen.getByText("somenewcause")).toBeInTheDocument(); + }); + }); + // gh-#429: personas now author the copy this table exists to triage, so each row names who // authored it without an admin having to expand into the system prompt to find out. describe("Scenario: rows show which persona authored the call", () => { diff --git a/admin-ui/app/(authed)/booth-log/LlmCallsFeed.tsx b/admin-ui/app/(authed)/booth-log/LlmCallsFeed.tsx index e1c4f3c1..11e11dcf 100644 --- a/admin-ui/app/(authed)/booth-log/LlmCallsFeed.tsx +++ b/admin-ui/app/(authed)/booth-log/LlmCallsFeed.tsx @@ -31,6 +31,21 @@ const MODE_LABELS: Record = { normal: "Normal", soft: "Soft", ha * "never drop an unknown kind" discipline STATUS_LABELS/MODE_LABELS already follow). */ const KIND_LABELS: Record = { copy: "Copy", crosstalk: "Crosstalk" }; +/** SPEC F139.1, STORY-353, PLAN T334 — WHY this call resolved the way it did + * (GenWave.Tts.LlmCallCause), a finer-grained sibling of STATUS_LABELS above. Keyed on the wire's + * own lowercase, no-separator enum spelling (mirrors STATUS_LABELS/MODE_LABELS/KIND_LABELS' own + * convention); an unstyled cause this UI doesn't specifically label still shows its raw text. */ +const CAUSE_LABELS: Record = { + success: "Success", + timeout: "Timeout", + overlength: "Over length", + truthgatereject: "Truth-gate reject", + connectionfailure: "Connection failure", + canceledbywindow: "Canceled by window", + emptycompletion: "Empty completion", + malformedresponse: "Malformed response", +}; + /** gh-#142: the MODE column is the F69/F70 degradation ladder, which nothing on the page said — * a native-title flyover per chip explains the rung without spending any screen real estate. * Wording tracks DegradationMode's own docs. gh-#210: these same three lines also feed the Mode @@ -104,6 +119,14 @@ function KindChip({ kind }: { kind: string }): ReactNode { return {KIND_LABELS[kind] ?? kind}; } +/** SPEC F139.1, STORY-353, PLAN T334 — neutral tone, same reasoning as KindChip immediately + * above: Cause is supplementary detail alongside the Status chip's already-colored verdict + * ("failed" is danger; "timeout" just names which kind of failed it was), never a second, + * redundant severity judgment painted onto the same row. */ +function CauseChip({ cause }: { cause: string }): ReactNode { + return {CAUSE_LABELS[cause] ?? cause}; +} + function ModeChip({ mode }: { mode: string }): ReactNode { return ( @@ -124,9 +147,10 @@ function DetailField({ label, value }: { label: string; value: string | null }): } /** - * The LLM call inspector's table (PLAN T41, STORY-196, SPEC F73.1-F73.2; persona column gh-#429): - * newest-first rows of time / persona / status chip / mode chip / elapsed / a truncated response - * preview, each expandable to the full system prompt, user prompt, and raw response text — + * The LLM call inspector's table (PLAN T41/T334, STORY-196/353, SPEC F73.1-F73.2, F139.1; persona + * column gh-#429): newest-first rows of time / persona / kind chip / status chip / cause chip / + * mode chip / elapsed / a truncated response preview, each expandable to the full system prompt, + * user prompt, and raw response text — * admin-only debug detail, never persisted (see GenWave.Tts.LlmCallRing's own remarks). Loading/ * empty/error idioms match the booth log's own BoothLogFeed (skeleton rows, EmptyState, a quiet * unavailable hint on a poll failure that keeps whatever was already loaded). @@ -180,6 +204,11 @@ export function LlmCallsFeed({ entries, error, timeZone }: LlmCallsFeedProps): R Persona Kind Status + {/* SPEC F139.1, STORY-353, PLAN T334 — Cause is Status's own finer-grained sibling + ("failed" -> "timeout"/"connectionfailure"/…), a column of its own rather than + folded into Status so an operator can still scan Status's coarse ok/failed + read without every row's cell text growing. */} + Cause {/* gh-#210: the header carries the house `?` flyover (gh-#145 pattern) — the per-chip native titles above survive for hover users, but touch and anyone @@ -225,6 +254,9 @@ export function LlmCallsFeed({ entries, error, timeZone }: LlmCallsFeedProps): R + + + @@ -245,7 +277,7 @@ export function LlmCallsFeed({ entries, error, timeZone }: LlmCallsFeedProps): R {isExpanded && ( - +
diff --git a/admin-ui/app/(authed)/booth-log/useLlmCalls.ts b/admin-ui/app/(authed)/booth-log/useLlmCalls.ts index 7bea0b9d..fa189b91 100644 --- a/admin-ui/app/(authed)/booth-log/useLlmCalls.ts +++ b/admin-ui/app/(authed)/booth-log/useLlmCalls.ts @@ -9,7 +9,9 @@ import { fetchLlmCalls, type LlmCallEntry } from "@/lib/llm-calls-api"; const LLM_CALLS_POLL_INTERVAL_MS = 12000; export interface UseLlmCallsResult { - /** Newest-first, exactly as the endpoint returns it. `null` until the first poll resolves. */ + /** Newest-first, the `calls` half of the endpoint's own `{ calls, causeSummary }` response (SPEC + * F139.2, PLAN T334 — `fetchLlmCalls` unwraps it; see that function's own remarks for why + * `causeSummary` stops there and never reaches this hook). `null` until the first poll resolves. */ entries: LlmCallEntry[] | null; /** True when the most recent poll failed; `entries` is left untouched (usePoll's contract — a * caller renders a quiet degrade, never discards what's already loaded). */ diff --git a/admin-ui/app/(authed)/dashboard/StatusTiles.tsx b/admin-ui/app/(authed)/dashboard/StatusTiles.tsx index b1e6cb98..bfe021a0 100644 --- a/admin-ui/app/(authed)/dashboard/StatusTiles.tsx +++ b/admin-ui/app/(authed)/dashboard/StatusTiles.tsx @@ -39,6 +39,57 @@ function llmTileVariant(llm: StatusResponse["llm"]): "neutral" | "ok" | "warning return llm.lastOutcome === "failed" ? "warning" : "ok"; } +/** + * SPEC F139.2, STORY-353, PLAN T334 — the sentence-ready noun phrase for one dominant-cause count, + * singular/plural handled (mirrors `playableTracksCaption`'s own convention one tile over). Keyed + * on the wire's own lowercase, no-separator enum spelling (`GenWave.Tts.LlmCallCause.ToString()`, + * SPEC F73.1's existing `status`/`mode` convention) — the station's own words for each cause, + * rather than the wire's terse identifier leaking straight onto the tile. + * + * No `success` key: `LlmCallCauseCounters.DominantFailure` (the api-side read this line's own + * `dominantCause` comes from) filters `Success` out at the source — `llm.dominantCause` can never + * carry it, so a map entry for it would be dead weight, not a missing case. + * + * `canceledbywindow`/`malformedresponse` ARE kept even though `DominantFailure` is called scoped to + * `LlmCallKind.Copy` here (see `StatusController`'s own remarks) and `GenWave.Tts.LlmCopyWriter` + * never stamps either cause — only the crosstalk lane (`CrosstalkStockWorker`/ + * `CrosstalkScriptParser`) does. Both are unreachable on THIS tile today, by construction, not by + * omission — left in so this map stays a complete mirror of `LlmCallCause` (the same "never drop an + * unknown kind" discipline `CAUSE_LABELS` in `LlmCallsFeed.tsx` already follows for the OTHER, + * kind-unscoped surface) rather than something a future edit "fixes" by deleting two lines that look + * unused. + */ +const DOMINANT_CAUSE_NOUNS: Record = { + timeout: ["timeout", "timeouts"], + overlength: ["over-length reply", "over-length replies"], + truthgatereject: ["truth-gate reject", "truth-gate rejects"], + connectionfailure: ["connection failure", "connection failures"], + canceledbywindow: ["break-window cancellation", "break-window cancellations"], + emptycompletion: ["empty reply", "empty replies"], + malformedresponse: ["malformed reply", "malformed replies"], +}; + +/** + * SPEC F139.2, STORY-353, PLAN T334 — the red tile's "why" line, e.g. "Red: 6 timeouts in the + * last 24h, gemma3:12b" (the F139.2 worked example's own shape, sentence-cased per house copy + * rule). `null` whenever the api has nothing to explain (`dominantCause`/`dominantCauseCount`/ + * `dominantCauseModel` travel together — see `StatusResponse.llm`'s own remarks) — the caller only + * invokes this once the tile is already red, so a `null` here is simply unused, never rendered as + * an empty line. Names the true rolling window (24h, SPEC F139.2's own retention) rather than the + * spec's illustrative "last hour" — the tile never claims a narrower window than the counters + * actually track. + */ +function dominantCauseLine(llm: StatusResponse["llm"]): string | null { + const cause = llm.dominantCause; + const count = llm.dominantCauseCount; + const model = llm.dominantCauseModel; + if (cause == null || count == null || model == null) return null; + + const [singular, plural] = DOMINANT_CAUSE_NOUNS[cause] ?? [cause, cause]; + const noun = count === 1 ? singular : plural; + return `Red: ${count} ${noun} in the last 24h, ${model}`; +} + /** SPEC F99.5, F100.3, STORY-256 AC4 — the Voice tile has no "disabled" state (the primary engine * is always configured): "warning" when the cached verdict is unhealthy, "ok" otherwise (including * the brief startup window before the first probe cycle completes — a degraded read is never @@ -125,9 +176,14 @@ export function StatusTiles({ status, error, timeZone }: StatusTilesProps): Reac

{status.llm.activePersona}

)} {status.llm.lastOutcome === "failed" && ( -

- Last completion failed — falling back to templated copy -

+ <> +

+ Last completion failed — falling back to templated copy +

+ {/* SPEC F139.2, STORY-353, PLAN T334 — the gh-#365 acceptance ("no SSH, no + Loki, no darts at Llm settings"): a red tile also names WHY, not only THAT. */} + + )} )} @@ -194,6 +250,15 @@ function TileHeadline({ value, caption }: { value: number; caption: string }): R ); } +/** SPEC F139.2, STORY-353, PLAN T334 — the LLM tile's "why" line, or nothing at all when the api + * has no dominant cause to report (see `dominantCauseLine`'s own remarks). A small component + * rather than calling `dominantCauseLine` twice at each call site (`null`-check, then render). */ +function DominantCauseLine({ llm }: { llm: StatusResponse["llm"] }): ReactNode { + const line = dominantCauseLine(llm); + if (line === null) return null; + return

{line}

; +} + function TileSkeleton(): ReactNode { return (
diff --git a/admin-ui/app/(authed)/settings/SettingsForm.tsx b/admin-ui/app/(authed)/settings/SettingsForm.tsx index a42a5534..5c764050 100644 --- a/admin-ui/app/(authed)/settings/SettingsForm.tsx +++ b/admin-ui/app/(authed)/settings/SettingsForm.tsx @@ -413,7 +413,7 @@ const FIELD_HELP_TEXT: Record = { // ── Crosstalk two-voice banter (SPEC F127.4, F127.8) ──────────────────────────────────────── "Crosstalk:DurationTargetSeconds": "The longest a generated two-voice banter exchange may run, in seconds, before it is " + - "discarded and skipped rather than aired. Defaults to 25. Accepted range: 5–120.", + "discarded and skipped rather than aired. Defaults to 50. Accepted range: 5–120.", "Crosstalk:Shows": "A JSON array of show SLUGS allowed to carry two-voice banter — a show's stable URL-safe " + "identity, not its display name (e.g. \"morning-drive\" for a show named \"Morning Drive\"), " + diff --git a/admin-ui/lib/broadcast-api.ts b/admin-ui/lib/broadcast-api.ts index a6a48c2a..f3e1ce90 100644 --- a/admin-ui/lib/broadcast-api.ts +++ b/admin-ui/lib/broadcast-api.ts @@ -60,6 +60,22 @@ export interface StatusResponse { activePersona: string | null; lastOutcome: "ok" | "failed" | null; lastAttemptAt: string | null; + /** + * SPEC F139.2, STORY-353, PLAN T334 — the F139 cause taxonomy's own read of "why is the tile + * red": the highest-count non-success cause GenWave.Tts.LlmCallCauseCounters recorded for + * Copy-kind calls in the rolling 24h window, plus its count and the model it was recorded + * against. All three are `null` together whenever there is nothing to explain (nothing but + * Success recorded, or nothing at all) — the same "quiet is not a fault" posture `lastOutcome` + * above already follows. Optional on the wire (PLAN T334 adds these three fields after several + * other spec files already built their own `StatusResponse` fixture literals): every fixture + * that omits them still satisfies this type, so this addition never forces an edit onto a file + * this task doesn't otherwise touch — mirrors `NowPlayingTrackWire.kind`'s own "one deploy of + * backward tolerance" convention above, minus the deploy: this is a same-task compatibility + * choice, not a rollout one. + */ + dominantCause?: string | null; + dominantCauseCount?: number | null; + dominantCauseModel?: string | null; }; /** * SPEC F99.5, F100.3, STORY-256 AC4 — the primary voice engine's own cached health verdict. diff --git a/admin-ui/lib/llm-calls-api.ts b/admin-ui/lib/llm-calls-api.ts index a2959ccf..aced4d01 100644 --- a/admin-ui/lib/llm-calls-api.ts +++ b/admin-ui/lib/llm-calls-api.ts @@ -1,6 +1,7 @@ -// Client-side wire shape + fetcher for the LLM call inspector (PLAN T41, STORY-196, SPEC -// F73.1-F73.2). Browser fetches go through the Next.js same-origin rewrite (/api/* -> api:8080), -// same convention as lib/booth-log-api.ts — never lib/api.ts's apiGet, which is server-only. +// Client-side wire shape + fetcher for the LLM call inspector (PLAN T41/T334, STORY-196/353, SPEC +// F73.1-F73.2, F139.2). Browser fetches go through the Next.js same-origin rewrite +// (/api/* -> api:8080), same convention as lib/booth-log-api.ts — never lib/api.ts's apiGet, which +// is server-only. /** * One completed LLM call (SPEC F73.1) — `status`/`mode` are plain strings on the wire @@ -28,12 +29,46 @@ export interface LlmCallEntry { * ordinary blurb miss. Plain string, not a closed union, for the same reason `status`/`mode` * are above. */ kind: string; + /** SPEC F139.1, STORY-353, PLAN T334 — WHY this call resolved the way it did + * (GenWave.Tts.LlmCallCause): a finer-grained sibling of `status` above (e.g. a `"failed"` + * status might be a `"timeout"` or a `"connectionfailure"` cause). Plain string, not a closed + * union, same reason as `status`/`mode`/`kind`. */ + cause: string; + /** SPEC F139.2, STORY-353, PLAN T334 — the completions model this call used. Never `null`, same + * as the wire's own `GenWave.Host.Api.LlmCallDto.Model`. */ + model: string; } /** - * GET /api/llm-calls (SPEC F73.1-F73.2) — every call the ring currently holds, newest first. No - * paging: the ring is capped at a small size (~50) by construction, so the whole response is - * always a single, small round-trip. + * GET /api/llm-calls's `causeSummary` (SPEC F139.2, STORY-353, PLAN T334) — one row of the + * rolling 24h by-(cause, model, kind) count, riding the SAME response as {@link LlmCallEntry} + * (see `GenWave.Host.Api.LlmCallsResponseDto`'s own remarks for why one response, not two). Not + * yet consumed by this page's own UI (the dashboard's health tile computes its own dominant-cause + * line from a smaller, `/api/status`-scoped read instead — `broadcast-api.ts`'s own + * `StatusResponse.llm`) — this type exists so the wire's own shape stays fully typed for whichever + * future admin surface reads it. + */ +export interface LlmCallCauseSummaryEntry { + cause: string; + model: string; + kind: string; + count: number; +} + +/** Wire shape of `GET /api/llm-calls` itself (SPEC F139.2, PLAN T334) — mirrors + * `GenWave.Host.Api.LlmCallsResponseDto`. */ +interface LlmCallsResponse { + calls: LlmCallEntry[]; + causeSummary: LlmCallCauseSummaryEntry[]; +} + +/** + * GET /api/llm-calls (SPEC F73.1-F73.2, F139.2) — every call the ring currently holds, newest + * first. No paging: the ring is capped at a small size (~50) by construction, so the whole + * response is always a single, small round-trip. Returns only {@link LlmCallEntry}'s own `calls` + * array — this page's presentational components (`LlmCallsFeed`) never needed the `causeSummary` + * half added alongside it at T334, so there is no reason to thread an unused field through + * `useLlmCalls`/`LlmCallsView` just because the wire happens to carry it. */ export async function fetchLlmCalls(): Promise { const response = await fetch("/api/llm-calls", { @@ -43,5 +78,6 @@ export async function fetchLlmCalls(): Promise { if (!response.ok) { throw new Error(`GET /api/llm-calls failed: ${response.status}`); } - return (await response.json()) as LlmCallEntry[]; + const body = (await response.json()) as LlmCallsResponse; + return body.calls; } diff --git a/src/GenWave.Host/Api/LlmCallCauseSummaryDto.cs b/src/GenWave.Host/Api/LlmCallCauseSummaryDto.cs new file mode 100644 index 00000000..11a3ee38 --- /dev/null +++ b/src/GenWave.Host/Api/LlmCallCauseSummaryDto.cs @@ -0,0 +1,14 @@ +namespace GenWave.Host.Api; + +/// +/// One row of GET /api/llm-calls' causeSummary array (SPEC F139.2, STORY-353, PLAN +/// T334) — a direct projection of : how many calls landed +/// on for / within the rolling 24h window +/// tracks. / are +/// lowercased the same way every other enum-backed field on this endpoint already is (SPEC F73.1's +/// status/mode, F127.11's kind on ) — this is the +/// admin-only debug lens's own aggregate, not a public metrics surface, so plain strings over a +/// closed union keep the wire tolerant of a taxonomy that may still grow (SPEC F139.1's own history: +/// eight values already, up from seven at T330 review). +/// +public sealed record LlmCallCauseSummaryDto(string Cause, string Model, string Kind, int Count); diff --git a/src/GenWave.Host/Api/LlmCallDto.cs b/src/GenWave.Host/Api/LlmCallDto.cs index c73101af..0e8f5c8e 100644 --- a/src/GenWave.Host/Api/LlmCallDto.cs +++ b/src/GenWave.Host/Api/LlmCallDto.cs @@ -11,6 +11,11 @@ namespace GenWave.Host.Api; /// empty string. (SPEC F127.11, PLAN T282) is "copy" for every ordinary /// segment-copy call or "crosstalk" for a call /// — so an operator can tell "why was there no banter" apart from an ordinary blurb miss. +/// / (SPEC F139.1-F139.2, STORY-353, PLAN T334) carry +/// / +/// verbatim, lowercased the same way // +/// already are — the per-row half of the F139 taxonomy reaching this wire; is +/// never for the same reason that field already isn't on the domain record. /// public sealed record LlmCallDto( long Seq, @@ -25,4 +30,6 @@ public sealed record LlmCallDto( string? Response, int PromptChars, int ResponseChars, - string Kind); + string Kind, + string Cause, + string Model); diff --git a/src/GenWave.Host/Api/LlmCallsController.cs b/src/GenWave.Host/Api/LlmCallsController.cs index bf246ab3..7bfbfff7 100644 --- a/src/GenWave.Host/Api/LlmCallsController.cs +++ b/src/GenWave.Host/Api/LlmCallsController.cs @@ -5,32 +5,38 @@ namespace GenWave.Host.Api; /// -/// The LLM call inspector's admin-only read endpoint (SPEC F73.1-F73.2, STORY-196, T41) — a debug -/// lens, NOT an audit trail: every entry currently holds (the last -/// ~ calls — on-air renders, Soft-cadence attempts, and -/// operator previews alike), newest first, full prompt/response text included. Never persisted -/// (F73.3): this endpoint only ever reads the one in-memory singleton -/// LlmCopyWriter.RequestCleanedCompletionAsync records into — nothing here ever touches disk or a -/// database, so a process restart clears it with no explicit "clear" step to forget. Deny-by-default -/// like every other admin route: no , no public reachability -/// (F73.2). +/// The LLM call inspector's admin-only read endpoint (SPEC F73.1-F73.2, F139.2, STORY-196/353, T41, +/// T334) — a debug lens, NOT an audit trail: every entry currently holds +/// (the last ~ calls — on-air renders, Soft-cadence +/// attempts, and operator previews alike), newest first, full prompt/response text included, plus the +/// rolling 24h summary alongside it (see +/// 's own remarks for why one wrapped response, not a second +/// endpoint). Never persisted (F73.3, F139.3): this endpoint only ever reads the two in-memory +/// singletons writes into together — nothing here ever +/// touches disk or a database, so a process restart clears both with no explicit "clear" step to +/// forget. Deny-by-default like every other admin route: no , +/// no public reachability (F73.2). /// [ApiController] [Route("api/llm-calls")] [AdminSurface] [Authorize(Policy = AuthorizationPolicies.PlayoutRead)] -public sealed class LlmCallsController(LlmCallRing ring) : ControllerBase +public sealed class LlmCallsController(LlmCallRing ring, LlmCallCauseCounters causeCounters) : ControllerBase { /// - /// GET /api/llm-calls — every call the ring currently holds, newest first (SPEC F73.1). No - /// paging: the ring is capped at (~50) by construction, - /// so the whole thing is always a small, single-round-trip response. + /// GET /api/llm-calls — every call the ring currently holds, newest first (SPEC F73.1), plus the + /// F139.2 rolling 24h cause counters (SPEC F139.2, PLAN T334) in the SAME response + /// (). No paging: the ring is capped at + /// (~50) by construction, and the counters are already a + /// small, pre-aggregated read () — the whole thing stays + /// a single, small round-trip. /// [HttpGet] public IActionResult List() { - var rows = ring.Snapshot().Select(ToDto).ToList(); - return Ok(rows); + var calls = ring.Snapshot().Select(ToDto).ToList(); + var causeSummary = causeCounters.Snapshot().Select(ToSummaryDto).ToList(); + return Ok(new LlmCallsResponseDto(calls, causeSummary)); } static LlmCallDto ToDto(LlmCallRecord record) => new( @@ -46,5 +52,13 @@ public IActionResult List() record.Response, (record.PromptSystem?.Length ?? 0) + (record.PromptUser?.Length ?? 0), record.Response?.Length ?? 0, - record.Kind.ToString().ToLowerInvariant()); + record.Kind.ToString().ToLowerInvariant(), + record.Cause.ToString().ToLowerInvariant(), + record.Model); + + static LlmCallCauseSummaryDto ToSummaryDto(LlmCallCauseCount count) => new( + count.Cause.ToString().ToLowerInvariant(), + count.Model, + count.Kind.ToString().ToLowerInvariant(), + count.Count); } diff --git a/src/GenWave.Host/Api/LlmCallsResponseDto.cs b/src/GenWave.Host/Api/LlmCallsResponseDto.cs new file mode 100644 index 00000000..80655aee --- /dev/null +++ b/src/GenWave.Host/Api/LlmCallsResponseDto.cs @@ -0,0 +1,18 @@ +namespace GenWave.Host.Api; + +/// +/// Response shape for GET /api/llm-calls (SPEC F73.1-F73.2, F139.2, STORY-196/353, PLAN T334) +/// — mirrors 's own established "array plus metadata, one object" shape +/// one controller over, rather than inventing a second one: is exactly what the +/// endpoint returned bare before this task (STORY-196), newest first, capped at ring size; +/// is the F139.2 rolling 24h counters riding the SAME round trip so the +/// admin llm-calls page never issues a second request just to explain the rows it already has (the +/// gh-#558 "no new chatty poller" lesson applies here too, even off the dashboard's own poll cadence: +/// one request beats two regardless of which page is asking). A bare JSON array has no room for a +/// second, named field alongside it — that is the whole reason this wraps rather than keeping the +/// pre-T334 shape, the "no new endpoint unless the existing shape genuinely can't carry it" call PLAN +/// T334 asked this controller to make. +/// +public sealed record LlmCallsResponseDto( + IReadOnlyList Calls, + IReadOnlyList CauseSummary); diff --git a/src/GenWave.Host/Api/StatusController.cs b/src/GenWave.Host/Api/StatusController.cs index 9717b4e4..bd85f71f 100644 --- a/src/GenWave.Host/Api/StatusController.cs +++ b/src/GenWave.Host/Api/StatusController.cs @@ -22,6 +22,7 @@ public sealed class StatusController( IOptionsMonitor stationMonitor, IOptionsMonitor llmMonitor, LlmCopyStatusHolder llmStatusHolder, + LlmCallCauseCounters llmCauseCounters, DegradationController degradationController, VoiceHealthReader voiceHealthReader, IActivePersonaAccessor personaAccessor, @@ -31,7 +32,7 @@ public sealed class StatusController( /// GET /api/status — cookie-auth (covered by the deny-by-default fallback policy when /// Admin:Password is set, same as every other /api/* controller). Returns: /// { startedAt, catalog: { ready, enriching, failed, unavailable }, safeScope: { libraryIds, playable }, - /// llm: { enabled, model, activePersona, lastOutcome, lastAttemptAt }, + /// llm: { enabled, model, activePersona, lastOutcome, lastAttemptAt, dominantCause, dominantCauseCount, dominantCauseModel }, /// degradation: { mode, pinned, since, cause }, /// voice: { engine, degraded, reason, checkedAt } }. /// @@ -54,6 +55,18 @@ public sealed class StatusController( /// dependency at all, by construction — an idle station polling this endpoint sends the LLM zero /// requests. /// + /// llm.dominantCause/dominantCauseCount/dominantCauseModel (SPEC F139.2, + /// STORY-353, PLAN T334) are 's own read, + /// restricted to — the SAME kind lastOutcome above reflects, + /// so this never names a crosstalk-only cause for a tile that went red over an ordinary segment + /// miss. All three are together whenever nothing but + /// (or nothing at all) was recorded for Copy calls in the + /// rolling 24h window — the Admin UI only renders the line once lastOutcome == "failed" + /// anyway, so a null here on a green tile is simply unused, never a fault. This rides the SAME + /// poll as every other llm.* field (no new endpoint, no new poller — the gh-#558 lesson): + /// is an in-memory read over already-aggregated + /// counters, exactly as cheap as below. + /// /// degradation (SPEC F69.5, STORY-188) comes from /// — called here, not just read from a cached /// field, so a just-applied pin or an elapsed probe cooldown is visible on THIS poll rather than @@ -80,6 +93,7 @@ public async Task Get(CancellationToken ct) var llmConfig = llmMonitor.CurrentValue; var llmEnabled = !string.IsNullOrEmpty(llmConfig.Endpoint); var lastAttempt = llmStatusHolder.Last; + var dominantFailure = llmCauseCounters.DominantFailure(LlmCallKind.Copy); var degradation = degradationController.Evaluate(); var voice = voiceHealthReader.Evaluate(); @@ -107,6 +121,9 @@ public async Task Get(CancellationToken ct) ? null : lastAttempt.Outcome == LlmAttemptOutcome.Ok ? "ok" : "failed", lastAttemptAt = lastAttempt?.AttemptedAt, + dominantCause = dominantFailure?.Cause.ToString().ToLowerInvariant(), + dominantCauseCount = dominantFailure?.Count, + dominantCauseModel = dominantFailure?.Model, }, degradation = new { diff --git a/src/GenWave.Host/Configuration/SettingValidator.cs b/src/GenWave.Host/Configuration/SettingValidator.cs index 1dab5c11..51187c30 100644 --- a/src/GenWave.Host/Configuration/SettingValidator.cs +++ b/src/GenWave.Host/Configuration/SettingValidator.cs @@ -189,11 +189,12 @@ public SettingValidator(IConfiguration configuration, ThemeCatalog? themeCatalog // a negative value would still resolve safely if it slipped through some other path. internal const int ContextPersonaIdMin = 0; - // Crosstalk:DurationTargetSeconds (SPEC F127.4, STORY-326, PLAN T282) — CrosstalkOptions' own - // [Range(1, int.MaxValue)] (boot-enforced via ValidateDataAnnotations, the - // Llm:MaxCopyChars precedent); this validator adds the F53.1 settings-API-only ceiling. Floor of - // 5s guards a degenerate near-zero target from rejecting every exchange outright; 120s (2 - // minutes) is comfortably past the spec'd 25s default while still bounding a fat-finger entry. + // Crosstalk:DurationTargetSeconds (SPEC F127.4, STORY-326, PLAN T282; amended PLAN T333 to the + // ratified 50s default) — CrosstalkOptions' own [Range(1, int.MaxValue)] (boot-enforced via + // ValidateDataAnnotations, the Llm:MaxCopyChars precedent); this validator adds the F53.1 + // settings-API-only ceiling. Floor of 5s guards a degenerate near-zero target from rejecting + // every exchange outright; 120s (2 minutes) is comfortably past the ratified 50s default while + // still bounding a fat-finger entry. internal const int CrosstalkDurationTargetSecondsMin = 5; internal const int CrosstalkDurationTargetSecondsMax = 120; diff --git a/src/GenWave.Host/Configuration/StationSettingsAllowlist.cs b/src/GenWave.Host/Configuration/StationSettingsAllowlist.cs index d4288798..c2c9f72b 100644 --- a/src/GenWave.Host/Configuration/StationSettingsAllowlist.cs +++ b/src/GenWave.Host/Configuration/StationSettingsAllowlist.cs @@ -463,8 +463,9 @@ public static IReadOnlyList IconPackChoices(IReadOnlyList // Crosstalk two-voice banter, the duration-fit knob (SPEC F127.4, STORY-326, PLAN T282) — // CrosstalkScriptWriter (GenWave.Tts) reads this fresh via IOptionsMonitor // on every generation attempt, so a PUT here reaches the very next attempt with no api - // restart. Defaults to the spec'd 25s; an estimate over target discards the WHOLE exchange - // rather than trimming a line (F127.4 — a cut dialogue line breaks the reaction to it). + // restart. Defaults to the ratified 50s (PLAN T333 amendment); an estimate over target + // discards the WHOLE exchange rather than trimming a line (F127.4 — a cut dialogue line + // breaks the reaction to it). new("Crosstalk:DurationTargetSeconds", SettingApplyMode.Live, SettingKind.Number, "seconds"), // Crosstalk scope/cadence (SPEC F127.8, STORY-328, PLAN T285) — Shows is a JSON array of // enabled show SLUGS, never display names (T175's "names slugs, not labels" rule — a rename diff --git a/src/GenWave.Host/Crosstalk/CrosstalkStockWorker.cs b/src/GenWave.Host/Crosstalk/CrosstalkStockWorker.cs index 878269de..9b8d8358 100644 --- a/src/GenWave.Host/Crosstalk/CrosstalkStockWorker.cs +++ b/src/GenWave.Host/Crosstalk/CrosstalkStockWorker.cs @@ -94,7 +94,10 @@ public sealed class CrosstalkStockWorker( IOptionsMonitor ttsOptions, OnAirRenderGate onAirRenderGate, ILogger log, - TimeProvider timeProvider) : BackgroundService + TimeProvider timeProvider, + IOptionsMonitor llmOptions, + LlmCallRecorder recorder, + IDegradationModeReader degradationMode) : BackgroundService { /// Outer tick cadence — deliberately much coarser than the 3s feeder tick this is /// opportunistic, off-clock work (SPEC F127.7): frequent enough that a freshly-opened stock slot @@ -253,6 +256,12 @@ internal async Task TickOnceAsync(CancellationToken ct) var outcome = await GenerateAndAssembleAsync(attempt, cast, ct); RecordPacingOutcome(outcome, attemptStartedAt); + // SPEC F139.1 (STORY-353, PLAN T330): the ONE place a break-window abandon becomes an + // LlmCallRecorder entry — see RecordWindowCancellation's own remarks for why the stamp + // has to happen HERE, not inside CrosstalkScriptWriter/CrosstalkAssembler themselves. + if (outcome.CancelledByBreakWindow) + RecordWindowCancellation(cast, attemptStartedAt); + if (outcome.Assembled is not { } assembled) { // PLAN T286 review F4: only a genuine discard costs the show a cooldown — a break @@ -333,6 +342,42 @@ void RecordPacingOutcome(GenerationOutcome outcome, DateTimeOffset attemptStarte pacing.RecordCompleted(elapsed); } + /// + /// SPEC F139.1 (STORY-353, PLAN T330) — the ONE place a break-window abandon becomes an + /// entry (). Neither + /// nor can record this + /// themselves: their own catches see only "the caller's + /// own ct fired", which is IDENTICAL whether that ct came from a break window opening + /// ( cancelling workCts) or from a genuine host shutdown + /// (stoppingToken, linked into that same workCts) — see + /// 's own cancellation-handling remarks. Only + /// 's own catch, which watches workCts/stoppingToken + /// directly, can honestly tell the two apart — SPEC F139's own "reuse the signal, don't re-derive + /// it" — so the stamp happens HERE, once already has the answer + /// (), not inside either Tts-layer writer. + /// + /// + /// // + /// stay — the abandoned attempt's own + /// prompt lived inside 's now-unwound stack frame and was never + /// captured up here, the same "faulted before this method had it in hand" case + /// 's own remarks already document for other faults. + /// mirrors 's own + /// "{Host} / {Neighbor}" shape, built from the SAME this tick already cast. + /// + /// + void RecordWindowCancellation(CrosstalkCastResult cast, DateTimeOffset attemptStartedAt) + { + var personaName = $"{cast.HostCard.Name} / {cast.NeighborCard.Name}"; + var model = llmOptions.CurrentValue.Model; + var elapsedMs = (long)(timeProvider.GetUtcNow() - attemptStartedAt).TotalMilliseconds; + + recorder.Record( + personaName, promptSystem: null, promptUser: null, response: null, attemptStartedAt, elapsedMs, + LlmCallOutcome.Failed, statusDetail: "a break window opened mid-flight; generation abandoned", + degradationMode.CurrentMode, LlmCallCause.CanceledByWindow, model, LlmCallKind.Crosstalk); + } + /// /// Runs then /// under a worker-owned diff --git a/src/GenWave.Host/appsettings.json b/src/GenWave.Host/appsettings.json index f68666cc..e5bd5432 100644 --- a/src/GenWave.Host/appsettings.json +++ b/src/GenWave.Host/appsettings.json @@ -51,7 +51,7 @@ "BlurbRetentionHours": 24 }, "Crosstalk": { - "DurationTargetSeconds": 25, + "DurationTargetSeconds": 50, "EveryNthAiring": 1 }, "DependencyHealth": { diff --git a/src/GenWave.Tts/ClaimCheckResult.cs b/src/GenWave.Tts/ClaimCheckResult.cs new file mode 100644 index 00000000..1baae5ac --- /dev/null +++ b/src/GenWave.Tts/ClaimCheckResult.cs @@ -0,0 +1,25 @@ +namespace GenWave.Tts; + +/// +/// The verdict / hand back +/// (SPEC F138.1): zero or more s, never a bare pass/fail bool alone — the +/// F138.4 ladder's re-ask prompt needs to NAME each violation, not just know one exists. +/// is a computed convenience (Violations.Count == 0), never an +/// independently-settable field a caller could desync from the list it describes. +/// +/// +/// Record equality note (no consumer relies on this today, T329 review round 3): the compiler- +/// generated Equals/GetHashCode this record derives from its positional +/// parameter compare that property by REFERENCE, not by sequence content — +/// has no structural equality of its own, so two results built from +/// separately-allocated-but-identical violation lists are NOT Equals-equal. If a future +/// consumer (T331/T332's ladder, a test) ever needs "same violations" comparison, it needs its own +/// sequence comparison (e.g. ), +/// not this record's own ==. +/// +/// +public sealed record ClaimCheckResult(IReadOnlyList Violations) +{ + /// True when is empty — the copy airs as written. + public bool Passed => Violations.Count == 0; +} diff --git a/src/GenWave.Tts/ClaimClass.cs b/src/GenWave.Tts/ClaimClass.cs new file mode 100644 index 00000000..280bb491 --- /dev/null +++ b/src/GenWave.Tts/ClaimClass.cs @@ -0,0 +1,26 @@ +namespace GenWave.Tts; + +/// +/// The claim classes can report a violation for (SPEC F138.1, F138.3): the +/// three F138.1 extracted-claim classes (, , +/// ) that checks against a segment's +/// fact block, plus — a clock-only fourth class (SPEC F138.3) that +/// alone produces. Daypart is not one of F138.1's three extracted +/// claim classes and never appears in a result; +/// is the one class shared by both entry points (a fact block can support/deny a weekday exactly like +/// a condition word, and the clock line can also confirm/deny one). +/// +public enum ClaimClass +{ + /// A run of digits (SPEC F138.1) — e.g. "21" or "108.8". only. + DigitRun, + + /// A weekday name (SPEC F138.1, F138.3) — e.g. "Saturday". + Weekday, + + /// A weather-condition word (SPEC F138.1) — e.g. "sunshine". only. + ConditionWord, + + /// A daypart word (SPEC F138.3) — e.g. "tonight". only. + Daypart, +} diff --git a/src/GenWave.Tts/ClaimViolation.cs b/src/GenWave.Tts/ClaimViolation.cs new file mode 100644 index 00000000..d0c48e45 --- /dev/null +++ b/src/GenWave.Tts/ClaimViolation.cs @@ -0,0 +1,44 @@ +namespace GenWave.Tts; + +using System.Diagnostics.CodeAnalysis; + +/// +/// One claim in candidate copy that could not support (SPEC F138.1-F138.3): +/// names which of the four subjects tripped, and +/// is the exact substring the checker matched in the COPY (original casing +/// preserved) — raw data for a future re-ask prompt (SPEC F138.4), not display text of its own; PLAN +/// T329's design constraint leaves rendering that prompt line to T331, deliberately. +/// +/// is set ONLY by , on a clock mismatch — +/// the correct weekday name ('s own ToString() spelling, e.g. +/// "Sunday"), or the correct daypart category word (e.g. "morning") — so a re-ask prompt can name both +/// the mistake and the fix in one line. never sets it: there is no +/// single "correct" fix for an unsupported fact claim, only "the fact block never said this", so it is +/// null there by construction, not by omission. +/// +/// +/// Safe to interpolate (T329 review round 1 finding): is provably +/// closed-vocabulary-or-digit-shaped — it can only ever be a digit run, one of +/// 's seven names, one of +/// 's words, or one of +/// 's words — so a future re-ask prompt (SPEC +/// F138.4, PLAN T331) may interpolate it directly into prompt text without fence-forging risk; rely on +/// it knowingly rather than re-deriving the guarantee at the call site. +/// +/// +public sealed record ClaimViolation(ClaimClass Class, string Token, string? Expected = null) +{ + /// + /// True when this violation carries a correct-value fix ( set) — i.e. it + /// came from , never (see + /// this record's own remarks above on which checker sets and why). PLAN + /// T332 review round-2 finding: LlmCopyWriter.DescribeViolationForLog and + /// LlmPromptBuilder.DescribeViolationForReask both key their own clock-vs-facts split on + /// this ONE property now, rather than each independently re-deriving "is this a clock violation" + /// from 's own nullability at two separate call sites across a module + /// boundary — a duplication that could silently drift the day either one grew a different (wrong) + /// test. + /// + [MemberNotNullWhen(true, nameof(Expected))] + public bool IsClockClaim => Expected is not null; +} diff --git a/src/GenWave.Tts/ClaimVocabulary.cs b/src/GenWave.Tts/ClaimVocabulary.cs new file mode 100644 index 00000000..18d5ff66 --- /dev/null +++ b/src/GenWave.Tts/ClaimVocabulary.cs @@ -0,0 +1,124 @@ +namespace GenWave.Tts; + +/// +/// The versioned, curated vocabularies and +/// match against (SPEC F138.1, F138.3, F138.6) — plain data, no I/O, no settings: the same purity +/// posture as the checkers themselves. Curated, +/// not exhaustive by design (see 's own false-positive-posture remarks): a +/// real weekday/condition/daypart/month word missing from one of these lists is simply never extracted +/// as a claim at all, which is the SAFE gap to have — it can neither wrongly pass nor wrongly fail, it +/// is just invisible to the checker, the same as any word outside these four subjects already is. +/// +/// +/// Each list below is exposed as a single pipe-delimited internal const string "vN" alternation +/// — the one canonical source of truth 's and 's +/// own [GeneratedRegex] extraction patterns interpolate directly (a compile-time-constant +/// expression), so a matching regression can +/// never drift from the vocabulary that produced it. There is no parallel IReadOnlyList<string> +/// view of any of these anymore (T329 review round 2): nothing outside the regex patterns themselves +/// ever needed one, and a second, unread copy is exactly the kind of drift risk this file exists to +/// avoid. Bump the "vN" marker in a list's own remarks whenever an entry is added or removed, so a +/// matching regression can be traced to a vocabulary change rather than a logic change. +/// +/// +static class ClaimVocabulary +{ + /// + /// v1 (PLAN T329, 2026-08-20) — the seven Gregorian weekday names, English only, lowercase + /// (matching is always case-insensitive; every consumer applies + /// or an explicit ordinal-ignore-case comparison, never relying on this casing itself). + /// + internal const string WeekdayAlternation = "sunday|monday|tuesday|wednesday|thursday|friday|saturday"; + + /// + /// v2 (T329 review round 2, 2026-08-20) — dropped clear from v1's set. Its dominant sense in + /// DJ prose is not weather at all ("let me be clear", "clear away the cobwebs", "make it clear") — + /// far more common on air than "clear skies" — so it was producing condition-word false claims on + /// ordinary, weather-free copy. The remaining words below are kept even though each has its own + /// well-known non-weather sense too (a metaphorical "storm of great music", the movie "Purple + /// Rain", the band "Snow Patrol") — that risk is accepted (the false-positive posture: a spurious + /// re-ask costs one retry, never silence) because those words are still weather-dominant in typical + /// DJ copy, unlike "clear"; a future vocabulary edit should read this remark before adding another + /// word back rather than re-litigating the same call from scratch. + /// + internal const string ConditionWordAlternation = + "sunny|sunshine|rain|rainy|rainfall|overcast|cloudy|snow|snowy|storm|stormy|" + + "thunderstorm|thunderstorms|foggy|fog|windy|drizzle|drizzly|hail|sleet|humid|mist|misty"; + + /// + /// v1 (PLAN T329, 2026-08-20) — the daypart word set (SPEC F138.3's own stated minimum: + /// morning/afternoon/evening/tonight/night). "tonight" and "night" name the SAME daypart CATEGORY + /// (see ) — gh-#438's own exhibit used "Tonight", and a listener hears the + /// two as interchangeable. + /// + internal const string DaypartWordAlternation = "morning|afternoon|evening|tonight|night"; + + /// + /// v1 (PLAN T333 review advisory A2, 2026-08-20) — the twelve Gregorian month names, English + /// only, lowercase (matching is always case-insensitive). Moved here from a -local + /// const at review: this class's own "no second list" discipline (see this file's own class + /// remarks) applies to the F138.6 digit-date shape exactly as it does to the F138.1/F138.3 three + /// — a month word missing from this list is simply never extracted as a date claim, the same + /// safe gap every other vocabulary here leaves. 's own + /// DateClaimRx is the only consumer today — a BARE month word is never itself a claim there (see + /// that regex's own remarks for why only month-PLUS-day is extracted). + /// + internal const string MonthAlternation = + "january|february|march|april|may|june|july|august|september|october|november|december"; + + /// + /// Canonical daypart CATEGORY for a daypart word (SPEC F138.3): identity for every word except + /// "tonight", which folds onto "night" (see 's own remarks). + /// compares/dedupes by category, never raw words, so "tonight" + /// and "night" are always treated as the one claim. + /// + public static string CategoryOf(string daypartWord) => + string.Equals(daypartWord, "tonight", StringComparison.OrdinalIgnoreCase) + ? "night" + : daypartWord.ToLowerInvariant(); + + /// + /// True when daypart CATEGORY 's own window (SPEC F138.3, amended T329 + /// review round 1 — a broadcast-dayparting convention, not a physical law) includes station-local + /// hour (24-hour clock, 0-23 — the same range + /// already returns). Windows OVERLAP rather than + /// partition — a claim passes if the clock hour falls in ANY window the claimed word's own + /// category names, so "Good evening" and "Good night" are both true statements at 21:00, and + /// neither one is a lie: + /// + /// Morning: 05:00-11:59 + /// Afternoon: 12:00-17:59 + /// Evening: 17:00-22:59 + /// Night: 21:00-04:59 (wraps past midnight — the small hours read as "night", not "morning") + /// + /// calls this once per claimed word's own category — never a + /// single "the" category for an hour; see for that single-category + /// mapping, kept ONLY to fill on a genuine mismatch. + /// + public static bool HourIsInCategory(string category, int hour) => category switch + { + "morning" => hour is >= 5 and <= 11, + "afternoon" => hour is >= 12 and <= 17, + "evening" => hour is >= 17 and <= 22, + "night" => hour is >= 21 and <= 23 or >= 0 and <= 4, + _ => throw new ArgumentOutOfRangeException(nameof(category), category, "Unknown daypart category."), + }; + + /// + /// The single canonical daypart CATEGORY for a station-local hour (SPEC F138.3, 24-hour clock, + /// 0-23) — a non-overlapping PARTITION, unlike 's overlapping + /// windows above. uses this ONLY to fill + /// on a genuine mismatch (a re-ask prompt needs exactly one + /// "the correct answer is X" to name, not a set); it never drives the pass/fail decision itself. + /// Boundaries here match 's own windows' non-overlapping halves. + /// + public static string CategoryForHour(int hour) => hour switch + { + >= 5 and <= 11 => "morning", + >= 12 and <= 16 => "afternoon", + >= 17 and <= 20 => "evening", + >= 21 and <= 23 => "night", + >= 0 and <= 4 => "night", + _ => throw new ArgumentOutOfRangeException(nameof(hour), hour, "Station-local hour must be 0-23."), + }; +} diff --git a/src/GenWave.Tts/CopyClaims.cs b/src/GenWave.Tts/CopyClaims.cs new file mode 100644 index 00000000..41795ad1 --- /dev/null +++ b/src/GenWave.Tts/CopyClaims.cs @@ -0,0 +1,366 @@ +using System.Text.RegularExpressions; + +namespace GenWave.Tts; + +/// +/// The mechanical claim checker (SPEC F138.1-F138.3, gh-#434, gh-#438): pure and static, no I/O, no +/// settings reads, zero non-BCL dependencies beyond 's own plain data — +/// the exact purity posture (F68.6), named by F138.1 itself. Two entry points: +/// (F138.2, the context lane's "did the model invent a fact") and +/// (F138.3, every patter kind's "did the model lie about the clock"). Both are +/// pure functions of their arguments — the re-ask/template ladder (F138.4), the prompt guard line +/// (F138.5), and every other stateful or config-reading decision live at the call sites this checker is +/// built for (PLAN T331/T332), never here. +/// +/// +/// False-positive posture (governs every ambiguous decision below): when a match is uncertain, +/// this checker PASSES rather than rejects. A missed fabrication only ever airs a line no worse than +/// the pre-F138 status quo; a false rejection spends the F138.4 ladder's one re-ask on copy that was +/// actually fine, and can needlessly push good copy onto the deterministic template fallback. Every +/// heuristic here — whole-token digit support, word-boundary condition matching, present-frame-only +/// weekday/daypart extraction, title-substring exemption, "missing from the vocabulary" simply never +/// becoming a claim at all — is chosen with this bias, not tightened further even where a smarter rule +/// is possible. +/// +/// +/// +/// Input assumption — what looks like when it reaches this checker: the +/// intended caller (PLAN T331/T332) hands this 's POST-hygiene, +/// PRE- text — ApplyCopyHygiene's output, or a CleanCopy sentence +/// salvage of it — never the raw model reply, and never the post- +/// speech form. Two matching choices below depend on this: case survives (so this checker matches +/// case-insensitively itself rather than trusting 's later lowercase flatten to +/// have already happened), and punctuation/unit symbols are still literal (no F68 unit-expansion has +/// run yet, so a temperature still reads "21°C" rather than "21 degrees Celsius" — harmless either way, +/// since only ever needs the leading digits). +/// +/// +public static partial class CopyClaims +{ + /// + /// SPEC F138.2 — every digit run, weekday name (present-frame only — see below), and weather + /// condition word claims must be supported by + /// — the segment's own RAW facts text (the same string the caller passes as + /// LlmPromptBuilder.BuildContextFactsLine's own facts parameter, BEFORE that method + /// fences it with its "Use only these facts. Do not add facts." framing — + /// is never the already-fenced prompt line itself): a digit run must appear as a WHOLE TOKEN (see + /// below); a weekday or condition word must appear as a WHOLE WORD, case-insensitively (see + /// for the exact vocabularies this draws from). Daypart words are not + /// an F138.1 claim class and are never extracted here — they only ever matter against the clock + /// (see ). + /// + /// + /// Digit-run tokenization and support (amended T329 review round 1, the F135.5 precedent): a + /// digit run is a maximal [0-9]+(?:\.[0-9]+)? match — contiguous digits, plus one optional + /// embedded decimal point (so "108.8" is ONE token, never split into "108" and "8"). A copy token + /// is supported iff it is EQUAL to one of the fact block's own digit-run tokens, OR some fact-block + /// token starts with the copy token followed by a literal "." (the deliberate decimal-prefix + /// allowance kept explicitly: "108" is supported by a fact block carrying "108.8"). This replaces + /// the original literal-substring reading, which made every short number unfalsifiable the moment a + /// fact block carried a date or timestamp — "1" would have been "supported" by any fact block + /// containing "14:37" purely because "1" is a substring of "14", never mind that "1" never appears + /// as its OWN token anywhere. A hyphenated range ("12-15") tokenizes into its two ENDPOINTS ("12", + /// "15") for free, since a hyphen is not a digit — so a range's own printed endpoints support + /// themselves, but a value strictly BETWEEN them (e.g. copy claims "13" against a fact reading + /// "12-15") is not equal to either endpoint and is reported as unsupported — a known, accepted + /// conservative gap (full numeric-range interpolation was judged unjustified complexity for a + /// fixed, narrow claim surface; see the false-positive-posture remarks above for why this is an + /// acceptable place to lean the other way, toward flagging, given the re-ask cost is bounded to one + /// retry). A second, symmetric, still-accepted gap: whole-token equality means a ROUNDED claim + /// against a precise fact is flagged even when a person would call it correct — "21" against a fact + /// block reading "20.6" is not equal to "20.6" and is not a "20.6"-style decimal PREFIX of it + /// either, so it violates, even though "21" is simply "20.6" rounded. Left as-is rather than adding + /// numeric-rounding logic: the false-positive posture accepts an occasional needless re-ask here + /// over adding a second, fuzzier definition of "supported" alongside the decimal-prefix rule. A + /// third, related gap: a LEADING ZERO is never stripped — "8" against a fact block reading + /// "2026-08-08" violates, because the fact block's own digit-run token there is "08", and the + /// equality check above treats "8" and "08" as different + /// tokens (an ordinal-string, not a numeric-value, comparison). Deliberate and conservative, the + /// same posture as the other two gaps above: a date component spelled with its own leading zero + /// is common in fact blocks (ISO dates) but rare in spoken copy ("the 8th", not "the 08th"), so + /// this gap almost never fires on real copy, and when it does, flagging costs one re-ask rather + /// than risking a false pass. + /// + /// + /// + /// No track-title exemption here (deliberate, unlike ): this method + /// takes no trackTitle parameter, because no track-bearing copy reaches + /// today — a request's own Track is always null (see + /// LlmPromptBuilder.BuildUserContent's own T224 remarks and Orchestrator.BuildContextSegmentRequestAsync's + /// literal at that position), and + /// carries no track either. Trigger: the day any future hands + /// track-bearing copy through this same fact-checking path, this exemption gap needs revisiting — + /// a title like "Purple Rain" or "Snow Patrol" would otherwise trip the condition-word class the + /// same way it can already trip 's weekday/daypart classes without the + /// exemption carries. + /// + /// + public static ClaimCheckResult CheckFacts(string copy, string factBlock) + { + ArgumentNullException.ThrowIfNull(copy); + ArgumentNullException.ThrowIfNull(factBlock); + + var violations = new List(); + var factDigitTokens = DigitRunRx().Matches(factBlock).Select(match => match.Value).ToArray(); + + foreach (var token in DistinctTokens(DigitRunRx().Matches(copy))) + { + var supported = factDigitTokens.Any(fact => + string.Equals(fact, token, StringComparison.Ordinal) || + fact.StartsWith(token + ".", StringComparison.Ordinal)); + if (!supported) + violations.Add(new ClaimViolation(ClaimClass.DigitRun, token)); + } + + foreach (var (token, _, _) in DistinctPresentFrameWeekdays(copy)) + { + if (!ContainsWord(factBlock, token)) + violations.Add(new ClaimViolation(ClaimClass.Weekday, token)); + } + + foreach (var token in DistinctTokens(ConditionWordRx().Matches(copy))) + { + if (!ContainsWord(factBlock, token)) + violations.Add(new ClaimViolation(ClaimClass.ConditionWord, token)); + } + + return new ClaimCheckResult(violations); + } + + /// + /// SPEC F138.3 — every weekday/daypart claim in must match the clock this + /// break was actually written against: , the SAME instant + /// LlmPromptBuilder.BuildStationClockLine renders into the prompt (amended T329 review round + /// 1 — one parameter, not a separately-computed weekday/hour pair, so + /// prompt and check provably read the same instant and 's + /// hour-range validation is unreachable from here: is always + /// 0-23 by construction). A named weekday that doesn't match 's + /// own , or a daypart word whose category (see + /// ) has no window covering 's + /// own (see ), is a + /// violation carrying the correct value in . Applies uniformly + /// to every LLM patter kind (F138.3's own "all patter kinds" — this method does not know or care + /// which kind called it; that gate lives at the T332 call site). + /// + /// + /// Present-frame-only extraction (amended T329 review round 1 — read literally, the original + /// letter rejected the bread of DJ patter: anticipation "join us next Friday", recall "last + /// Saturday's show", "coming up tonight" said from a morning hour): a weekday or daypart word is + /// a CLOCK CLAIM at all only when it is asserted as the present frame, under a small closed set of + /// markers immediately preceding it. Weekdays: "this {weekday}", "today is {weekday}", "it is + /// {weekday}"/"it's {weekday}", "happy {weekday}" — and, structurally for free (nothing here anchors + /// on what follows the weekday), "{weekday} {daypart}" whenever the weekday itself is one of those + /// four, e.g. "this Saturday morning". Dayparts: greeting/copula only — "good {daypart}", "it is + /// {daypart}"/"it's {daypart}" — deliberately NOT "this {daypart}" ("we opened this morning" said at + /// night is recall, not a lie about tonight). Daypart windows OVERLAP rather than partition (see + /// ): a word passes if the hour falls in ANY window its + /// own category names — "Good evening" at 21:00 is not a lie just because 21:00 is also "night". + /// Everything else — last/next/every/a/on-a/tomorrow/yesterday {weekday}, a possessive + /// {weekday}'s, a plural {weekday}s, or a bare mention with no marker at all — is displaced or + /// generic reference, never extracted as a claim (the false-positive posture: when in doubt, pass). + /// Both gh-#438 aired exhibits still violate under this narrowed rule. + /// + /// + /// + /// Track-title exemption (F138.3): excludes any weekday/daypart + /// claim whose OWN matched word falls entirely inside a literal (case-insensitive) occurrence of the + /// title text somewhere in — "Saturday Night Fever" mentioned by name never + /// trips the gate, including a present-frame-marked mention like "it's Saturday Night Fever". + /// Documented limit: only an EXACT, literal (whole, contiguous) mention of the title text is exempt; + /// a paraphrase or a partial quote of it gets no exemption, because a checker this simple has no way + /// to distinguish a title reference from a genuine claim other than the title's own literal text + /// appearing verbatim. + /// + /// + /// + /// Daypart violations dedupe by CATEGORY, not raw word (review finding): "tonight" and + /// "night" are the SAME claim (), so a line naming both + /// reports at most one daypart violation, keyed on the first-seen spelling — never two violations + /// for what is, to a listener, one lie about the same instant. + /// + /// + public static ClaimCheckResult CheckClock(string copy, DateTimeOffset stationLocalNow, string? trackTitle = null) + { + ArgumentNullException.ThrowIfNull(copy); + + var titleSpans = FindTitleSpans(copy, trackTitle); + var violations = new List(); + var expectedWeekday = stationLocalNow.DayOfWeek.ToString(); + var clockHour = stationLocalNow.Hour; + + foreach (var (token, index, length) in DistinctPresentFrameWeekdays(copy)) + { + if (IsExempt(index, length, titleSpans)) + continue; + + if (!string.Equals(token, expectedWeekday, StringComparison.OrdinalIgnoreCase)) + violations.Add(new ClaimViolation(ClaimClass.Weekday, token, expectedWeekday)); + } + + foreach (var (token, _, category) in DistinctPresentFrameDayparts(copy, titleSpans)) + { + if (!ClaimVocabulary.HourIsInCategory(category, clockHour)) + violations.Add(new ClaimViolation(ClaimClass.Daypart, token, ClaimVocabulary.CategoryForHour(clockHour))); + } + + return new ClaimCheckResult(violations); + } + + /// + /// Every distinct (case-insensitive, first-occurrence-casing-wins) present-frame weekday claim in + /// — see 's own remarks for the exact marker set — + /// as (token, character index, character length) triples, the span covering only the WEEKDAY word + /// itself, never its marker, so a title-exemption or dedup check tests the claim's own span. + /// + static IEnumerable<(string Token, int Index, int Length)> DistinctPresentFrameWeekdays(string copy) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in PresentFrameWeekdayRx().Matches(copy)) + { + var group = match.Groups["weekday"]; + if (seen.Add(group.Value)) + yield return (group.Value, group.Index, group.Length); + } + } + + /// + /// Every distinct present-frame daypart claim in — see + /// 's own remarks for the exact marker set — as (token, character index, + /// daypart category) triples, deduped by CATEGORY (not raw word, so "tonight" and "night" collapse + /// into the one claim) and excluding any match the track-title exemption covers. + /// + static IEnumerable<(string Token, int Index, string Category)> DistinctPresentFrameDayparts( + string copy, List<(int Start, int End)> titleSpans) + { + var seenCategories = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in PresentFrameDaypartRx().Matches(copy)) + { + var group = match.Groups["daypart"]; + if (IsExempt(group.Index, group.Length, titleSpans)) + continue; + + var category = ClaimVocabulary.CategoryOf(group.Value); + if (seenCategories.Add(category)) + yield return (group.Value, group.Index, category); + } + } + + /// + /// Every LITERAL (case-insensitive) occurrence of inside + /// , as start/end character spans — the exemption zones + /// checks a match against. Empty when is null, + /// blank, or never mentioned verbatim. + /// + static List<(int Start, int End)> FindTitleSpans(string copy, string? trackTitle) + { + var spans = new List<(int Start, int End)>(); + if (string.IsNullOrEmpty(trackTitle)) + return spans; + + var searchFrom = 0; + while (true) + { + var found = copy.IndexOf(trackTitle, searchFrom, StringComparison.OrdinalIgnoreCase); + if (found < 0) + break; + + spans.Add((found, found + trackTitle.Length)); + searchFrom = found + trackTitle.Length; // advance past the whole match — a title mention + // cannot meaningfully overlap itself, and this + // avoids re-scanning inside a match already found + } + + return spans; + } + + /// True when a claim span [, + ) falls entirely inside one of — the F138.3 track-title exemption. + static bool IsExempt(int index, int length, List<(int Start, int End)> titleSpans) => + titleSpans.Exists(span => index >= span.Start && index + length <= span.End); + + /// + /// The distinct (case-insensitive, first-occurrence-casing-wins) matched values of + /// , in the order first seen — collapses repeated mentions of the same + /// claim ("sunny... sunny again") into a single reported violation rather than one per occurrence. + /// + static IEnumerable DistinctTokens(IEnumerable matches) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var match in matches) + { + if (seen.Add(match.Value)) + yield return match.Value; + } + } + + /// + /// Whole-word (letter/digit-boundary), case-insensitive search for inside + /// — the F138.2 "by token" support rule for weekday/condition claims. + /// A manual boundary check rather than a dynamically-built : + /// varies per call (it is the extracted claim, not a fixed pattern), so it cannot be a + /// [GeneratedRegex] — those require a compile-time-constant pattern (see this file's own + /// // + /// for the fixed patterns that CAN be, and are). + /// + static bool ContainsWord(string haystack, string word) + { + var searchFrom = 0; + while (true) + { + var found = haystack.IndexOf(word, searchFrom, StringComparison.OrdinalIgnoreCase); + if (found < 0) + return false; + + var before = found == 0 || !char.IsLetterOrDigit(haystack[found - 1]); + var afterIndex = found + word.Length; + var after = afterIndex == haystack.Length || !char.IsLetterOrDigit(haystack[afterIndex]); + if (before && after) + return true; + + searchFrom = found + 1; + } + } + + // Maximal digit run, with at most one embedded decimal point (see CheckFacts's own remarks for + // the documented tokenization rule this implements — decimals stay one token, ranges tokenize + // into their two endpoints for free). [0-9] rather than \d (review finding): explicit ASCII digit + // class, not the Unicode-digit-aware shorthand — station facts/copy are always ASCII numerals. + [GeneratedRegex(@"[0-9]+(?:\.[0-9]+)?")] + private static partial Regex DigitRunRx(); + + // Present-frame weekday marker (SPEC F138.3, amended T329 review round 1) — see CheckClock's own + // remarks for the exact five-shape marker set this implements; the weekday itself is captured + // separately from its marker (group "weekday") so a title-exemption/dedup check can test the + // claim's own span. Interpolates ClaimVocabulary's own const alternation directly — a + // compile-time-constant expression, so this stays source-generated. + // + // it[’']s (review round 3 fix): BOTH apostrophe forms — straight U+0027 and curly U+2019 + // (RIGHT SINGLE QUOTATION MARK) — mark "it's", never only the ASCII one. SpeechText's own + // curly->straight fold (SpeechText.cs, ApplyCopyHygiene->Normalize) runs AFTER this checker by + // design (this class's own remarks: the checker sees POST-hygiene, PRE-Normalize text), and + // LlmCopyWriter already treats U+2019 as an apostrophe in three other places (SentenceBoundaryPattern, + // the "here's" probe, IsApostrophe), so a model emitting "It’s Saturday" in exactly this + // window is the expected case, not an edge one. The pattern below spells the curly form as + // the \u2019 regex escape, not the raw glyph, to keep this source file itself ASCII — the next + // edit here must keep BOTH forms, not silently narrow back to one. + [GeneratedRegex( + $@"\b(?:this|happy|today\s+is|it\s+is|it[\u2019']s)\s+(?{ClaimVocabulary.WeekdayAlternation})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex PresentFrameWeekdayRx(); + + // Present-frame daypart marker (SPEC F138.3, amended) — greeting/copula only; deliberately NOT + // "this {daypart}" (see CheckClock's own remarks for why). Captures the daypart word alone (group + // "daypart"), same reasoning as the weekday marker above — including the same it[’']s + // both-apostrophe-forms fix (see PresentFrameWeekdayRx's own remarks for why it must stay both). + [GeneratedRegex( + $@"\b(?:good|it\s+is|it[\u2019']s)\s+(?{ClaimVocabulary.DaypartWordAlternation})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex PresentFrameDaypartRx(); + + // Internal, not private (PLAN T333 review advisory A1): CrosstalkScriptParser's own F138.6 + // weather-condition check reuses this SAME compiled pattern rather than keeping a byte-identical + // copy that could silently drift the day either one changes — the one-canonical-source discipline + // ClaimVocabulary.ConditionWordAlternation already establishes one level up, extended to the + // compiled regex built from it. + [GeneratedRegex($@"\b(?:{ClaimVocabulary.ConditionWordAlternation})\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + internal static partial Regex ConditionWordRx(); +} diff --git a/src/GenWave.Tts/CrosstalkOptions.cs b/src/GenWave.Tts/CrosstalkOptions.cs index 303cf88a..f5636bba 100644 --- a/src/GenWave.Tts/CrosstalkOptions.cs +++ b/src/GenWave.Tts/CrosstalkOptions.cs @@ -18,14 +18,19 @@ public sealed class CrosstalkOptions /// /// The spoken-duration target a validated must fit under (SPEC /// F127.4) — an estimate over this rejects the WHOLE exchange (never a trim; see - /// 's own remarks). Defaults to the spec'd 25 seconds. Live via - /// , read fresh by - /// on every generation attempt (mirrors every other - /// live-adjustable leaf this project's options classes carry), so an operator PUT reaches the - /// very next attempt with no api restart. + /// 's own remarks). + /// ⚠️ Amended (Dean, 2026-08-20, PLAN T333): defaults to 50 seconds, ratified from + /// two days of live convergence after the 2026-08-17 live bump (mid-show airs both days; the + /// 42-45s class survives; only 53-54s overshoots die) — the shipped default now matches that + /// ratified target directly, so the demo's own live override retires (config on the demo box, + /// never repo code). The original 25s was the T282 paper-audition posture (12.5% accept — punchy + /// but starving). Live via , + /// read fresh by on every generation attempt (mirrors every + /// other live-adjustable leaf this project's options classes carry), so an operator PUT reaches + /// the very next attempt with no api restart. /// [Range(1, int.MaxValue)] - public int DurationTargetSeconds { get; set; } = 25; + public int DurationTargetSeconds { get; set; } = 50; /// /// Raw JSON array of enabled show SLUGS (station.show.slug, db/35's own unique stable diff --git a/src/GenWave.Tts/CrosstalkPromptBuilder.cs b/src/GenWave.Tts/CrosstalkPromptBuilder.cs index de6245fe..3b368813 100644 --- a/src/GenWave.Tts/CrosstalkPromptBuilder.cs +++ b/src/GenWave.Tts/CrosstalkPromptBuilder.cs @@ -38,6 +38,26 @@ static class CrosstalkPromptBuilder /// const int MaxSoulChars = 4000; + /// + /// SPEC F138.6's narrow anti-fabrication clause — 's + /// sibling one project seam over, joining the banter scaffold beside its other style rules (never a + /// separate paragraph, exactly like 's own F138.5 + /// guard line one method over). Forbids exactly the shapes 's + /// own F138.6 mechanical checks look for (a real frequency, call sign, place name, weather + /// condition, or date) and explicitly ALLOWS the opposite — invented lore is good radio, not a + /// violation, so the prompt says so directly rather than leaving a model to guess whether "the + /// legendary DJ Ghost of the Graveyard Shift" is forbidden fabrication or welcome color. Real-place + /// invention is prompt-only by design (SPEC F138.6): a checker cannot tell a real city from an + /// invented one, so this clause is the ONLY place that half of the rule is ever enforced at all. + /// Comma-free (gh-#303 style lesson, 's own + /// precedent) — one short sentence per forbidden shape rather than a comma-joined list, so the + /// prompt asking the model to avoid fabricated specifics does not itself read like one. + /// + const string AntiFabricationClause = + "Never mention a real radio frequency. Never mention a real call sign. Never mention a real " + + "place name. Never mention a real weather condition. Never mention a real date. Invented " + + "recurring characters running gags and station mythology are welcome."; + /// /// The banter scaffold plus both speakers' persona sections (SPEC F127.3). Deliberately never /// mentions a track, a song, or "what's playing" — this writer's caller (a LATER task's @@ -51,8 +71,24 @@ static class CrosstalkPromptBuilder /// not a figure scaled off (the per-LINE budget, which has /// no fixed relationship to how many lines a script carries). /// + /// + /// + /// ⚠️ Amended (PLAN T333 review round 1, 2026-08-20 — probe-proven F2): + /// appends beside + /// — the SAME F138.5 guard line every other patter prompt already carries. Before this amendment, + /// crosstalk was the only patter kind whose F138.3 clock check ran with NO prompt-side rule at + /// all: the model was told the clock as a fact (the station clock line in the user content) but + /// never instructed not to contradict it, then silently discarded for a wrong-day claim with no + /// re-ask and no template to fall back on (F127.4) — pure F140-class generation waste on the + /// fenced ollama for a mistake the prompt itself never warned against. + /// is the SAME generation-time instant + /// threads into both this guard line and 's own clock + /// check () — one shared instant, never two + /// separately-computed ones. + /// /// - public static string BuildSystemPrompt(PersonaCard hostCard, PersonaCard neighborCard, int durationTargetSeconds) + public static string BuildSystemPrompt( + PersonaCard hostCard, PersonaCard neighborCard, int durationTargetSeconds, DateTimeOffset stationLocalNow) { // Same spoken-rate estimate CrosstalkScriptParser.Parse applies to an accepted script — the // STATED word budget asks for what the duration gate will accept, no headroom added (unlike @@ -72,6 +108,7 @@ public static string BuildSystemPrompt(PersonaCard hostCard, PersonaCard neighbo $"Across the WHOLE exchange use no more than approximately {wordBudget} words total. " + "Both DJs must speak at least once. Keep each line short and conversational, no commas, " + "no stage directions, no emoji, no markdown formatting. " + + AntiFabricationClause + " " + LlmPromptBuilder.BuildClockGuardLine(stationLocalNow) + " " + $"To have one speaker briefly cut in over the other's line, tag that one line " + $"\"{CrosstalkScriptParser.HostTag} {CrosstalkScriptParser.InterjectionMarker}: \" " + $"or \"{CrosstalkScriptParser.NeighborTag} {CrosstalkScriptParser.InterjectionMarker}: " + diff --git a/src/GenWave.Tts/CrosstalkScriptParser.cs b/src/GenWave.Tts/CrosstalkScriptParser.cs index 50653e23..60f9f3cc 100644 --- a/src/GenWave.Tts/CrosstalkScriptParser.cs +++ b/src/GenWave.Tts/CrosstalkScriptParser.cs @@ -1,15 +1,34 @@ namespace GenWave.Tts; +using System.Text.RegularExpressions; using GenWave.Core.Domain; /// /// Strict parse + validation for a completion reply (SPEC F127.3, -/// F127.4, STORY-326 AC2, AC3, AC4, AC6). Fail-closed by construction: the FIRST rule a reply breaks -/// is the one returned — no partial credit, no salvage, no template rung (F127.4's "the failure mode -/// is skip"). // are the -/// single source of truth for the wire format — states the exact -/// same three tokens in the instructions it builds, so the model is never asked to emit a shape this -/// parser doesn't also accept. +/// F127.4, F138.6, STORY-326 AC2, AC3, AC4, AC6, STORY-352). Fail-closed by construction: the FIRST +/// rule a reply breaks is the one returned — no partial credit, no salvage, no template rung (F127.4's +/// "the failure mode is skip"). // +/// are the single source of truth for the wire format — states the +/// exact same three tokens in the instructions it builds, so the model is never asked to emit a shape +/// this parser doesn't also accept. +/// +/// +/// The F138.6 truth discard reasons (PLAN T333): once a reply clears every SHAPE rule above (both +/// speakers present, alternation, per-line hygiene/budget), it must also clear four MECHANICAL truth +/// checks — 's own / +/// (frequency/call-sign shapes), (the SAME F138.1 vocabulary, +/// no second list), (digit-date shapes), and +/// (the T329 present-frame clock predicate, reused verbatim — see 's own +/// remarks for the shared generation-time instant). Every one of +/// these is a SHAPE a checker can verify mechanically. +/// Real-geography invention — a fabricated city, venue, or landmark — is deliberately NOT checked here: +/// F138.6 states plainly that a checker cannot tell a real place from an invented one, so that half of +/// the anti-fabrication rule lives ONLY in 's own prompt clause, never +/// pretended into a regex here. A truth-check failure stamps — +/// never (the shape was fine; the CONTENT is the problem) +/// and never a re-ask (F127.4 has none for crosstalk — a truth discard is silent, and the stock worker +/// tries again on its own cadence, exactly like every other discard this method returns). +/// /// /// /// Produces / directly (round-2 @@ -19,7 +38,7 @@ namespace GenWave.Tts; /// 's own remarks for why the shared enum lives in Abstractions. /// /// -static class CrosstalkScriptParser +static partial class CrosstalkScriptParser { /// The literal line prefix a HOST turn is tagged with. Deliberately a fixed ROLE token, /// never the persona's own display name — a card whose Name contains a colon, a space, or @@ -66,12 +85,19 @@ static class CrosstalkScriptParser /// /// Parses and fully validates one completion reply into a (SPEC - /// F127.3, F127.4). is the per-line char budget (the SAME + /// F127.3, F127.4, F138.6). is the per-line char budget (the SAME /// Llm:MaxCopyChars ceiling an ordinary blurb carries — no second setting); a line over it /// discards the WHOLE exchange, never a trim (F127.4). is - /// the live value. + /// the live value. + /// (PLAN T333) is the SAME generation-time instant + /// already threads into for this exact request + /// () — never a freshly-read clock, so the + /// F138.3 clock check below provably judges the script against the SAME clock the prompt stated, + /// the identical one-shared-instant discipline 's own remarks + /// require of every other patter kind. /// - public static CrosstalkWriteResult Parse(string rawResponse, int maxLineChars, int durationTargetSeconds) + public static CrosstalkWriteResult Parse( + string rawResponse, int maxLineChars, int durationTargetSeconds, DateTimeOffset stationLocalNow) { var rawLines = rawResponse .Split('\n') @@ -81,8 +107,11 @@ public static CrosstalkWriteResult Parse(string rawResponse, int maxLineChars, i if (rawLines.Count is < MinLines or > MaxLines) { + // SPEC F139.1 (T330 review round 1 amendment): a malformed-SHAPE reject — the reply came + // back with content, it just never fit the required line count (an over-MaxLines reply is + // the amendment's own exhibit: TOO MUCH content is not "empty" by any honest reading). return Discarded( - $"expected {MinLines}-{MaxLines} speaker-tagged lines, got {rawLines.Count}"); + $"expected {MinLines}-{MaxLines} speaker-tagged lines, got {rawLines.Count}", LlmCallCause.MalformedResponse); } var lines = new List(rawLines.Count); @@ -90,28 +119,39 @@ public static CrosstalkWriteResult Parse(string rawResponse, int maxLineChars, i { if (!TryParseLine(rawLine, out var speaker, out var isInterjection, out var rawText)) { + // SPEC F139.1 (T330 review round 1 amendment): an unrecognized speaker tag is a + // malformed SHAPE, not empty content. return Discarded( $"line does not match the '{HostTag}:'/'{NeighborTag}:' speaker-tag format: " + - $"\"{TruncateForEcho(rawLine)}\""); + $"\"{TruncateForEcho(rawLine)}\"", LlmCallCause.MalformedResponse); } var cleaned = LlmCopyWriter.ApplyCopyHygiene(rawText); if (cleaned.Length == 0) - return Discarded($"a {DescribeSpeaker(speaker)} line was empty after cleanup"); + { + // SPEC F139.1 (T330 review round 1 amendment): the ONE parser reject that STAYS + // EmptyCompletion — the tag matched correctly (the SHAPE was fine), only the text + // after it was empty. See LlmCallCause's own remarks for why this is the deliberate + // holdout among the parser's reject branches. + return Discarded($"a {DescribeSpeaker(speaker)} line was empty after cleanup", LlmCallCause.EmptyCompletion); + } if (cleaned.Length > maxLineChars) { return Discarded( $"a {DescribeSpeaker(speaker)} line ({cleaned.Length} chars) exceeded the " + - $"{maxLineChars}-char per-line budget — no line is ever trimmed (SPEC F127.4)"); + $"{maxLineChars}-char per-line budget — no line is ever trimmed (SPEC F127.4)", LlmCallCause.OverLength); } lines.Add(new CrosstalkAiredLine(speaker, cleaned, isInterjection)); } + // SPEC F139.1 (T330 review round 1 amendment): a missing HOST/NEIGHBOR turn is a malformed + // SHAPE too — every line matched the speaker-tag format individually, but the exchange as a + // whole never took the required two-voice shape. if (lines.All(line => line.Speaker != CrosstalkSpeaker.Host)) - return Discarded($"no {HostTag} line appeared — both speakers must be present"); + return Discarded($"no {HostTag} line appeared — both speakers must be present", LlmCallCause.MalformedResponse); if (lines.All(line => line.Speaker != CrosstalkSpeaker.Neighbor)) - return Discarded($"no {NeighborTag} line appeared — both speakers must be present"); + return Discarded($"no {NeighborTag} line appeared — both speakers must be present", LlmCallCause.MalformedResponse); for (var i = 1; i < lines.Count; i++) { @@ -123,19 +163,55 @@ public static CrosstalkWriteResult Parse(string rawResponse, int maxLineChars, i if (lines[i].Speaker == lines[i - 1].Speaker) { + // SPEC F139.1 (T330 review round 1 amendment): broken alternation is a malformed + // SHAPE — see this file's own MinLines/MaxLines check above for the amendment's + // full rationale. return Discarded( $"speaker alternation broken at line {i + 1} (mark an overlapping line as " + - $"'{InterjectionMarker}' instead)"); + $"'{InterjectionMarker}' instead)", LlmCallCause.MalformedResponse); + } + } + + // SPEC F138.6 (PLAN T333): the truth discard reasons — checked ONCE against the WHOLE + // script's cleaned text (never per line): every violation discards the WHOLE exchange + // regardless which line carried it (F127.4's own "no salvage"), so there is nothing a + // per-line pass would buy over one scan of the joined text. Runs AFTER every shape rule + // above has already passed — a truth check never runs against a reply that was going to be + // discarded as malformed anyway, keeping the FIRST-rule-wins discipline this method opens + // with intact for shape failures, with truth checked as the final gate before the duration + // estimate. + var scriptText = string.Join(' ', lines.Select(line => line.Text)); + + // Table-driven (PLAN T333 review advisory A3): four near-identical shape checks, tried in + // this fixed order, the first match wins. Each pairs a compiled pattern with the honest, + // operator-facing noun phrase for its own reason line — adding a fifth shape is a one-line + // table entry, never a fifth copy-pasted if-block. + foreach (var (pattern, description) in TruthShapeChecks) + { + if (pattern.Match(scriptText) is { Success: true } match) + { + return Discarded( + $"the script named {description} (\"{match.Value}\") — " + + "SPEC F138.6 forbids real-world verifiables in banter", LlmCallCause.TruthGateReject); } } + var clockResult = CopyClaims.CheckClock(scriptText, stationLocalNow); + if (!clockResult.Passed) + { + var violation = clockResult.Violations[0]; + return Discarded( + $"the script claimed \"{violation.Token}\" but the station clock reads " + + $"{violation.Expected} — SPEC F138.6/F138.3 clock violation", LlmCallCause.TruthGateReject); + } + var totalChars = lines.Sum(line => line.Text.Length); var estimatedSeconds = totalChars / CharsPerSecond; if (estimatedSeconds > durationTargetSeconds) { return Discarded( $"estimated {estimatedSeconds:F1}s exceeds the {durationTargetSeconds}s " + - $"{nameof(CrosstalkOptions.DurationTargetSeconds)} target"); + $"{nameof(CrosstalkOptions.DurationTargetSeconds)} target", LlmCallCause.OverLength); } return new CrosstalkWriteResult.Accepted(new CrosstalkAiredScript(lines)); @@ -187,5 +263,69 @@ static string DescribeSpeaker(CrosstalkSpeaker speaker) => static string TruncateForEcho(string text) => text.Length <= MaxEchoedLineChars ? text : text[..MaxEchoedLineChars] + "…"; - static CrosstalkWriteResult.Discarded Discarded(string reason) => new(reason); + static CrosstalkWriteResult.Discarded Discarded(string reason, LlmCallCause cause) => new(reason, cause); + + /// + /// The F138.6 truth-shape table (PLAN T333 review advisory A3) — one entry per mechanical shape + /// check, tried in this fixed order by 's own truth-check loop. Built from + /// already-compiled instances (each [GeneratedRegex] method below + /// returns the SAME cached singleton on every call, so building this table once at static-init + /// costs nothing extra) rather than four separate near-identical if-blocks — adding a fifth shape + /// is a one-line entry here, never a fifth copy-pasted branch. + /// + static readonly (Regex Pattern, string Description)[] TruthShapeChecks = + [ + (FrequencyRx(), "a real-world radio frequency"), + (CallSignRx(), "a real-world call sign"), + // Reuses CopyClaims.ConditionWordRx directly (PLAN T333 review advisory A1) — the SAME + // compiled pattern the F138.1 fact-block checker uses, never a byte-identical copy that + // could silently drift the day either one changes. + (CopyClaims.ConditionWordRx(), "a real-world weather condition"), + (DateClaimRx(), "a real-world date"), + ]; + + // SPEC F138.6: frequency shapes. + // ⚠️ Widened (PLAN T333 review round 1, 2026-08-20 — probe-proven F1): the original FM branch + // required a decimal point, mirroring F138.6's own literal example ("\d+\.\d FM") too narrowly — + // "Radio 101 FM"/"108 FM" (real FM frequencies are commonly spoken as a bare integer, decimal + // omitted) ACCEPTED under that rule. That is the wrong direction to lean: a false PASS here airs + // a fabricated broadcast fact (the exact harm F138.6 exists to stop), while a false DISCARD costs + // only a silent restock (F127.4/F140 — no ladder, no template). FM now matches EITHER a decimal + // frequency ("98.7 FM") OR a bare 2-3 digit integer one ("101 FM", "88 FM") — the FM broadcast + // band is 88-108 MHz, always 2-3 integer digits either way, so both spellings are equally real. + // FM carries NO clock-time collision to guard against (unlike AM below): nobody says "9 FM" to + // mean a time of day — only "AM"/"PM" pair with clock hours in English — so the FM branch needs + // no digit-count floor the way the AM branch does. AM instead requires a THREE-OR-FOUR-DIGIT run + // (real AM frequencies run 540-1700 kHz, never fewer than three digits) rather than a decimal — + // the edge this shape exists to dodge (task-pinned): "9 AM"/"12 AM" is a clock TIME, one or two + // digits, never a station's dial position; "610 AM"/"1010 AM" is a frequency. Case-insensitive: + // unlike ClaimVocabulary's own dropped "clear", neither "fm" nor "am" has a common innocent + // same-shape sense worth protecting mid-sentence, and a model's casing is not part of the contract. + [GeneratedRegex(@"\b\d+\.\d+\s?FM\b|\b\d{2,3}\s?FM\b|\b\d{3,4}\s?AM\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex FrequencyRx(); + + // SPEC F138.6: K/W-prefixed 3-4 letter call signs — US broadcast call signs always start with K + // or W and are conventionally written in full caps. Deliberately CASE-SENSITIVE (no IgnoreCase): + // an ordinary sentence-case word ("Well", "Keep", "Wednesday") never matches [A-Z]{2,3} after its + // first letter, so case sensitivity alone is most of this shape's false-positive defense. The + // residual gap — a genuine all-caps EXCLAMATION or INTERJECTION family that happens to start with + // K/W ("WOW", "WHOA", "WHAT", or a K/W-initial vanity handle like "KDJ") — is accepted rather than + // narrowed further: unlike the ordinary blurb checker's one-re-ask-to-spend posture, a crosstalk + // false discard here costs nothing but a silent restock (F127.4/F140 — no ladder, no template, + // the stock worker simply tries again on its own cadence). + [GeneratedRegex(@"\b[KW][A-Z]{2,3}\b", RegexOptions.CultureInvariant)] + private static partial Regex CallSignRx(); + + // SPEC F138.6: digit-date shapes, scoped honestly to what a checker can verify mechanically — + // a calendar-shaped year ("2026"), a month name immediately followed by a day number + // (optionally ordinal-suffixed, "August 20"/"August 20th"), or the bare "Nth of" ordinal-date + // shape ("the 20th of"). A BARE small number ("twenty minutes", "give me a 20") never trips any + // of the three branches — the month/ordinal marker is required, never inferred from magnitude + // alone, exactly the false-positive posture CopyClaims documents for its own digit-run class. + // ClaimVocabulary.MonthAlternation is the one canonical month list (PLAN T333 review advisory + // A2) — no second, GenWave.Tts-local copy. + [GeneratedRegex( + $@"\b(?:19|20)\d{{2}}\b|\b(?:{ClaimVocabulary.MonthAlternation})\s+\d{{1,2}}(?:st|nd|rd|th)?\b|\b\d{{1,2}}(?:st|nd|rd|th)\s+of\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex DateClaimRx(); } diff --git a/src/GenWave.Tts/CrosstalkScriptWriter.cs b/src/GenWave.Tts/CrosstalkScriptWriter.cs index 21e9cf70..cf837815 100644 --- a/src/GenWave.Tts/CrosstalkScriptWriter.cs +++ b/src/GenWave.Tts/CrosstalkScriptWriter.cs @@ -18,7 +18,8 @@ namespace GenWave.Tts; /// One completion, whole exchange (SPEC F127.3). Unlike , this /// writer NEVER degrades to a template — F127.4 is skip-only: any failure (disabled endpoint, /// transport fault, a completion truncated at max_tokens, malformed reply, a line failing -/// hygiene/budget, an over-target duration estimate) returns +/// hygiene/budget, an over-target duration estimate, a F138.6 truth-gate violation) returns +/// /// with one reason, logged at Information /// (never WARN — banter is optional color, a miss is not an outage) and recorded into /// under so /api/llm-calls can @@ -44,7 +45,7 @@ public sealed class CrosstalkScriptWriter( IHttpClientFactory httpClientFactory, IOptionsMonitor llmOptions, IOptionsMonitor crosstalkOptions, - LlmCallRing callRing, + LlmCallRecorder recorder, IDegradationModeReader degradationMode, ILogger logger, TimeProvider timeProvider) @@ -103,14 +104,22 @@ public async Task WriteExchangeAsync(CrosstalkExchangeRequ // SPEC F140 review finding F3: this is the archetypal pre-flight refusal — zero I/O, // resolves in microseconds — so it carries GenerationAttempted: false (see // CrosstalkWriteResult.Discarded's own remarks for why a pacing caller cares). + // + // SPEC F139.1 (T330 review advisory): Cause is a DELIBERATE ConnectionFailure here, not a + // default filled just to satisfy the parameter — systemPrompt: null skips the ring/counter + // record below (nothing was ever attempted, mirroring GenerationAttempted: false above), + // so this value only ever surfaces on the returned Discarded itself, never on a ring row. + // ConnectionFailure is the honest answer regardless: an unset Llm:Endpoint IS "nowhere + // configured to connect to" — a connection-layer fact, not a shape or a timeout one. return Discard( - "Llm:Endpoint is not configured", personaName, startedAt, mode, systemPrompt: null, userPrompt: null, - generationAttempted: false); + "Llm:Endpoint is not configured", LlmCallCause.ConnectionFailure, personaName, startedAt, mode, + systemPrompt: null, userPrompt: null, cfg.Model, generationAttempted: false); } var durationTargetSeconds = crosstalkOptions.CurrentValue.DurationTargetSeconds; - var systemPrompt = CrosstalkPromptBuilder.BuildSystemPrompt(request.HostCard, request.NeighborCard, durationTargetSeconds); + var systemPrompt = CrosstalkPromptBuilder.BuildSystemPrompt( + request.HostCard, request.NeighborCard, durationTargetSeconds, request.StationLocalNow); var userPrompt = CrosstalkPromptBuilder.BuildUserContent( request, LlmPromptBuilder.BuildStationClockLine(request.StationLocalNow)); @@ -165,15 +174,18 @@ public async Task WriteExchangeAsync(CrosstalkExchangeRequ { return Discard( "the completion was cut short by max_tokens (finish_reason: length) — a truncated reply is never aired", - personaName, startedAt, mode, systemPrompt, userPrompt, raw); + LlmCallCause.OverLength, personaName, startedAt, mode, systemPrompt, userPrompt, cfg.Model, raw); } - var result = CrosstalkScriptParser.Parse(raw, cfg.MaxCopyChars, durationTargetSeconds); + var result = CrosstalkScriptParser.Parse(raw, cfg.MaxCopyChars, durationTargetSeconds, request.StationLocalNow); return result switch { - CrosstalkWriteResult.Accepted => Accept(result, personaName, systemPrompt, userPrompt, raw, startedAt, mode), + CrosstalkWriteResult.Accepted => Accept( + result, personaName, systemPrompt, userPrompt, raw, startedAt, mode, cfg.Model), + // discarded.Cause was decided once, at the source, inside CrosstalkScriptParser.Parse's + // own reject branches (SPEC F139.1) — never re-derived here from discarded.Reason's text. CrosstalkWriteResult.Discarded discarded => Discard( - discarded.Reason, personaName, startedAt, mode, systemPrompt, userPrompt, raw), + discarded.Reason, discarded.Cause, personaName, startedAt, mode, systemPrompt, userPrompt, cfg.Model, raw), _ => throw new System.Diagnostics.UnreachableException($"Unhandled {nameof(CrosstalkWriteResult)} case."), }; } @@ -185,7 +197,7 @@ public async Task WriteExchangeAsync(CrosstalkExchangeRequ } catch (Exception ex) { - var (outcome, detail) = LlmCopyWriter.ClassifyForRing(ex); + var (outcome, cause, detail) = LlmCopyWriter.ClassifyForRing(ex); // SPEC F140 review finding F3: an HttpRequestException with no StatusCode is .NET's own // signal that no response was ever received (a connect refusal, DNS failure, TLS @@ -196,49 +208,50 @@ public async Task WriteExchangeAsync(CrosstalkExchangeRequ // so it keeps GenerationAttempted's own default of true. var generationAttempted = ex is not HttpRequestException { StatusCode: null }; - callRing.Record( + recorder.Record( personaName, systemPrompt, userPrompt, response: null, startedAt, ElapsedMs(startedAt), - outcome, detail, mode, LlmCallKind.Crosstalk); + outcome, detail, mode, cause, cfg.Model, LlmCallKind.Crosstalk); logger.LogInformation( "Crosstalk exchange discarded (persona: {PersonaName}): {Detail}", personaName.ReplaceLineEndings(" "), detail.ReplaceLineEndings(" ")); - return new CrosstalkWriteResult.Discarded(detail, generationAttempted); + return new CrosstalkWriteResult.Discarded(detail, cause, generationAttempted); } } CrosstalkWriteResult Accept( CrosstalkWriteResult result, string personaName, string systemPrompt, string userPrompt, string raw, - DateTimeOffset startedAt, DegradationMode mode) + DateTimeOffset startedAt, DegradationMode mode, string model) { - callRing.Record( + recorder.Record( personaName, systemPrompt, userPrompt, raw, startedAt, ElapsedMs(startedAt), - LlmCallOutcome.Ok, statusDetail: null, mode, LlmCallKind.Crosstalk); + LlmCallOutcome.Ok, statusDetail: null, mode, LlmCallCause.Success, model, LlmCallKind.Crosstalk); return result; } /// /// The one discard path every failure funnels through (SPEC F127.4) — records into - /// (skipped entirely when is null, i.e. - /// nothing was ever attempted — the disabled-endpoint short-circuit, mirroring + /// (skipped entirely when is null, + /// i.e. nothing was ever attempted — the disabled-endpoint short-circuit, mirroring /// 's own "disabled means no ring entry" posture) and logs /// exactly one Information line (never WARN — F127.4's own posture: a discard is discipline, not - /// an outage). + /// an outage). (SPEC F139.1, PLAN T330) is decided by the CALLER, at the + /// point it already knows why — this method never inspects 's text. /// CrosstalkWriteResult.Discarded Discard( - string reason, string personaName, DateTimeOffset startedAt, DegradationMode mode, - string? systemPrompt, string? userPrompt, string? raw = null, bool generationAttempted = true) + string reason, LlmCallCause cause, string personaName, DateTimeOffset startedAt, DegradationMode mode, + string? systemPrompt, string? userPrompt, string model, string? raw = null, bool generationAttempted = true) { if (systemPrompt is not null) { - callRing.Record( + recorder.Record( personaName, systemPrompt, userPrompt, raw, startedAt, ElapsedMs(startedAt), - LlmCallOutcome.Rejected, reason, mode, LlmCallKind.Crosstalk); + LlmCallOutcome.Rejected, reason, mode, cause, model, LlmCallKind.Crosstalk); } logger.LogInformation( "Crosstalk exchange discarded (persona: {PersonaName}): {Reason}", personaName.ReplaceLineEndings(" "), reason.ReplaceLineEndings(" ")); - return new CrosstalkWriteResult.Discarded(reason, generationAttempted); + return new CrosstalkWriteResult.Discarded(reason, cause, generationAttempted); } long ElapsedMs(DateTimeOffset startedAt) => (long)(timeProvider.GetUtcNow() - startedAt).TotalMilliseconds; diff --git a/src/GenWave.Tts/CrosstalkWriteResult.cs b/src/GenWave.Tts/CrosstalkWriteResult.cs index 05719cdc..4eb34bea 100644 --- a/src/GenWave.Tts/CrosstalkWriteResult.cs +++ b/src/GenWave.Tts/CrosstalkWriteResult.cs @@ -29,6 +29,14 @@ public sealed record Accepted(CrosstalkAiredScript Script) : CrosstalkWriteResul /// transport miss) — one string, one source of truth for "why was there no banter" across the log /// line, the ring, and this return value. /// + /// + /// SPEC F139.1 (STORY-353, PLAN T330): the F139 cause this discard stamps into + /// , decided once at the SOURCE that already knows why — each + /// reject branch names its own, and + /// 's own finish_reason: length/exception-catch discards + /// carry theirs — never re-derived downstream from 's text. See + /// 's own remarks for the full resolution-point map. + /// /// /// SPEC F140 review finding F3 (STORY-354, PLAN T328): ONLY when this /// discard happened WITHOUT ever attempting a generation — Llm:Endpoint unset, or a @@ -41,5 +49,5 @@ public sealed record Accepted(CrosstalkAiredScript Script) : CrosstalkWriteResul /// takes (GenWave.Host.Crosstalk.CrosstalkStockPacing) reads this to decide whether an /// elapsed time is worth blending into its rolling estimate at all. /// - public sealed record Discarded(string Reason, bool GenerationAttempted = true) : CrosstalkWriteResult; + public sealed record Discarded(string Reason, LlmCallCause Cause, bool GenerationAttempted = true) : CrosstalkWriteResult; } diff --git a/src/GenWave.Tts/LlmCallCause.cs b/src/GenWave.Tts/LlmCallCause.cs new file mode 100644 index 00000000..3f2ced55 --- /dev/null +++ b/src/GenWave.Tts/LlmCallCause.cs @@ -0,0 +1,112 @@ +namespace GenWave.Tts; + +/// +/// The F139 cause taxonomy (SPEC F139.1, STORY-353, PLAN T330) — WHY an LLM call resolved the way it +/// did, stamped onto every alongside its existing, coarser +/// (which stays exactly as it was — F139.1's own "nothing else about F73 +/// changes"). Where answers "did the ring show usable text or not", +/// answers "why is the red tile red" (SPEC F139.2, F139.4) — the taxonomy +/// the dominant-cause admin surface (PLAN T334) groups by, alongside model and +/// . +/// +/// +/// Resolution-point map (PLAN T330). : +/// an exact-fit or salvaged-trim CleanCopy result is ; a full reject is +/// when a candidate existed but none survived the cap, or +/// when hygiene left nothing at all +/// ( is the source of truth, decided once at +/// — never re-derived from any later string). Its own catch-all +/// (, shared with ) maps +/// our own Llm:TimeoutSeconds budget elapsing to and everything else +/// (non-2xx, connect failure, malformed endpoint URI, bad JSON) to — +/// PLAN T334's own doc note: that catch-all has no finer split between "a response arrived but was +/// non-2xx" and "no response ever arrived at all" either; +/// (the HTTP status or exception type name) is what carries that finer distinction, for whichever +/// LATER task wants to split it out of rather than this enum growing a +/// ninth value for it today. +/// +/// +/// +/// /: an accepted script is +/// ; a finish_reason: length truncation, a per-line budget overrun, or an +/// over-target duration estimate are all (the reply came back but did not +/// fit — this fold is unchanged by the amendment below). ⚠️ Amended (T330 review round 1, +/// 2026-08-20 — the F135.5 precedent, SPEC F139.1): every OTHER parse-shape reject — a line count +/// outside - +/// (in EITHER direction: too few is genuinely empty-ish, but too MANY is the reviewer's own exhibit — +/// a reply carrying MORE content than the shape allows is not "empty" by any honest reading), an +/// unrecognized speaker tag, a missing HOST or NEIGHBOR turn, or broken speaker alternation — is +/// , not : the reply came back and had +/// CONTENT, it just never took the required shape. Folding these into EmptyCompletion sent the +/// operator to the wrong levers (endpoint, max_tokens) when the right answer is "this model can't +/// follow the output format", and corrupted the F138.7 model-floor signal for which malformed-shape is +/// the single most model-discriminating cause. The one parser reject that STAYS +/// : a line whose tag matched correctly but whose text was empty after +/// hygiene cleanup — the SHAPE was fine, only the content was missing, the same "nothing usable +/// resulted" story 's own empty-hygiene case tells. +/// is stamped by GenWave.Host.Crosstalk.CrosstalkStockWorker alone, never by +/// itself — that writer's own +/// catch cannot tell a break-window abandon apart from a host shutdown (both surface identically as +/// "the caller's ct fired"); only the stock worker, which owns the linked +/// pair, can honestly distinguish the two. +/// +/// +/// +/// ⚠️ Amended (PLAN T333, 2026-08-20 — SPEC F138.6): 's +/// own truth discard reasons (a frequency/call-sign shape, a weather-condition word, a digit-date +/// shape, or a violation) all stamp +/// too — the crosstalk seam's own copy of the SAME truth gate already +/// stamps it for (PLAN T331). Deliberately NOT : every one of these +/// rejects a script that already cleared shape validation (both speakers, alternation, per-line +/// budget) — the STRUCTURE was fine, the CONTENT lied. +/// +/// +public enum LlmCallCause +{ + /// The call produced usable copy — an exact fit, or content otherwise accepted as-is + /// (a salvage still counts: the copy aired). + Success, + + /// Our own Llm:TimeoutSeconds budget elapsed before a response arrived. + Timeout, + + /// The reply came back but did not fit a length/duration constraint this project + /// enforces — the gh-#277 family (a Llm:MaxCopyChars reject with no sentence-boundary + /// salvage) and its analogues (a finish_reason: length + /// truncation, a per-line budget overrun, an over-target duration estimate). + OverLength, + + /// The copy failed the F138 truth gate (SPEC F138, STORY-350/351/352): stamped by + /// 's own ladder (PLAN T331) and by 's + /// mechanical F138.6 discard reasons (PLAN T333) alike. + TruthGateReject, + + /// A non-2xx status, a connect failure, a malformed endpoint URI, or bad JSON — any + /// completions fault other than this call's own timeout budget elapsing (see this enum's own + /// class remarks for the T334 doc note on where the non-2xx/no-response distinction lives + /// instead). + ConnectionFailure, + + /// A generation was abandoned mid-flight because a + /// break window opened (SPEC F127.7, F140.2) — stamped only by + /// GenWave.Host.Crosstalk.CrosstalkStockWorker, the one place that can tell this apart from + /// an ordinary caller cancellation; see this enum's own class remarks. + CanceledByWindow, + + /// The completions endpoint returned 2xx, but nothing usable resulted: hygiene left an + /// empty string (), or a line + /// matched its speaker tag correctly but was empty after cleanup (see this enum's own class + /// remarks for why that ONE parser reject stays here while every other shape reject moved to + /// ). + EmptyCompletion, + + /// + /// ⚠️ Added (T330 review round 1, 2026-08-20 — SPEC F139.1 amendment): the reply came back WITH + /// content, but that content never took the shape requires — + /// an unrecognized speaker tag, a line count outside its 3-8 range (too few OR too many), a + /// missing HOST or NEIGHBOR turn, or broken speaker alternation. See this enum's own class remarks + /// for the full amendment rationale and the one parser reject that deliberately stays + /// instead. + /// + MalformedResponse, +} diff --git a/src/GenWave.Tts/LlmCallCauseCount.cs b/src/GenWave.Tts/LlmCallCauseCount.cs new file mode 100644 index 00000000..97f10f95 --- /dev/null +++ b/src/GenWave.Tts/LlmCallCauseCount.cs @@ -0,0 +1,11 @@ +namespace GenWave.Tts; + +/// +/// One aggregated row of (SPEC F139.2, STORY-353, PLAN +/// T330): how many calls landed on for / +/// within the rolling 24h window. This is the seam PLAN T334 reads to build the +/// /api/llm-calls counter summary (GenWave.Host.Api.LlmCallCauseSummaryDto) and, +/// via , the red health tile's "dominant recent +/// cause" line. +/// +public sealed record LlmCallCauseCount(LlmCallCause Cause, string Model, LlmCallKind Kind, int Count); diff --git a/src/GenWave.Tts/LlmCallCauseCounters.cs b/src/GenWave.Tts/LlmCallCauseCounters.cs new file mode 100644 index 00000000..5af54521 --- /dev/null +++ b/src/GenWave.Tts/LlmCallCauseCounters.cs @@ -0,0 +1,140 @@ +namespace GenWave.Tts; + +/// +/// The F139.2 rolling 24h counter store (STORY-353, PLAN T330): how many LLM calls landed on each +/// (, model, ) combination within the last 24 hours +/// — the seam PLAN T334 reads (via and ) to build +/// the /api/llm-calls counter summary and the red health tile's "dominant recent cause" line. +/// No persistence of any kind (F139.3, F73.3/F73.4 stand): +/// this class's only dependency is , so a process restart clears it by +/// construction, the exact same posture itself documents. +/// +/// +/// Deliberately NOT composed inside . Story196_LlmCallInspector's +/// own F73.3 structural proof pins that class to EXACTLY ONE constructor parameter +/// (IOptionsMonitor<LlmOptions>) as evidence it cannot persist anything — adding a second +/// dependency there would break that proof for an unrelated reason. is +/// where the two independent observers of "a call resolved" reunite (SPEC F139 review finding F2, +/// PLAN T330) — every // +/// GenWave.Host.Crosstalk.CrosstalkStockWorker call site feeds that ONE class, which then calls +/// both this class's own and — never one class +/// doing both jobs itself. +/// +/// +/// +/// Rolling 24h via hourly buckets, aged lazily (SPEC F139.2). Each stamps +/// the CURRENT UTC hour's bucket for the given key; a bucket more than 24h older than "now" is dropped +/// the next time either or runs, never on a background +/// timer of its own. Memory is bounded by (at most 25 live hourly buckets) × (distinct cause/model/kind +/// combinations actually seen) — never an unbounded list of raw call timestamps, and never a sweep an +/// operator has to remember exists. +/// +/// Precision note: because entries are grouped by hour rather than by exact timestamp, the +/// true retention window is "24h to 25h", never a razor's-edge 24h+1-second cutoff — an entry recorded +/// at the very start of its own hourly bucket can be retained for up to an additional ~59 minutes +/// past a strict 24h before that bucket ages out. Acceptable for a "dominant recent cause" admin +/// surface (SPEC F139.2/F139.4); nothing here promises second-level retention precision. +/// +/// +public sealed class LlmCallCauseCounters(TimeProvider timeProvider) +{ + static readonly TimeSpan RollingWindow = TimeSpan.FromHours(24); + + readonly object gate = new(); + + // Hour-bucket start (UTC, truncated to the hour) -> per-(cause, model, kind) count within that + // hour. A plain Dictionary (T330 review advisory), not a SortedDictionary: neither Record's own + // Prune call nor Snapshot ever reads buckets in hour order — Prune filters Keys by a cutoff + // comparison (order-independent) and Snapshot sums every bucket's Values into one unordered + // total — so a SortedDictionary's O(log n) insert/lookup would only be paying for an ordering + // guarantee this class never spends. + readonly Dictionary> buckets = new(); + + /// Counts one resolved call under its current-hour bucket (SPEC F139.2). Called from + /// — the one shared method that reunites this call with + /// at every resolution point (see that class's own remarks for + /// why this store still stays a separate singleton rather than folding into the ring itself). + public void Record(LlmCallCause cause, string model, LlmCallKind kind) + { + var now = timeProvider.GetUtcNow(); + var hour = TruncateToHour(now); + + lock (gate) + { + if (!buckets.TryGetValue(hour, out var counts)) + { + counts = []; + buckets[hour] = counts; + } + + var key = (cause, model, kind); + counts[key] = counts.GetValueOrDefault(key) + 1; + + Prune(hour); + } + } + + /// Every (cause, model, kind) combination counted within the rolling 24h window, summed + /// across hourly buckets — the read seam T334's counter summary/health tile consume. Ages out on + /// the 24-25h hourly-bucket band this class's own remarks document (STORY-353 AC2, amended at + /// T330 review) — never a razor's-edge 24h+1s cutoff, always over-retention rather than under. + public IReadOnlyList Snapshot() + { + lock (gate) + { + Prune(TruncateToHour(timeProvider.GetUtcNow())); + + var totals = new Dictionary<(LlmCallCause Cause, string Model, LlmCallKind Kind), int>(); + foreach (var counts in buckets.Values) + { + foreach (var (key, count) in counts) + totals[key] = totals.GetValueOrDefault(key) + count; + } + + return totals + .Select(entry => new LlmCallCauseCount(entry.Key.Cause, entry.Key.Model, entry.Key.Kind, entry.Value)) + .ToList(); + } + } + + /// + /// The single highest-count non- row within the rolling 24h + /// window, restricted to (SPEC F139.2, PLAN T334) — the red health tile's + /// "dominant recent cause" line reads directly off this, never re-deriving it from + /// itself. Ties (equal counts) break first by 's + /// own declaration order, then by an ordinal comparison of — + /// deterministic, never "whichever the dictionary happens to enumerate first". + /// + /// + /// when nothing but (or nothing at all) + /// was recorded for within the window. Restricted to one kind + /// rather than pooling Copy and Crosstalk together: the "LLM" dashboard tile this feeds + /// (GenWave.Host.Api.StatusController) reflects LlmCopyStatusHolder's own + /// last-attempt verdict — a copy-writer failure, never a crosstalk one — so its explanation has to + /// stay scoped to the SAME kind that made the tile red in the first place, or the line would name + /// a cause the operator's own red tile was never actually about. + /// + /// + public LlmCallCauseCount? DominantFailure(LlmCallKind kind) => + Snapshot() + .Where(row => row.Kind == kind && row.Cause != LlmCallCause.Success) + .OrderByDescending(row => row.Count) + .ThenBy(row => row.Cause) + .ThenBy(row => row.Model, StringComparer.Ordinal) + .FirstOrDefault(); + + /// Drops every bucket more than older than + /// — called from inside by both and , + /// so a quiet counter (no calls at all) still self-trims the moment anyone reads or writes it, with + /// no timer of its own (this class's own remarks). + void Prune(DateTimeOffset nowHour) + { + var cutoff = nowHour - RollingWindow; + var stale = buckets.Keys.Where(hour => hour < cutoff).ToList(); + foreach (var hour in stale) + buckets.Remove(hour); + } + + static DateTimeOffset TruncateToHour(DateTimeOffset instant) => + new(instant.Year, instant.Month, instant.Day, instant.Hour, minute: 0, second: 0, instant.Offset); +} diff --git a/src/GenWave.Tts/LlmCallRecord.cs b/src/GenWave.Tts/LlmCallRecord.cs index 3b8f2027..61c41aea 100644 --- a/src/GenWave.Tts/LlmCallRecord.cs +++ b/src/GenWave.Tts/LlmCallRecord.cs @@ -40,6 +40,20 @@ namespace GenWave.Tts; /// ok/failed/timeout/trimmed/rejected (SPEC F73.1, F123.2-F123.4, F127.4). /// The HTTP status or exception type name for a non- outcome; for Ok and for alike — a trim is not a fault, so it carries no fault detail either. Carries the discard reason for (SPEC F127.4, F127.11). /// The degradation mode active at call time (SPEC F73.1, F69.1) — Normal/Soft/Hard. +/// +/// WHY this call resolved the way it did (SPEC F139.1, STORY-353, PLAN T330) — a finer-grained, +/// additive sibling to above (which is unchanged by T330: "nothing else about +/// F73 changes"). See 's own remarks for the full resolution-point map. +/// +/// +/// The completions model this call used (SPEC F139.2, PLAN T330) — Llm:Model at call time, +/// carried here so the rolling cause counters () can key on it +/// without a second lookup. Never (T330 review advisory): unlike +/// , every caller already has LlmOptions.CurrentValue in hand before +/// it can reach ANY call site — LlmOptions.Model itself +/// defaults to "", never null, so there is no genuine "config not in hand yet" case to document +/// here. +/// /// /// Which generation surface produced this call (SPEC F127.11, PLAN T282) — /// for every call itself records, for @@ -59,4 +73,6 @@ public sealed record LlmCallRecord( LlmCallOutcome Outcome, string? StatusDetail, DegradationMode Mode, + LlmCallCause Cause, + string Model, LlmCallKind Kind = LlmCallKind.Copy); diff --git a/src/GenWave.Tts/LlmCallRecorder.cs b/src/GenWave.Tts/LlmCallRecorder.cs new file mode 100644 index 00000000..d456b3c4 --- /dev/null +++ b/src/GenWave.Tts/LlmCallRecorder.cs @@ -0,0 +1,63 @@ +namespace GenWave.Tts; + +/// +/// The single Record point every resolved LLM call feeds (SPEC F139.1, F139.2, STORY-353, PLAN T330 — +/// review finding F2). Before this class existed, SIX call sites across two assemblies +/// ('s own success and catch-all paths, 's +/// own Accept/Discard/catch-all paths, and GenWave.Host.Crosstalk.CrosstalkStockWorker's own +/// break-window abandon) each wrote the SAME call immediately followed +/// by the SAME call, in lockstep, with only a code comment at +/// each site enforcing that the (cause, model, kind) triple passed to both stayed the same. A mutation +/// deleting just the counters half of that pair left every ring-facing fact green (T330 review's own +/// finding: "a mutation deleting ALL counter feeds pass green") — this class makes that mutation +/// impossible to express: there is no longer a "just the counters half" to delete, only one Record call +/// that does both or neither. +/// +/// +/// ONE required dependency pair (mirrors GenWave.Host.Crosstalk.CrosstalkStockPacing's own "the +/// ONE dependency" shape one project over) — and +/// stay separate singletons rather than merging into one class (see 's +/// own remarks for why: Story196_LlmCallInspector's F73.3 structural proof pins +/// 's constructor to exactly one parameter as evidence it cannot persist +/// anything, so a second dependency there would break that proof for an unrelated reason). This class is +/// where the two independent observers of "a call resolved" reunite into the one call site every writer +/// actually wants. +/// +/// +/// +/// Degradation mode stays a parameter, deliberately NOT a dependency here. Every caller already +/// reads itself, ONCE, at the moment its own generation +/// attempt STARTS ('s own remarks: "mode is read +/// fresh right here... reading it uniformly for every path keeps this the one recording point instead of +/// two") — a call can run for real wall-clock seconds (up to Llm:TimeoutSeconds) before it +/// resolves, and the ring entry is meant to reflect the mode active when the ATTEMPT started, not +/// whatever the mode happened to drift to by the time this class's own runs. Folding +/// in as a dependency here and reading it fresh at record time would +/// silently relocate that read to the wrong instant for every existing caller — so +/// stays threaded through exactly as it already was, each caller's own already-captured value. +/// One honest exception: GenWave.Host.Crosstalk.CrosstalkStockWorker.RecordWindowCancellation +/// reads at RECORD time (after the abandoned attempt +/// has already unwound), not at its own attempt start — that worker never captured the mode when the +/// attempt began, so this is the one caller for which the invariant above does not hold. +/// +/// +public sealed class LlmCallRecorder(LlmCallRing callRing, LlmCallCauseCounters causeCounters) +{ + /// + /// Records one resolved call into both the ring and the rolling counters — the one call every + /// former / pair collapses + /// to. Parameters mirror 's own exactly; / + /// / are the SAME triple both stores key on, passed + /// here exactly once rather than twice. + /// + public void Record( + string? personaName, string? promptSystem, string? promptUser, string? response, DateTimeOffset startedAt, + long elapsedMs, LlmCallOutcome outcome, string? statusDetail, DegradationMode mode, LlmCallCause cause, + string model, LlmCallKind kind = LlmCallKind.Copy) + { + callRing.Record( + personaName, promptSystem, promptUser, response, startedAt, elapsedMs, outcome, statusDetail, mode, + cause, model, kind); + causeCounters.Record(cause, model, kind); + } +} diff --git a/src/GenWave.Tts/LlmCallRing.cs b/src/GenWave.Tts/LlmCallRing.cs index e065c8f6..b5ba3dce 100644 --- a/src/GenWave.Tts/LlmCallRing.cs +++ b/src/GenWave.Tts/LlmCallRing.cs @@ -43,20 +43,23 @@ public sealed class LlmCallRing(IOptionsMonitor options) /// Llm:CallRingCapacity trims (or grows) the ring on the very next record. /// (gh-#429) is the caller's already-resolved /// result — this ring never re-derives a name itself, - /// it only stores what it was handed. (SPEC F127.11, PLAN T282) defaults - /// to so 's own two call sites need no - /// change — only ever passes . + /// it only stores what it was handed. / (SPEC + /// F139.1, PLAN T330) are the additive F139 fields — see 's own remarks. + /// (SPEC F127.11, PLAN T282) defaults to so + /// 's own two call sites need no change — only + /// (and, for a break-window abandon, GenWave.Host.Crosstalk.CrosstalkStockWorker) ever pass + /// . /// public void Record( string? personaName, string? promptSystem, string? promptUser, string? response, DateTimeOffset startedAt, - long elapsedMs, LlmCallOutcome outcome, string? statusDetail, DegradationMode mode, - LlmCallKind kind = LlmCallKind.Copy) + long elapsedMs, LlmCallOutcome outcome, string? statusDetail, DegradationMode mode, LlmCallCause cause, + string model, LlmCallKind kind = LlmCallKind.Copy) { lock (gate) { var record = new LlmCallRecord( ++nextSeq, personaName, promptSystem, promptUser, response, startedAt, elapsedMs, outcome, - statusDetail, mode, kind); + statusDetail, mode, cause, model, kind); ring.AddFirst(record); // newest first var capacity = options.CurrentValue.CallRingCapacity; diff --git a/src/GenWave.Tts/LlmCopyCleanupResult.cs b/src/GenWave.Tts/LlmCopyCleanupResult.cs index b473aeb7..dbfefc48 100644 --- a/src/GenWave.Tts/LlmCopyCleanupResult.cs +++ b/src/GenWave.Tts/LlmCopyCleanupResult.cs @@ -3,11 +3,12 @@ namespace GenWave.Tts; /// /// Outcome of 's hygiene pass plus the F123.2 sentence-boundary /// salvage (SPEC F123.2-F123.4, STORY-319, PLAN T263) — mirrors -/// 's closed-hierarchy shape so the three -/// outcomes (an exact fit, a salvaged trim, or a full reject) can never be confused with one another -/// the way a single nullable-string-plus-bool pair could — a trim always carries real text, a reject -/// never does, by construction rather than by convention. Internal: is -/// this type's only producer and only consumer. +/// 's closed-hierarchy shape so the outcomes +/// (an exact fit, a salvaged trim, a full hygiene reject, or — as of PLAN T331 — a +/// re-ask) can never be confused with one another the way a single +/// nullable-string-plus-bool pair could — a trim always carries real text, a reject never does, by +/// construction rather than by convention. Internal: is this type's only +/// producer and only consumer. /// internal abstract record LlmCopyCleanupResult { @@ -31,5 +32,26 @@ public sealed record Trimmed(string Text, int CharsBeforeTrim) : LlmCopyCleanupR /// FIRST sentence already exceeds the cap (nothing complete to cut at) — the pre-F123 reject /// stands, byte-identical to before T263. /// - public sealed record Rejected : LlmCopyCleanupResult; + /// + /// SPEC F139.1 (STORY-353, PLAN T330): for the over-length-with-no-salvage + /// case (a candidate existed but none survived maxChars — the gh-#277 family, + /// ); when hygiene left an empty + /// string outright (). Decided once, here at the source + /// (), never re-derived downstream from anything about the + /// text itself. + /// + public sealed record Rejected(bool WasOverLength) : LlmCopyCleanupResult; + + /// + /// The F138.4 ladder's own floor (T331 review finding F3): a re-ask that hygiene would otherwise + /// have accepted — it fit, or salvaged to a real sentence — but that STILL failed + /// a second time. Distinct from the plain + /// case (hygiene itself found nothing usable) precisely so + /// 's own failure WARN can name the real cause — the truth + /// gate rejected genuinely well-formed copy — rather than the wrong-lever "empty or exceeded + /// Llm:MaxCopyChars" message a hygiene reject carries. is the re-ask's + /// own (SPEC F138.1-F138.2), carried through so that + /// WARN can also name the still-unsupported claim. + /// + public sealed record TruthGateRejected(IReadOnlyList Violations) : LlmCopyCleanupResult; } diff --git a/src/GenWave.Tts/LlmCopyWriter.cs b/src/GenWave.Tts/LlmCopyWriter.cs index 963bc3ab..8ef1965b 100644 --- a/src/GenWave.Tts/LlmCopyWriter.cs +++ b/src/GenWave.Tts/LlmCopyWriter.cs @@ -124,7 +124,7 @@ public sealed class LlmCopyWriter( IActivePersonaAccessor personaAccessor, ILogger logger, TimeProvider timeProvider, - LlmCallRing callRing, + LlmCallRecorder recorder, IDegradationModeReader degradationMode, IStationClockProvider? stationClock = null, IContextPatterFactSource? patterFactSource = null, @@ -329,8 +329,12 @@ public async Task WriteAsync(SegmentRequest request, CancellationTo if (cleaned is null) { statusHolder.Record(LlmAttemptOutcome.Failed, attemptedAt); + // The reason NAMES the real cause (T331 review finding F3) — a truth-gate exhaustion + // is not "empty or exceeded Llm:MaxCopyChars": that wrong-lever WARN sent an operator + // at the endpoint/max-tokens settings for a failure those levers cannot fix. See + // DescribeNullTextReason's own remarks. LogFailure(request, persona, cfg.Model, attemptedAt, exception: null, - reason: "empty or exceeded Llm:MaxCopyChars after cleanup"); + reason: DescribeNullTextReason(cleanup)); return await fallback.WriteAsync(request, ct); } @@ -413,8 +417,17 @@ public async Task WritePreviewAsync( cfg, request, personaOverride, card: null, updateTasteMemory: false, patterFact: null, showFlavorFact: null, queueWaitBudget: TimeSpan.FromSeconds(cfg.PreviewQueueWaitSeconds), ct); var cleaned = TextOf(cleanup); + // DescribeNullTextReason (review round-2 finding F2, PLAN T332): the SAME reason + // WriteAsync's own failure WARN already names, reused here rather than a second, hardcoded + // "empty or over-length" message — a truth-gate reject reaching this branch (a preview + // exhausting the F138.4 ladder, now reachable since T332 widened the ladder to every kind) + // deserves the honest cause too, not the wrong-lever hygiene wording that sends an operator + // at settings a truth-gate failure has nothing to do with. Safe to surface on this + // authenticated admin surface: every interpolated fragment is either a fixed phrase or a + // ClaimViolation.Token, which is provably digit-shaped or closed-vocabulary (that type's + // own remarks), never free text. return cleaned is null - ? new PersonaPreviewResult.Failed("The LLM returned empty or over-length copy.") + ? new PersonaPreviewResult.Failed($"The LLM reply was rejected ({DescribeNullTextReason(cleanup)}).") : new PersonaPreviewResult.Success(cleaned); } catch (OperationCanceledException) when (ct.IsCancellationRequested) @@ -481,6 +494,26 @@ void LogFailure( detail.ReplaceLineEndings(" "), outcome); } + /// + /// One INFORMATION line (SPEC F123.4) whenever a completion's own hygiene pass needed the + /// sentence-boundary salvage — a trim is discipline, not an outage, so it gets its own quiet lane + /// rather than promoting to 's WARN. Shared by the first completion and + /// the F138.4 re-ask alike (STORY-350, PLAN T331), so a re-ask reply that also needed salvaging + /// is observable exactly the same way the first one always was — extracted here so the two call + /// sites cannot drift onto two different messages for the same event. + /// + void LogIfTrimmed(SegmentRequest request, string? personaName, LlmCopyCleanupResult cleanup) + { + if (cleanup is not LlmCopyCleanupResult.Trimmed trimmed) + return; + + logger.LogInformation( + "LLM copy for {Kind} trimmed to the last complete sentence under Llm:MaxCopyChars " + + "(persona: {PersonaName}): {CharsBefore} -> {CharsAfter} chars", + request.Kind, (personaName ?? "none").ReplaceLineEndings(" "), trimmed.CharsBeforeTrim, + trimmed.Text.Length); + } + /// /// SPEC F107.5 (STORY-298, PLAN T225) — the patter lane's ONE pull point in this writer: called /// exclusively from , and only for the two music-adjacent kinds a patter @@ -551,6 +584,13 @@ async Task RequestCleanedCompletionAsync( // preview call never passes through DegradationGatedCopyWriter (SPEC F69.4), so there is no // caller-evaluated mode available for that path, and reading IDegradationModeReader // uniformly for every path keeps this the one recording point instead of two. + // + // startedAt is REASSIGNED, not read-only (T331 review finding F4b): it names WHICHEVER call + // is currently in flight, mirroring userPrompt's own reassignment below — RunTruthGateLadderAsync + // moves it to a re-ask's own dispatch instant the moment that call actually fires, so the + // catch-all far below (and this render's OWN success recording, if the ladder never fires at + // all) both always attribute a fault/success to the call that actually produced it, never to + // an earlier call's timing. var startedAt = timeProvider.GetUtcNow(); var mode = degradationMode.CurrentMode; // gh-#429: the SAME card-first-then-legacy-row precedence the prompt's own self-name-mention @@ -602,8 +642,12 @@ async Task RequestCleanedCompletionAsync( // SystemRandomSource already standardize on. The builder enforces the persona gate // itself: with no persona section there is no line, however the roll lands. var mentionOwnName = Random.Shared.NextDouble() < LlmPromptBuilder.SelfNameMentionProbability; + // Captured once (SPEC F138.5, PLAN T331): BuildSystemPrompt's own clock guard line and + // BuildUserContent's station clock line below must provably read the SAME instant — the + // identical discipline CopyClaims.CheckClock's own remarks already require of its caller. + var stationLocalNow = StationLocalNow(); systemPrompt = LlmPromptBuilder.BuildSystemPrompt( - LlmPromptBuilder.BuildPersonaSection(persona, card, mentionOwnName), cfg.MaxCopyChars); + LlmPromptBuilder.BuildPersonaSection(persona, card, mentionOwnName), cfg.MaxCopyChars, stationLocalNow); // Read HERE — after WaitAsync above, i.e. already inside the single-flight critical // section — not by the caller before this method was ever invoked (SPEC F83.1, T65 @@ -620,7 +664,7 @@ async Task RequestCleanedCompletionAsync( // patterFact above was null (context wins) — BuildUserContent enforces that structurally // too (its own defense-in-depth `?? ` fallback), so this call passes both through as-is. userPrompt = LlmPromptBuilder.BuildUserContent( - request, LlmPromptBuilder.BuildStationClockLine(StationLocalNow()), previouslyVoicedTasteNotes, + request, LlmPromptBuilder.BuildStationClockLine(stationLocalNow), previouslyVoicedTasteNotes, patterFact?.Fact, showFlavorFact); // No boot-frozen BaseAddress (F36.2) — the endpoint is read from CurrentValue above and an @@ -629,33 +673,7 @@ async Task RequestCleanedCompletionAsync( // so a live PUT to Llm:Endpoint applies on the next render. var requestUri = EndpointUri.Combine(cfg.Endpoint, "/v1/chat/completions"); - var body = new - { - model = cfg.Model, - messages = new object[] - { - new { role = "system", content = systemPrompt }, - new { role = "user", content = userPrompt }, - }, - max_tokens = DeriveMaxTokens(cfg.MaxCopyChars), - }; - - using var httpRequest = new HttpRequestMessage(HttpMethod.Post, requestUri) - { - Content = JsonContent.Create(body), - }; - - // Bearer header rides only when an ApiKey is configured (env-only, F19.3/F34.3). - if (!string.IsNullOrEmpty(cfg.ApiKey)) - { - httpRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", cfg.ApiKey); - } - - var response = await http.SendAsync(httpRequest, timeoutCts.Token); - response.EnsureSuccessStatusCode(); // throws HttpRequestException on non-2xx - - var payload = await response.Content.ReadFromJsonAsync(timeoutCts.Token); - var text = payload?.Choices?.FirstOrDefault()?.Message?.Content ?? string.Empty; + var text = await PostCompletionAsync(http, requestUri, cfg, systemPrompt, userPrompt, timeoutCts.Token); // Written HERE, STILL inside the single-flight critical section (SPEC F83.1, T65 review // finding) — only for an on-air call (updateTasteMemory), and only once the call has @@ -671,31 +689,159 @@ async Task RequestCleanedCompletionAsync( // point (SPEC F123.2-F123.4, STORY-319, PLAN T263), so a trim is visible to the ring as // its own outcome instead of only discoverable by re-reading Response after the fact. var cleanup = CleanCopy(text, cfg.MaxCopyChars); - if (cleanup is LlmCopyCleanupResult.Trimmed trimmed) + LogIfTrimmed(request, personaName, cleanup); + + // SPEC F138.2/F138.3/F138.4 (STORY-350, STORY-351, PLAN T331/T332) — the truth-gate + // stage, covering every LLM-authored kind, not the context lane alone. Composed as ONE + // check (CheckTruthGate below), never two chained ladder runs — the T331 reviewer ruling + // RunTruthGateLadderAsync's own remarks restate: a second ladder invocation could never + // see the first's own re-ask reply, so a ContextSegment render facing BOTH a fact + // violation and a clock violation must clear both in the SAME gate/re-ask cycle. factBlock + // is non-null ONLY for a ContextSegment render carrying a non-empty fact block (F138.2's + // own "never even ask" discipline for the FACTS half specifically — see below); the clock + // half of CheckTruthGate always runs for every kind reaching this point, ContextSegment + // included (F138.3's own "all patter kinds": LeadIn, BackAnnounce, and — since they + // resolve through this exact same seam — SignOff/SignOn get it for free, no separate + // wiring needed). + // + // No Kind-based whole-gate carve-out for a factless ContextSegment (review round-2 + // finding F1, PLAN T332 — an earlier revision of this method skipped the WHOLE gate, + // clock half included, for that case; deleted): CheckTruthGate's own factBlock is-null + // guard already keeps F138.2's "never even ask" scoped to the FACTS half alone, so + // deleting the second, Kind-based gate here is strictly less code covering strictly more + // spec, not a behavior loss. A factless ContextSegment is structurally unreachable on the + // AIR path — Orchestrator.BuildContextSegmentRequestAsync's own blank-facts guard + // (Orchestrator.cs, "no segment facts (SPEC F107.6)") never builds a ContextSegment + // SegmentRequest without one — but IS reachable from GenWave.Host.Api.PersonaController.Preview + // (TryParseKind accepts any SegmentKind name, and a preview never supplies ContextFacts at + // all), so a factless ContextSegment preview now gets the clock half exactly like every + // other kind, rather than silently skipping it. Also gated on TextOf(cleanup): hygiene + // already rejecting the reply outright (empty, or over-length with nothing salvageable) + // falls straight through to the unchanged switch/record below — there is no candidate text + // to check claims against, and the existing OverLength/EmptyCompletion rung already covers + // that case correctly. + var factBlock = request.Kind == SegmentKind.ContextSegment + && request.ContextFacts is { } contextFacts && !string.IsNullOrWhiteSpace(contextFacts) + ? contextFacts + : null; + + if (TextOf(cleanup) is { } candidate) { - // One INFORMATION line (F123.4) — a trim is discipline, not an outage, so it gets its - // own quiet lane rather than promoting to LogFailure's WARN. - logger.LogInformation( - "LLM copy for {Kind} trimmed to the last complete sentence under Llm:MaxCopyChars " + - "(persona: {PersonaName}): {CharsBefore} -> {CharsAfter} chars", - request.Kind, (personaName ?? "none").ReplaceLineEndings(" "), trimmed.CharsBeforeTrim, - trimmed.Text.Length); + var ladderResult = await RunTruthGateLadderAsync( + candidateText => CheckTruthGate(candidateText, factBlock, stationLocalNow, request.Track?.Title), + LlmPromptBuilder.BuildTruthGateReaskLine, candidate, text); + if (ladderResult is not null) + return ladderResult; } // Ok still records the RAW reply (SPEC F73.1) regardless of outcome — a full reject // (empty/no-sentence-fits) stays Ok exactly as before T263 (a hygiene decision the caller // makes, not a fact about whether the call itself succeeded; see LlmCallOutcome.Ok's own // remarks); a trim gets its own finer-grained outcome instead (LlmCallOutcome.Trimmed). - var ringOutcome = cleanup switch - { - LlmCopyCleanupResult.Trimmed => LlmCallOutcome.Trimmed, - LlmCopyCleanupResult.Fits or LlmCopyCleanupResult.Rejected => LlmCallOutcome.Ok, - _ => throw new UnreachableException($"Unhandled {nameof(LlmCopyCleanupResult)} case."), - }; - callRing.Record( + // + // SPEC F139.1 (STORY-353, PLAN T330): the additive Cause field splits the SAME three + // shapes finer still — a Trimmed salvage still aired, so it is Success exactly like an + // exact Fits; a full Rejected splits on WHY nothing survived (LlmCopyCleanupResult.Rejected's + // own WasOverLength, decided once at CleanCopy, never re-derived here). + var (ringOutcome, cause) = ClassifyCleanup(cleanup); + recorder.Record( personaName, systemPrompt, userPrompt, text, startedAt, ElapsedMs(startedAt), - ringOutcome, statusDetail: null, mode); + ringOutcome, statusDetail: null, mode, cause, cfg.Model); return cleanup; + + // SPEC F138.4 (STORY-350, PLAN T331) — one truth-gate ladder run: gate + // firstCandidate against check, and on a violation, exactly ONE re-ask (its added + // prompt line built by buildReaskLine), then reclassify. A LOCAL function (T331 review + // finding — this method was pushing 190 lines with a SECOND ladder, PLAN T332's clock + // check, landing at this exact seam next) — not a private instance method — because it + // needs to REASSIGN userPrompt/startedAt and read http/requestUri/timeoutCts exactly the + // way the enclosing method already does (review finding F4b: the SAME reassign-not-shadow + // discipline userPrompt already followed for its own prompt text, now shared by startedAt + // too, so a fault raised by the re-ask's own call is attributed — prompt AND timing alike + // — to that call, never to the first). Returns null when check passed firstCandidate + // outright — the caller's own signal to fall through to its unchanged, non-gated + // classify/record path above; a non-null return is always this ladder's OWN final word, + // and the caller records nothing further. + // + // PLAN T332 folds facts and clock into ONE call to this same ladder rather than a second + // one (CheckTruthGate, the caller's own composite check, below) — never two chained + // ladder runs: a second RunTruthGateLadderAsync invocation could never see the first's + // own re-ask reply, so it could only ever check the ORIGINAL candidate a second time, + // leaving whatever the first ladder's own re-ask actually said unchecked by the second + // check entirely. Composing the two checks into one function, called once, is what keeps + // this a single gate/re-ask/reclassify cycle regardless of how many claim classes it + // covers. + // + // Plain // comments, not /// (T331 review finding): a local function's XML doc comment + // never renders anywhere doc-gen actually reads it today, and would flag CS1587 the day + // a future doc-gen pass enables XML output for this project. + async Task RunTruthGateLadderAsync( + Func check, Func, string> buildReaskLine, + string firstCandidate, string firstRawText) + { + var gateResult = check(firstCandidate); + if (gateResult.Passed) + return null; + + // The rejected first call gets its own honest ring entry (SPEC F138.4: each call in + // the ladder is its own call with its own entry) BEFORE the re-ask fires — never + // silently folded into whichever entry the re-ask itself produces. + recorder.Record( + personaName, systemPrompt, userPrompt, firstRawText, startedAt, ElapsedMs(startedAt), + LlmCallOutcome.Ok, statusDetail: null, mode, LlmCallCause.TruthGateReject, cfg.Model); + + // Exactly ONE re-ask (F138.4), naming the violation. userPrompt AND startedAt are both + // REASSIGNED (never shadowed by a new local) so a fault raised by THIS call — timeout, + // non-2xx, connect — is attributed by this method's own catch-all below to the + // re-ask's own prompt and its own start time, and degrades through the EXACT SAME path + // an ordinary single-call failure already does: no new exception handling, no new + // "longer hold". Reusing timeoutCts.Token (not a fresh CancelAfter) is what bounds the + // whole ladder to this render's existing Llm:TimeoutSeconds budget — whatever the + // first call already spent is gone, and a budget that expires mid-reask throws + // OperationCanceledException exactly as it always would have for one call. + userPrompt = $"{userPrompt}\n{buildReaskLine(gateResult.Violations)}"; + startedAt = timeProvider.GetUtcNow(); + var reaskText = await PostCompletionAsync(http, requestUri, cfg, systemPrompt, userPrompt, timeoutCts.Token); + var reaskCleanup = CleanCopy(reaskText, cfg.MaxCopyChars); + LogIfTrimmed(request, personaName, reaskCleanup); + + // The tri-state (no candidate to even check / checked-and-failed / checked-and-passed) + // folds to ONE clear bool (T331 review finding) instead of re-deriving + // "is { Passed: false }" at both the classify site and the final-return site below: + // reaskViolations is non-null exactly when the gate was actually checked AND failed. + var reaskGateResult = TextOf(reaskCleanup) is { } reaskCandidate ? check(reaskCandidate) : null; + var reaskViolations = reaskGateResult is { Passed: false } failed ? failed.Violations : null; + + var (reaskOutcome, reaskCause) = reaskViolations is null + ? ClassifyCleanup(reaskCleanup) + : (LlmCallOutcome.Ok, LlmCallCause.TruthGateReject); + recorder.Record( + personaName, systemPrompt, userPrompt, reaskText, startedAt, ElapsedMs(startedAt), + reaskOutcome, statusDetail: null, mode, reaskCause, cfg.Model); + + // F138.4's floor: a still-violating re-ask never airs. A TruthGateRejected here is + // what makes TextOf(...) null for it (T331 review finding F3 — its own distinct shape + // from a hygiene Rejected, so WriteAsync's own failure WARN can name the real cause), + // so the caller (WriteAsync/WritePreviewAsync) degrades it exactly like any other + // reject, straight into fallback.WriteAsync — an EXISTING rung, never a new one + // invented here. WHERE that rung actually lands differs by kind, unchanged by this + // ladder (PLAN T332 investigation): ContextSegment/SignOff/SignOn never reach air even + // as template copy — TtsSegmentSource's own non-fresh-copy guard drops the render + // outright (F92.4/F92.5/F107.6's skip-never-silence posture); LeadIn/BackAnnounce carry + // no such guard, so PatterTemplateRenderer's deterministic template airs instead — its + // FIXED PROSE never states a weekday/daypart on its own (see + // PatterTemplateRenderer.Expand's own LeadIn/BackAnnounce arms; it does interpolate the + // track's own title/artist verbatim, and a title CAN legally name a day — "Saturday + // Night Fever" — but that interpolated text is never gate-checked, since a template + // render never passes through this ladder at all), the same F92-era floor a hygiene + // reject already degraded those two kinds to before this gate ever existed. A re-ask + // that passed hygiene but still failed the gate is the only case needing a synthetic + // TruthGateRejected instead of reaskCleanup itself: reaskCleanup there still carries + // real (unusable) text that must never reach TextOf. + return reaskViolations is { } violations + ? new LlmCopyCleanupResult.TruthGateRejected(violations) + : reaskCleanup; + } } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -706,10 +852,14 @@ async Task RequestCleanedCompletionAsync( } catch (Exception ex) { - var (outcome, detail) = ClassifyForRing(ex); - callRing.Record( + // userPrompt/startedAt name WHICHEVER call actually faulted (T331 review finding F4b): a + // fault raised by RunTruthGateLadderAsync's own re-ask is attributed to the re-ask's own + // prompt and dispatch instant, never the first call's, because both were REASSIGNED (not + // shadowed) the moment that call fired — see this method's own startedAt remarks above. + var (outcome, cause, detail) = ClassifyForRing(ex); + recorder.Record( personaName, systemPrompt, userPrompt, response: null, startedAt, ElapsedMs(startedAt), - outcome, detail, mode); + outcome, detail, mode, cause, cfg.Model); throw; } finally @@ -718,6 +868,73 @@ async Task RequestCleanedCompletionAsync( } } + /// + /// The composite truth-gate check gates on (SPEC F138.2, + /// F138.3, STORY-350, STORY-351, PLAN T332) — kept beside the ladder call site above, deliberately + /// small and pure: a straight union of 's violations (only when + /// is non-null — the context lane's own fact-block claim classes, + /// meaningless for any kind with no fact block to check against) with + /// 's violations (unconditional — F138.3's own "all patter + /// kinds"), into ONE . This is the whole reason a ContextSegment + /// render can clear both claim families in a SINGLE gate/re-ask cycle instead of two chained ladder + /// runs (the reviewer-ruled constraint 's own remarks + /// restate): the ladder only ever sees one , never two, so it has no + /// way to run twice even by accident. Both source checks stay 's own pure + /// static functions of their arguments; this method adds no logic beyond composing their two + /// violation lists. + /// + static ClaimCheckResult CheckTruthGate(string copy, string? factBlock, DateTimeOffset stationLocalNow, string? trackTitle) + { + var violations = new List(); + if (factBlock is not null) + violations.AddRange(CopyClaims.CheckFacts(copy, factBlock).Violations); + + violations.AddRange(CopyClaims.CheckClock(copy, stationLocalNow, trackTitle).Violations); + return new ClaimCheckResult(violations); + } + + /// + /// Posts one chat-completion request and returns the raw reply text (SPEC F34.3, F123.1) — the + /// exact wire call both the first completion and the F138.4 re-ask fire (STORY-350, PLAN T331), + /// extracted so the ladder's second call is provably the SAME request shape as the first rather + /// than a hand-maintained second copy of the body/header/parse logic. is + /// always timeoutCts.Token from the one caller () + /// — this method holds no state and starts no clock of its own, so a re-ask sharing that same + /// token shares that render's existing budget rather than getting a fresh one (F138.4's "never a + /// longer feeder hold"). + /// + static async Task PostCompletionAsync( + HttpClient http, Uri requestUri, LlmOptions cfg, string systemPrompt, string userPrompt, CancellationToken ct) + { + var body = new + { + model = cfg.Model, + messages = new object[] + { + new { role = "system", content = systemPrompt }, + new { role = "user", content = userPrompt }, + }, + max_tokens = DeriveMaxTokens(cfg.MaxCopyChars), + }; + + using var httpRequest = new HttpRequestMessage(HttpMethod.Post, requestUri) + { + Content = JsonContent.Create(body), + }; + + // Bearer header rides only when an ApiKey is configured (env-only, F19.3/F34.3). + if (!string.IsNullOrEmpty(cfg.ApiKey)) + { + httpRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", cfg.ApiKey); + } + + var response = await http.SendAsync(httpRequest, ct); + response.EnsureSuccessStatusCode(); // throws HttpRequestException on non-2xx + + var payload = await response.Content.ReadFromJsonAsync(ct); + return payload?.Choices?.FirstOrDefault()?.Message?.Content ?? string.Empty; + } + long ElapsedMs(DateTimeOffset startedAt) => (long)(timeProvider.GetUtcNow() - startedAt).TotalMilliseconds; // gh-#117 — the ONE place this writer resolves "station-local now" for the prompt's clock @@ -728,20 +945,24 @@ DateTimeOffset StationLocalNow() => stationClock?.LocalNow ?? TimeZoneInfo.ConvertTime(timeProvider.GetUtcNow(), timeProvider.LocalTimeZone); /// - /// Classifies a completion fault for (SPEC F73.1): the ONE other + /// Classifies a completion fault for (SPEC F73.1, F139.1): the ONE other /// source reaching this catch-all (the caller's own /// cancellation is already filtered out by the clause above) is RequestCleanedCompletionAsync's - /// own timeoutCts firing — , distinct from a generic - /// . Deliberately independent of 's - /// own detail switch (SPEC F69.7) — that one feeds a WARN line and has no need to split - /// out timeout, so duplicating this small a classification is simpler than threading a shared - /// helper through two call sites with different needs. + /// own timeoutCts firing — /, + /// distinct from a generic /. + /// The F139 taxonomy has no finer split for "a response arrived but was non-2xx" versus "no + /// response ever arrived at all" — both land on here, + /// same as they already share . Deliberately independent of + /// 's own detail switch (SPEC F69.7) — that one feeds a WARN line + /// and has no need to split out timeout, so duplicating this small a classification is simpler + /// than threading a shared helper through two call sites with different needs. /// - internal static (LlmCallOutcome Outcome, string Detail) ClassifyForRing(Exception ex) => ex switch + internal static (LlmCallOutcome Outcome, LlmCallCause Cause, string Detail) ClassifyForRing(Exception ex) => ex switch { - OperationCanceledException => (LlmCallOutcome.Timeout, "Llm:TimeoutSeconds exceeded"), - HttpRequestException { StatusCode: { } status } => (LlmCallOutcome.Failed, $"HTTP {(int)status}"), - _ => (LlmCallOutcome.Failed, ex.GetType().Name), + OperationCanceledException => (LlmCallOutcome.Timeout, LlmCallCause.Timeout, "Llm:TimeoutSeconds exceeded"), + HttpRequestException { StatusCode: { } status } => + (LlmCallOutcome.Failed, LlmCallCause.ConnectionFailure, $"HTTP {(int)status}"), + _ => (LlmCallOutcome.Failed, LlmCallCause.ConnectionFailure, ex.GetType().Name), }; /// @@ -765,8 +986,9 @@ internal static int DeriveMaxTokens(int maxCopyChars) => /// Extracts the airable/previewable text from a (SPEC /// F123.2): an exact fit and a salvaged trim both hand back real copy — the caller cannot tell /// (and does not need to) which one it got, since both are genuinely LLM-authored — and only a - /// full reject hands back null, so the caller degrades exactly as it did before T263. Shared by - /// and so the two can never read the + /// full reject (hygiene's own, or the F138.4 ladder's + /// floor, PLAN T331) hands back null, so the caller degrades exactly as it did before T263. Shared + /// by and so the two can never read the /// closed hierarchy differently. /// static string? TextOf(LlmCopyCleanupResult cleanup) => cleanup switch @@ -774,6 +996,81 @@ internal static int DeriveMaxTokens(int maxCopyChars) => LlmCopyCleanupResult.Fits fits => fits.Text, LlmCopyCleanupResult.Trimmed trimmed => trimmed.Text, LlmCopyCleanupResult.Rejected => null, + LlmCopyCleanupResult.TruthGateRejected => null, + _ => throw new UnreachableException($"Unhandled {nameof(LlmCopyCleanupResult)} case."), + }; + + /// + /// Names the real cause of a null result for 's own + /// failure WARN (SPEC F69.7, T331 review finding F3, generalized PLAN T332): a hygiene reject + /// splits on exactly as it always has, + /// but a floor gets its OWN sentence naming + /// the truth gate and every still-violating claim — never the hygiene wording ("empty or exceeded + /// Llm:MaxCopyChars after cleanup"), which sends an operator at the wrong levers (endpoint, + /// max_tokens) for a failure neither lever can fix. + /// + /// + /// No longer names "the context fact gate" specifically (T331 pickup, PLAN T332 — the wording was + /// hardcoded for STORY-350's single check before STORY-351's clock check reused this same floor): + /// renders each violation HONESTLY per its own class — + /// true (a weekday) reads "wrong-day claim", true (a + /// daypart) reads "wrong-time-of-day claim"; false (a fact-block violation) reads "unsupported + /// claim" — so a clock rejection is never misreported as a fact-block one or vice versa, whichever + /// kind's ladder floor produced it. + /// + /// + /// Comma-free, sentence-fragment style (matches every other reason) — + /// this never reaches prompt text, only a log line, but stays + /// safe to interpolate directly regardless (that type's own remarks: provably digit-shaped or + /// closed-vocabulary). + /// + static string DescribeNullTextReason(LlmCopyCleanupResult cleanup) => cleanup switch + { + LlmCopyCleanupResult.Rejected { WasOverLength: true } => "exceeded Llm:MaxCopyChars after cleanup", + LlmCopyCleanupResult.Rejected { WasOverLength: false } => "empty after cleanup", + LlmCopyCleanupResult.TruthGateRejected truthGate => + "the truth gate rejected the re-ask too (" + + $"{string.Join(" and ", truthGate.Violations.Select(DescribeViolationForLog).Distinct(StringComparer.OrdinalIgnoreCase))})", + _ => throw new UnreachableException( + $"{nameof(TextOf)} already returns non-null text for any other {nameof(LlmCopyCleanupResult)} case."), + }; + + /// + /// One violation's own honest fragment for (SPEC F138.2, + /// F138.3, PLAN T332): is the ONE discriminator this + /// method (and , one module over) keys the + /// facts-vs-clock split on — see that property's own remarks. Within the clock branch, + /// 's own still separates the two clock claim + /// shapes: a wrong weekday reads "wrong-day claim", a wrong daypart reads "wrong-time-of-day claim". + /// + static string DescribeViolationForLog(ClaimViolation violation) => (violation.IsClockClaim, violation.Class) switch + { + (true, ClaimClass.Daypart) => $"wrong-time-of-day claim: {violation.Token}", + (true, _) => $"wrong-day claim: {violation.Token}", + (false, _) => $"unsupported claim: {violation.Token}", + }; + + /// + /// Maps a to its ring outcome/cause pair (SPEC F73.1, F139.1) + /// — extracted (STORY-350, PLAN T331) so 's ordinary + /// success path and its F138.4 re-ask path share ONE classification rather than two hand-kept + /// copies of the same switch. Ok still records the RAW reply regardless of outcome — a full + /// reject (empty/no-sentence-fits) stays Ok exactly as before T263 (a hygiene decision the caller + /// makes, not a fact about whether the call itself succeeded; see 's + /// own remarks); a trim gets its own finer-grained outcome instead (). + /// The additive Cause field (STORY-353, PLAN T330) splits the SAME three shapes finer still — a + /// Trimmed salvage still aired, so it is Success exactly like an exact Fits; a full Rejected + /// splits on WHY nothing survived (, + /// decided once at , never re-derived here). Never called for a cleanup the + /// F138.2 truth gate already rejected — that path stamps + /// itself, bypassing this map entirely (see that call site's own remarks). + /// + static (LlmCallOutcome Outcome, LlmCallCause Cause) ClassifyCleanup(LlmCopyCleanupResult cleanup) => cleanup switch + { + LlmCopyCleanupResult.Trimmed => (LlmCallOutcome.Trimmed, LlmCallCause.Success), + LlmCopyCleanupResult.Fits => (LlmCallOutcome.Ok, LlmCallCause.Success), + LlmCopyCleanupResult.Rejected { WasOverLength: true } => (LlmCallOutcome.Ok, LlmCallCause.OverLength), + LlmCopyCleanupResult.Rejected { WasOverLength: false } => (LlmCallOutcome.Ok, LlmCallCause.EmptyCompletion), _ => throw new UnreachableException($"Unhandled {nameof(LlmCopyCleanupResult)} case."), }; @@ -815,14 +1112,14 @@ static LlmCopyCleanupResult CleanCopy(string raw, int maxChars) var text = ApplyCopyHygiene(raw); if (text.Length == 0) - return new LlmCopyCleanupResult.Rejected(); + return new LlmCopyCleanupResult.Rejected(WasOverLength: false); if (text.Length <= maxChars) return new LlmCopyCleanupResult.Fits(text); var salvaged = TrimToLastCompleteSentence(text, maxChars); return salvaged is null - ? new LlmCopyCleanupResult.Rejected() + ? new LlmCopyCleanupResult.Rejected(WasOverLength: true) : new LlmCopyCleanupResult.Trimmed(salvaged, CharsBeforeTrim: text.Length); } diff --git a/src/GenWave.Tts/LlmPromptBuilder.cs b/src/GenWave.Tts/LlmPromptBuilder.cs index 01716807..4a80e0db 100644 --- a/src/GenWave.Tts/LlmPromptBuilder.cs +++ b/src/GenWave.Tts/LlmPromptBuilder.cs @@ -68,8 +68,25 @@ static class LlmPromptBuilder /// the length instruction with a numeric word figure (see ): /// stated, not enforced — the model reads this as a target, while T262's max_tokens cap and /// T263's sentence-trim salvage are what actually bound and clean up the reply. + /// + /// + /// (SPEC F138.5, STORY-350, PLAN T331) appends the + /// anti-fabrication clock guard line (see ) when supplied — the + /// guard line itself is pinned by its own dedicated fact + /// (Story350_ContextFactGate.ScenarioTheLadderDegrades.The_guard_line_rides_the_prompt), + /// which DOES supply a clock. Optional, default — NOT because the guard + /// line is optional in production (LlmCopyWriter.RequestCleanedCompletionAsync, the one + /// production call site, always supplies it, so every prompt that actually reaches the model + /// carries the line): a REQUIRED parameter would fail every pre-F138 spec call site that already + /// constructs this prompt with only personaSection/maxCopyChars (Issue151/152/188/303) + /// to COMPILE, since none of them supply a third argument today. (Amended T331 review round 1: the + /// earlier "stays byte-identical" framing here was false — those specs pin SUBSTRINGS via + /// Assert.Contains, not the whole prompt, so an appended trailing guard line would not have + /// broken them at runtime either way; the true, sole reason this stays optional is the compile-time + /// one above.) + /// /// - public static string BuildSystemPrompt(string? personaSection, int maxCopyChars) + public static string BuildSystemPrompt(string? personaSection, int maxCopyChars, DateTimeOffset? stationLocalNow = null) { // gh-#152: "personality-neutral" and a persona section's "Style: bubbly, energetic, // expressive" cancelled each other inside the SAME prompt. The neutral framing now applies @@ -123,11 +140,97 @@ public static string BuildSystemPrompt(string? personaSection, int maxCopyChars) "they/them/their unless the provided metadata explicitly states pronouns - never infer " + "gender from a name."; + // SPEC F138.5: appended only when a clock was supplied — see this method's own remarks on + // why the parameter is optional. String-concatenated onto scaffoldBody (not a separate + // paragraph) so it reads as one more scaffold instruction, exactly like the sentences before it. + if (stationLocalNow is { } now) + scaffoldBody += " " + BuildClockGuardLine(now); + return string.IsNullOrEmpty(personaSection) ? $"{NeutralOpening} {scaffoldBody}" : $"{PersonaOpening} {scaffoldBody}\n\n{personaSection}"; } + /// + /// SPEC F138.5 (STORY-350, PLAN T331) — the anti-fabrication clock guard line every patter + /// prompt now carries verbatim (weekday/daypart substituted): "It is {weekday} {daypart}. Never + /// name another day or time of day." Comma-free (the gh-#303 style lesson — prompt text is style + /// the model imitates, so a guard line leaning on commas would argue against itself). + /// + /// + /// is the SAME instant + /// renders for this render (the T329 precedent already + /// established for the checker side of this same clock: prompt and any future check must + /// provably read one shared instant, never two separately-computed ones). The weekday spelling + /// is 's own ToString() (e.g. "Saturday" — mirrors + /// 's own documented spelling), and the daypart word is + /// 's single canonical category for the hour — + /// never 's overlapping window set, since a guard + /// line states ONE daypart to hold to, not every window the hour happens to satisfy. + /// + /// + public static string BuildClockGuardLine(DateTimeOffset stationLocalNow) => + $"It is {stationLocalNow.DayOfWeek} {ClaimVocabulary.CategoryForHour(stationLocalNow.Hour)}. " + + "Never name another day or time of day."; + + /// + /// SPEC F138.4 (STORY-350/351, PLAN T331/T332) — the truth-gate ladder's own re-ask line: names + /// every claim and/or + /// rejected so the retry has something concrete to fix rather than a bare "try again". Generalized + /// (PLAN T332 — the original wording named "the facts above" unconditionally, which is wrong for + /// a LeadIn/BackAnnounce/SignOff/SignOn re-ask: those prompts carry no fact block at all): each + /// violation renders its OWN honest clause (see ) — a + /// fact-block claim states it was never in the facts, a clock claim states the correct + /// weekday/daypart by name — so a ContextSegment re-ask facing BOTH claim families at once (the + /// composite check, LlmCopyWriter.CheckTruthGate) still gets ONE re-ask line naming both, + /// never a line that misdescribes a clock claim as a missing fact or vice versa. Comma-free (the + /// gh-#303 style lesson, same as every other prompt line in this file) — multiple violations join + /// on " and " rather than a comma-delimited list. A violation's own + /// (and, for a clock claim, its own ) is safe to interpolate + /// directly without further escaping ('s own remarks: provably + /// digit-shaped or closed-vocabulary, never free text reachable from a fact block or copy). + /// + /// appends this line to the SAME user + /// prompt the rejected completion already saw — never a prompt rebuilt from scratch — so the + /// re-ask still carries every other instruction (the facts block when there is one, the F138.5 + /// clock guard line, segment framing, taste color) the original completion had; this method only + /// renders the one added line. + /// + /// + /// Deliberately opens with "Your last reply..." rather than a literal "Re-ask:" label (T331 review + /// advisory F5): a machine-looking prefix like that is exactly the kind of thing a model can echo + /// back verbatim into its own reply, and has no rule + /// that would ever strip it (that method's own gate is for a MODEL-authored preamble "Here's your + /// copy:", not an operator-authored one riding in the prompt itself) — plain declarative English + /// carries the same instruction with nothing label-shaped to leak. + /// + /// + public static string BuildTruthGateReaskLine(IReadOnlyList violations) + { + var claims = string.Join( + " and ", violations.Select(DescribeViolationForReask).Distinct(StringComparer.OrdinalIgnoreCase)); + + // "the above" (T332 review round-2 advisory), not "every one of those": the closing sentence + // must read naturally whether claims names ONE violation or several — "fixes every one of + // those" reads as a grammatical stumble for a single claim ("every one" implies more than + // one), while "the above" refers to whatever was just stated regardless of count. + return $"Your last reply got this wrong: {claims}. " + + "Write a new reply that corrects the above and adds nothing else unsupported."; + } + + /// + /// One violation's own honest re-ask clause (SPEC F138.4, PLAN T332): keys on + /// (T332 review round-2 finding — the SAME single + /// discriminator LlmCopyWriter.DescribeViolationForLog keys its own facts-vs-clock split + /// on, one module over) rather than re-testing 's own + /// nullability independently here. A clock violation states the correct weekday/daypart by name, + /// since the model has something concrete to correct TO; a fact-block violation states only that + /// the claim was never in the facts, since there is no single "correct" fix. + /// + static string DescribeViolationForReask(ClaimViolation violation) => violation.IsClockClaim + ? $"you said \"{violation.Token}\" but it is actually {violation.Expected}" + : $"you said \"{violation.Token}\" but that was never stated"; + /// /// gh-#150 — how often a persona-voiced break is asked to work the DJ's own name in. Real /// radio DJs occasionally say their own name; roughly one break in seven keeps it a habit, diff --git a/src/GenWave.Tts/TtsSegmentSource.cs b/src/GenWave.Tts/TtsSegmentSource.cs index 847db887..63134fd8 100644 --- a/src/GenWave.Tts/TtsSegmentSource.cs +++ b/src/GenWave.Tts/TtsSegmentSource.cs @@ -84,16 +84,23 @@ public sealed class TtsSegmentSource( // segment, the alternative is PatterTemplateRenderer's inert placeholder ("Here's // something worth knowing") standing in for actual facts, which defeats the entire point // of a context provider (never airable filler, SPEC F107.6). copy.FreshPerAiring false - // here means every writer in the chain missed: LlmCopyWriter's own three degrade paths - // (disabled endpoint, timeout/non-2xx/connect, empty-or-over-length after cleanup) AND - // DegradationGatedCopyWriter routing straight to TemplateCopyWriter — unconditionally in - // Hard mode, or off an unclaimed Soft cadence slot — bypassing LlmCopyWriter entirely. + // here means every writer in the chain missed: LlmCopyWriter's own FOUR degrade paths + // (disabled endpoint, timeout/non-2xx/connect, empty-or-over-length after cleanup, and — + // as of PLAN T332, SPEC F138.2-F138.4 — the truth-gate ladder exhausting its one re-ask) + // AND DegradationGatedCopyWriter routing straight to TemplateCopyWriter — unconditionally + // in Hard mode, or off an unclaimed Soft cadence slot — bypassing LlmCopyWriter entirely. // Every one of those returns template copy rather than throwing (ISegmentCopyWriter's own // never-throws contract), which is exactly why PatterTemplateRenderer still needs correct - // SignOff/SignOn/ContextSegment arms — they just must never reach air. One WARN, then - // null: ITtsSegmentSource already allows null-never-throws, and the Orchestrator's own - // drain arm treats a null render exactly like F92.4's "whichever piece rendered airs - // (else clean cut)"/F107.6's skip-never-silence posture. + // SignOff/SignOn/ContextSegment arms — they just must never reach air for these three + // kinds specifically (LeadIn/BackAnnounce carry no such guard below — their own template + // rung DOES reach air on a miss, truth-gate exhaustion included, since neither kind's + // FIXED PROSE states a weekday/daypart claim on its own; see PatterTemplateRenderer.Expand's + // own LeadIn/BackAnnounce arms — the interpolated track title/artist is the one part of + // that text NOT gate-checked either way, since a template render never reaches this ladder + // to begin with). One WARN, then null: + // ITtsSegmentSource already allows null-never-throws, and the Orchestrator's own drain arm + // treats a null render exactly like F92.4's "whichever piece rendered airs (else clean + // cut)"/F107.6's skip-never-silence posture. if (request.Kind is SegmentKind.SignOff or SegmentKind.SignOn or SegmentKind.ContextSegment && !copy.FreshPerAiring) { diff --git a/src/GenWave.Tts/TtsServiceCollectionExtensions.cs b/src/GenWave.Tts/TtsServiceCollectionExtensions.cs index 23583172..4a237bbb 100644 --- a/src/GenWave.Tts/TtsServiceCollectionExtensions.cs +++ b/src/GenWave.Tts/TtsServiceCollectionExtensions.cs @@ -236,6 +236,18 @@ public static IServiceCollection AddGenWaveTts(this IServiceCollection services, // ring — GET /api/llm-calls (GenWave.Host) reads the SAME singleton LlmCopyWriter // records into. No persistence dependency of any kind (F73.3) — see its own remarks. .AddSingleton() + // LlmCallCauseCounters (SPEC F139.2, STORY-353, PLAN T330): the rolling 24h cause + // counters, DELIBERATELY a separate singleton from LlmCallRing immediately above rather + // than composed inside it — see that class's own remarks for why (LlmCallRing's F73.3 + // structural proof pins its constructor to exactly one parameter). + .AddSingleton() + // LlmCallRecorder (SPEC F139.1/F139.2, STORY-353, PLAN T330 review finding F2): the ONE + // Record call site LlmCopyWriter/CrosstalkScriptWriter (both here) and CrosstalkStockWorker + // (GenWave.Host, resolved directly) now depend on as a REQUIRED constructor param, in place + // of taking LlmCallRing/LlmCallCauseCounters separately — see that class's own remarks for + // why folding the two writes into one call site closes the "delete only the counter half" + // mutation gap the pre-recorder duplication left open. + .AddSingleton() // LlmCopyWriter also consumes IActivePersonaAccessor (a host-registered seam) — // resolved per LLM render only, composing the active persona's backstory + style into // the prompt (SPEC F35.2/F35.3). Registered concretely ONCE and exposed under BOTH diff --git a/tests/GenWave.Host.Tests/Specs/Story084_StatusEndpoint.cs b/tests/GenWave.Host.Tests/Specs/Story084_StatusEndpoint.cs index eae66d5c..d6f054a9 100644 --- a/tests/GenWave.Host.Tests/Specs/Story084_StatusEndpoint.cs +++ b/tests/GenWave.Host.Tests/Specs/Story084_StatusEndpoint.cs @@ -156,6 +156,7 @@ static StatusController BuildController( stationMonitor, llmOptions, statusHolder, + new LlmCallCauseCounters(TimeProvider.System), degradationController, voiceHealthReader, new FakeActivePersonaAccessor(), diff --git a/tests/GenWave.Host.Tests/Specs/Story125_LlmStatus.cs b/tests/GenWave.Host.Tests/Specs/Story125_LlmStatus.cs index 1070e70c..d263d5ed 100644 --- a/tests/GenWave.Host.Tests/Specs/Story125_LlmStatus.cs +++ b/tests/GenWave.Host.Tests/Specs/Story125_LlmStatus.cs @@ -12,6 +12,11 @@ // break) but a runtime count of IHttpClientFactory.CreateClient calls across N real polls, proven // zero. The tile's UI half is jest (dashboard-llm-tile.spec.tsx). // +// ScenarioDominantCause (SPEC F139.2, STORY-353, PLAN T334) covers the llm aggregate's three newer +// fields the same way — StatusController constructed directly with a caller-supplied +// LlmCallCauseCounters, no live stack required. The tile's UI half for THESE fields is jest +// (health-tile-llm-cause.spec.tsx). +// // See docs/PLAN.md Epic T. using System.Net; @@ -143,7 +148,8 @@ public static class FeatureLlmStatus static StatusController BuildController( LlmOptions? llmOptions = null, LlmCopyStatusHolder? statusHolder = null, - Persona? activePersona = null) + Persona? activePersona = null, + LlmCallCauseCounters? causeCounters = null) { var resolvedLlmOptions = llmOptions ?? new LlmOptions(); var resolvedStatusHolder = statusHolder ?? new LlmCopyStatusHolder(); @@ -171,6 +177,7 @@ static StatusController BuildController( new FakeOptionsMonitor(BuildStationOptions()), llmOptionsMonitor, resolvedStatusHolder, + causeCounters ?? new LlmCallCauseCounters(TimeProvider.System), degradationController, voiceHealthReader, new FakeActivePersonaAccessor { Persona = activePersona }, @@ -247,6 +254,150 @@ public async Task LastOutcomeAndTimestampComeFromTheStatusHolder() } } + // --------------------------------------------------------------------- + // The F139.2 dominant-cause line (SPEC F139.2, STORY-353, PLAN T334) — StatusController's own + // read of LlmCallCauseCounters.DominantFailure, riding this SAME response (no new poller). + // --------------------------------------------------------------------- + + public sealed class ScenarioDominantCause + { + [Fact] + public async Task NoFailuresRecordedLeavesTheDominantCauseFieldsNull() + { + // Given a fresh, unobserved counter store (nothing recorded at all)... + var controller = BuildController( + llmOptions: new LlmOptions { Endpoint = "https://llm.example/v1", Model = "gpt-4o-mini" }, + causeCounters: new LlmCallCauseCounters(TimeProvider.System)); + + var result = await controller.Get(CancellationToken.None); + + // Then all three fields are absent-as-null — nothing to explain a tile that isn't red. + var llm = AsJson(result).GetProperty("llm"); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCause").ValueKind); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCauseCount").ValueKind); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCauseModel").ValueKind); + } + + [Fact] + public async Task RecordedFailuresNameTheHighestCountCauseAndItsModel() + { + // Given a mix of Copy-kind causes recorded within the rolling 24h window — Success is + // the OUTRIGHT numeric majority (5, vs Timeout's 2 and OverLength's 1), so this fact + // only discriminates a real Success-exclusion filter: deleting + // `&& row.Cause != LlmCallCause.Success` from DominantFailure would make THIS fact + // assert "success"/5, not "timeout"/2 — a same-count arrangement (review finding F1) + // would pass either way and prove nothing about the filter at all. + var counters = new LlmCallCauseCounters(TimeProvider.System); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.OverLength, "gemma3:12b", LlmCallKind.Copy); + var controller = BuildController( + llmOptions: new LlmOptions { Endpoint = "https://llm.example/v1", Model = "gemma3:12b" }, + causeCounters: counters); + + var result = await controller.Get(CancellationToken.None); + + // Then the tile's own dominant-cause line names Timeout, its count, and the model it was + // recorded against — Success is never a candidate (F139.2's own "why is the tile red") + // even though it outnumbers every real failure. + var llm = AsJson(result).GetProperty("llm"); + Assert.Equal("timeout", llm.GetProperty("dominantCause").GetString()); + Assert.Equal(2, llm.GetProperty("dominantCauseCount").GetInt32()); + Assert.Equal("gemma3:12b", llm.GetProperty("dominantCauseModel").GetString()); + } + + [Fact] + public async Task AllSuccessLeavesTheDominantCauseFieldsNull() + { + // Given ONLY Success recorded, Copy kind, no failures at all — the backend half of the + // "all three travel together" contract the admin-ui tile already pins on its own side + // (health-tile-llm-cause.spec.tsx's "quiet states stay quiet" scenario). + var counters = new LlmCallCauseCounters(TimeProvider.System); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "gemma3:12b", LlmCallKind.Copy); + var controller = BuildController( + llmOptions: new LlmOptions { Endpoint = "https://llm.example/v1", Model = "gemma3:12b" }, + causeCounters: counters); + + var result = await controller.Get(CancellationToken.None); + + // Then all three fields stay null — Success alone is nothing to explain. + var llm = AsJson(result).GetProperty("llm"); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCause").ValueKind); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCauseCount").ValueKind); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCauseModel").ValueKind); + } + + [Fact] + public async Task TiedCountsBreakByCauseDeclarationOrder() + { + // Given Timeout and ConnectionFailure tied at 2 apiece, same model — LlmCallCause + // declares Timeout before ConnectionFailure (SPEC F139.1's own enum order), so a + // deterministic tie-break must always prefer Timeout, never whichever the underlying + // dictionary happens to enumerate first (review finding F2). + var counters = new LlmCallCauseCounters(TimeProvider.System); + counters.Record(LlmCallCause.ConnectionFailure, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.ConnectionFailure, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "gemma3:12b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "gemma3:12b", LlmCallKind.Copy); + var controller = BuildController( + llmOptions: new LlmOptions { Endpoint = "https://llm.example/v1", Model = "gemma3:12b" }, + causeCounters: counters); + + var result = await controller.Get(CancellationToken.None); + + var llm = AsJson(result).GetProperty("llm"); + Assert.Equal("timeout", llm.GetProperty("dominantCause").GetString()); + } + + [Fact] + public async Task TiedCountsForTheSameCauseBreakByOrdinalModelName() + { + // Given the SAME cause tied at 2 apiece across two different models — the ordinally + // FIRST model name wins ("model-a" < "model-b"), never whichever the dictionary + // happens to enumerate first (review finding F2, same method one level down). + var counters = new LlmCallCauseCounters(TimeProvider.System); + counters.Record(LlmCallCause.Timeout, "model-b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "model-b", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "model-a", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "model-a", LlmCallKind.Copy); + var controller = BuildController( + llmOptions: new LlmOptions { Endpoint = "https://llm.example/v1", Model = "model-a" }, + causeCounters: counters); + + var result = await controller.Get(CancellationToken.None); + + var llm = AsJson(result).GetProperty("llm"); + Assert.Equal("model-a", llm.GetProperty("dominantCauseModel").GetString()); + } + + [Fact] + public async Task CrosstalkOnlyFailuresNeverNameTheCopyTileSDominantCause() + { + // Given a Crosstalk-kind failure only — the tile this endpoint's llm.* block feeds + // reflects LlmCopyStatusHolder's own Copy-only verdict, never a banter miss... + var counters = new LlmCallCauseCounters(TimeProvider.System); + counters.Record(LlmCallCause.CanceledByWindow, "gemma3:12b", LlmCallKind.Crosstalk); + var controller = BuildController( + llmOptions: new LlmOptions { Endpoint = "https://llm.example/v1", Model = "gemma3:12b" }, + causeCounters: counters); + + var result = await controller.Get(CancellationToken.None); + + // Then the Copy-scoped dominant-cause fields stay null — a crosstalk-only cause never + // leaks into a line that would misname why the COPY writer's own tile went red. + var llm = AsJson(result).GetProperty("llm"); + Assert.Equal(JsonValueKind.Null, llm.GetProperty("dominantCause").ValueKind); + } + } + // --------------------------------------------------------------------- // SAD PATH — status must never generate LLM traffic // --------------------------------------------------------------------- diff --git a/tests/GenWave.Host.Tests/Specs/Story196_LlmCallInspector.cs b/tests/GenWave.Host.Tests/Specs/Story196_LlmCallInspector.cs index bb065a2e..c09de9ec 100644 --- a/tests/GenWave.Host.Tests/Specs/Story196_LlmCallInspector.cs +++ b/tests/GenWave.Host.Tests/Specs/Story196_LlmCallInspector.cs @@ -8,9 +8,9 @@ // dependencies are even touched: a draft-fields preview never calls IPersonaStore/IAdminMediaLookup, // and each is Lazy-backed so merely resolving them via DI opens no connection, see // PersonaServiceCollectionExtensions' own remarks) — POST /api/personas/preview against a real -// Kestrel-backed completions stub (mirrors GenWave.Tts.Tests' MockCompletionsServer; redefined here -// rather than cross-referencing that test project, same as Story186's own file-scoped doubles), then -// GET /api/llm-calls and prove the ring shows exactly what the render produced. +// Kestrel-backed completions stub (Support/LlmCompletionsStub.cs — shared with +// Story353_LlmCauseTaxonomy.cs since T334 review round 1, advisory a), then GET /api/llm-calls and +// prove the ring shows exactly what the render produced. // // AC2 mirrors Story172_PublicListenerIsolation's own idiom for "both listeners": the internal // listener (no session -> 401, the same deny-by-default every other admin route gets) and the public @@ -26,104 +26,25 @@ using System.Net; using System.Net.Http.Json; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using GenWave.Host.Api; using GenWave.Host.Tests.Fakes; +using GenWave.Host.Tests.Support; using GenWave.Tts; namespace GenWave.Host.Tests.Specs; -// ── In-process stub / fakes ────────────────────────────────────────────────────────────────────── - -/// -/// Minimal Kestrel-backed stub for an OpenAI-compatible POST /v1/chat/completions endpoint — -/// mirrors GenWave.Tts.Tests.MockCompletionsServer (STORY-119), redefined here since -/// Host.Tests has no project reference to that test project (same "redefine, don't cross-reference" -/// convention Story186_CorrectionsObservability's own header note explains). Every request always -/// serves 200 with — this spec has no need for the fuller -/// Serve/Fail/Delay repertoire the Tts.Tests original carries. -/// -sealed class LlmCompletionsStub : IAsyncDisposable -{ - readonly WebApplication app; - - public string ReplyContent { get; set; } = "Great tune coming up, stay tuned."; - public Uri BaseUri { get; } - - LlmCompletionsStub(WebApplication app, Uri baseUri) - { - this.app = app; - BaseUri = baseUri; - } - - public static async Task StartAsync() - { - var builder = WebApplication.CreateSlimBuilder(); - builder.Logging.ClearProviders(); - builder.WebHost.UseUrls("http://127.0.0.1:0"); - - var app = builder.Build(); - LlmCompletionsStub? stubRef = null; - - app.MapPost("/v1/chat/completions", async (HttpContext ctx) => - { - var stub = stubRef; - if (stub is null) - { - ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; - return; - } - - ctx.Response.StatusCode = StatusCodes.Status200OK; - await ctx.Response.WriteAsJsonAsync( - new { choices = new[] { new { message = new { content = stub.ReplyContent } } } }, - ctx.RequestAborted); - }); - - await app.StartAsync(); - var stub = new LlmCompletionsStub(app, new Uri(app.Urls.First())); - stubRef = stub; - return stub; - } - - public async ValueTask DisposeAsync() => await app.DisposeAsync(); -} - // ── WebApplicationFactories ────────────────────────────────────────────────────────────────────── - -/// -/// Boots the real host with a real Llm:Endpoint (a genuine ) -/// so LlmCopyWriter/LlmCallRing/DegradationController are the exact production -/// singletons AddGenWaveTts wires — nothing about the LLM pipeline is faked. Only hosted -/// services are removed (no Liquidsoap/Postgres background work during this test); every -/// Postgres-backed controller dependency PersonaController needs is left as its REAL, -/// Lazy-backed registration (see the file header) since a draft-fields preview never forces any of -/// them to actually connect. -/// -file sealed class LlmCallInspectorWebFactory(string llmEndpoint) : WebApplicationFactory -{ - internal const string Password = "test-password-x9k3"; - - protected override void ConfigureWebHost(IWebHostBuilder builder) - { - builder.UseEnvironment("Development"); - builder.UseSetting("ConnectionStrings:Library", "Host=nowhere;Database=test"); - builder.UseSetting("Admin:Password", Password); - builder.UseSetting("Llm:Endpoint", llmEndpoint); - builder.UseSetting("Llm:Model", "test-model"); - builder.ConfigureTestServices(services => services.RemoveAll()); - } -} +// AC1's completions stub + web factory (Support/LlmCompletionsStub.cs's own LlmCompletionsStub / +// LlmCompletionsWebFactory) are shared with Story353_LlmCauseTaxonomy.cs — see that file's own +// header comment for the extraction rationale (T334 review round 1, advisory a). /// /// Boots the real host with no LLM configured at all (irrelevant to AC2 — nothing here ever calls @@ -158,6 +79,13 @@ file sealed record LlmCallRow( long Seq, string? PersonaName, DateTimeOffset StartedAt, long ElapsedMs, string Status, string? StatusDetail, string Mode, string? PromptSystem, string? PromptUser, string? Response, int PromptChars, int ResponseChars); +/// Wire shape of GET /api/llm-calls itself (SPEC F139.2, PLAN T334) — mirrors +/// without depending on it directly, same as +/// does for each entry. AC1/AC3 below only ever assert on +/// ; the F139.2 counter summary itself is covered by +/// Story353_LlmCauseTaxonomy.cs, not re-proven here. +file sealed record LlmCallsResponseWire(IReadOnlyList Calls); + public static class FeatureLlmCallInspector { static async Task LoginAsync(HttpClient client, string password) @@ -189,9 +117,9 @@ public async Task A_real_preview_render_is_readable_back_via_the_inspector_endpo { // Given a real persona preview render against a real (stub) completions endpoint... stub.ReplyContent = "Spinning up something great, stick around."; - await using var factory = new LlmCallInspectorWebFactory(stub.BaseUri.ToString()); + await using var factory = new LlmCompletionsWebFactory(stub.BaseUri.ToString()); var client = factory.CreateClient(); - await LoginAsync(client, LlmCallInspectorWebFactory.Password); + await LoginAsync(client, LlmCompletionsWebFactory.Password); // When the preview endpoint is driven — the exact production hand-off // (IPersonaPreviewWriter -> the real LlmCopyWriter -> RequestCleanedCompletionAsync) every @@ -201,9 +129,9 @@ public async Task A_real_preview_render_is_readable_back_via_the_inspector_endpo // Then the inspector endpoint shows exactly one entry, carrying prompt/response/timing/ // status/mode (SPEC F73.1) — read back as an admin, capped at ring size, newest first. - var rows = await client.GetFromJsonAsync>("/api/llm-calls"); - Assert.NotNull(rows); - var row = Assert.Single(rows!); + var response = await client.GetFromJsonAsync("/api/llm-calls"); + Assert.NotNull(response); + var row = Assert.Single(response!.Calls); Assert.True( row.Status == "ok" && @@ -305,27 +233,27 @@ public async Task A_new_host_instance_never_sees_the_previous_ones_entries() await using var stub = await LlmCompletionsStub.StartAsync(); // Given a ring entry recorded on a first host instance... - await using (var factory1 = new LlmCallInspectorWebFactory(stub.BaseUri.ToString())) + await using (var factory1 = new LlmCompletionsWebFactory(stub.BaseUri.ToString())) { var client1 = factory1.CreateClient(); - await LoginAsync(client1, LlmCallInspectorWebFactory.Password); + await LoginAsync(client1, LlmCompletionsWebFactory.Password); var preview = await client1.PostAsJsonAsync("/api/personas/preview", DraftPreviewBody()); Assert.Equal(HttpStatusCode.OK, preview.StatusCode); - var rows1 = await client1.GetFromJsonAsync>("/api/llm-calls"); - Assert.Single(rows1!); + var response1 = await client1.GetFromJsonAsync("/api/llm-calls"); + Assert.Single(response1!.Calls); } // When a brand-new host instance stands up — a fresh DI container, standing in for a // process restart (nothing about LlmCallRing could carry state across this boundary; // see the no-persistence-dependency fact above)... - await using var factory2 = new LlmCallInspectorWebFactory(stub.BaseUri.ToString()); + await using var factory2 = new LlmCompletionsWebFactory(stub.BaseUri.ToString()); var client2 = factory2.CreateClient(); - await LoginAsync(client2, LlmCallInspectorWebFactory.Password); + await LoginAsync(client2, LlmCompletionsWebFactory.Password); // Then its ring is empty (SPEC F73.3) — restart clears it, by construction. - var rows2 = await client2.GetFromJsonAsync>("/api/llm-calls"); - Assert.Empty(rows2!); + var response2 = await client2.GetFromJsonAsync("/api/llm-calls"); + Assert.Empty(response2!.Calls); } } } diff --git a/tests/GenWave.Host.Tests/Specs/Story317_SpecialsApi.cs b/tests/GenWave.Host.Tests/Specs/Story317_SpecialsApi.cs index 6c414e9c..6b832097 100644 --- a/tests/GenWave.Host.Tests/Specs/Story317_SpecialsApi.cs +++ b/tests/GenWave.Host.Tests/Specs/Story317_SpecialsApi.cs @@ -22,6 +22,7 @@ // GenWave.Orchestration.Tests/Story241_StationFollowsTheClock.cs's own ScenarioSpecialsRideTheCache // instead — this file stays scoped to the WIRE mapping described above. +using System.Globalization; using System.Net; using System.Net.Http.Json; using System.Text.Json; @@ -50,18 +51,26 @@ public sealed class ScenarioCrudThroughTheProductionSurface [Fact] public async Task CreateListsAndDeletesRoundTripThroughTheEndpoints() { - // Given an authenticated admin session and a known persona/show to reference + // Given an authenticated admin session and a known persona/show to reference. The + // factory is pinned to the file's own fake Today (hygiene fix, T330 round-2 review): the + // ORIGINAL fact never passed now: Today, so it defaulted to the REAL wall clock while + // still hardcoding a literal "2026-08-20" as "unambiguously future" — a date bomb that + // reds the instant real time reaches it. onDate is now computed off the SAME pinned Today + // every other date-sensitive fact in this file already uses, preserving the "5 days + // future" intent without depending on when this suite happens to run. var persona = new Persona(1, "Nova", "", "", "", DateTime.UtcNow, DateTime.UtcNow); var show = new Show(1, "Night Moves", "night-moves", null, null, null, null, DateTime.UtcNow, DateTime.UtcNow); var specialStore = new FakeScheduleSpecialStore(); await using var factory = new SpecialsApiWebFactory( - specialStore, personaStore: new FakePersonaStore([persona]), showStore: new FakeShowStore([show])); + specialStore, personaStore: new FakePersonaStore([persona]), showStore: new FakeShowStore([show]), + now: Today); var client = await SpecialsApiWebFactory.LoggedInClientAsync(factory); + var onDate = DateOnly.FromDateTime(Today.DateTime).AddDays(5); // When a special is created, then listed, then deleted via /api/schedule/specials var createResponse = await client.PostAsJsonAsync("/api/schedule/specials", new { - onDate = "2026-08-20", + onDate = onDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), startMinute = 540, endMinute = 720, personaId = persona.Id, @@ -82,7 +91,7 @@ public async Task CreateListsAndDeletesRoundTripThroughTheEndpoints() // Then creation succeeds naming every submitted field, the list carries the new row, and // the delete removes it cleanly Assert.Equal( - (Create: HttpStatusCode.Created, OnDate: new DateOnly(2026, 8, 20), PersonaId: (long?)persona.Id, + (Create: HttpStatusCode.Created, OnDate: onDate, PersonaId: (long?)persona.Id, ShowId: (long?)show.Id, Listed: true, Delete: HttpStatusCode.NoContent, GoneAfterDelete: true), (Create: createResponse.StatusCode, created.OnDate, created.PersonaId, created.ShowId, Listed: list!.Any(s => s.Id == created.Id), diff --git a/tests/GenWave.Host.Tests/Specs/Story350_TruthLaneEndToEnd.cs b/tests/GenWave.Host.Tests/Specs/Story350_TruthLaneEndToEnd.cs new file mode 100644 index 00000000..ad65ddc4 --- /dev/null +++ b/tests/GenWave.Host.Tests/Specs/Story350_TruthLaneEndToEnd.cs @@ -0,0 +1,286 @@ +// STORY-350, STORY-351, STORY-353 — the truth lane wired end to end (SPEC F138-F139 · PLAN T335) +// +// BDD specification — xUnit. Every fact below drives the REAL production DI graph +// (WebApplicationFactory, via the shared Support/LlmCompletionsStub.cs types T334 +// extracted) rather than a hand-built collaborator graph — the wire-proof acceptance PLAN T335 +// itself states: "real Kestrel, real render chain, real admin UI." +// +// ALTITUDE (read this before any fact below — every render seam here is chosen deliberately, not +// by default): +// +// * ScenarioContextSegmentReasksThenAirsTheCleanReply and +// ScenarioLeadInWrongWeekdayDegradesAndTheOperatorCanSeeWhy resolve the REAL, singleton +// ISegmentCopyWriter straight off the booted container (factory.Services.GetRequiredService) +// and call WriteAsync directly, rather than reaching it over HTTP. There is no HTTP surface +// that reaches it WITH the facts these scenarios need: PersonaController.Preview is bound +// straight to LlmCopyWriter (bypassing DegradationGatedCopyWriter entirely, SPEC F69.4) and +// never carries ContextFacts at all (PLAN T331's own "structurally ungated" finding) — the +// ONLY caller that ever builds a fact-bearing ContextSegment/LeadIn SegmentRequest is the +// Orchestrator's own background playout loop, which LlmCompletionsWebFactory removes along +// with every other IHostedService (no Liquidsoap/Postgres churn during a test). Resolving the +// interface straight off the container is therefore the HIGHEST seam honestly reachable: every +// collaborator downstream of it — the named HttpClient, DegradationController, and critically +// the LlmCallRing/LlmCallCauseCounters singletons — is the exact SAME production object +// GET /api/llm-calls and GET /api/status read moments later over real authenticated requests. +// +// * ScenarioCrosstalkTruthDiscardIsVisibleOnTheSurface resolves the real CrosstalkScriptWriter +// singleton the SAME way (it is an ordinary AddSingleton() in +// TtsServiceCollectionExtensions, reachable directly — no CrosstalkWorkerHarness detour +// needed: that harness hand-builds its OWN isolated LlmCallRing/counters pair unless a caller +// explicitly threads the production ones through it, so resolving the real DI singleton +// directly is the MORE honest seam here, not a fallback from it) and calls +// WriteExchangeAsync directly, against the SAME stub. +// +// Every scenario shares the SAME Support/LlmCompletionsStub.cs types Story196_LlmCallInspector.cs +// and Story353_LlmCauseTaxonomy.cs already use — extended minimally (T335) with call-sequenced +// QueueReplies and captured Requests, additively, so neither existing file changed. + +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using GenWave.Core.Abstractions; +using GenWave.Core.Domain; +using GenWave.Host; +using GenWave.Host.Tests; +using GenWave.Host.Tests.Support; +using GenWave.Tts; + +namespace GenWave.Host.Tests.Specs; + +// ── Wire shapes (mirrors Story353_LlmCauseTaxonomy.cs's own narrower-than-the-DTO idiom — a +// `file`-scoped redefinition per spec file, never a cross-file reference to the server DTOs) ────── + +/// Wire shape of one row from GET /api/llm-calls (SPEC F73.1, F139.1) — adds +/// Seq over Story353's own narrower row so a fact can order the ring's newest-first response +/// back into call order. +file sealed record LlmCallRow(long Seq, string Cause, string Model, string Kind); + +/// Wire shape of one causeSummary row (SPEC F139.2). +file sealed record LlmCallCauseSummaryRow(string Cause, string Model, string Kind, int Count); + +/// Wire shape of GET /api/llm-calls itself (SPEC F139.2). +file sealed record LlmCallsSurfaceResponse( + IReadOnlyList Calls, IReadOnlyList CauseSummary); + +/// Wire shape of the llm block on GET /api/status (SPEC F34.8, F139.2) — only +/// the three F139.2 dominant-cause fields this file asserts on. +file sealed record StatusLlmBlock(string? DominantCause, int? DominantCauseCount, string? DominantCauseModel); + +/// Wire shape of GET /api/status itself — only the llm block; every other +/// top-level field (catalog, safeScope, …) is simply never bound. +file sealed record StatusSurfaceResponse(StatusLlmBlock Llm); + +/// +/// The SAME production wiring LlmCompletionsWebFactory (Support/LlmCompletionsStub.cs) +/// configures — that type is sealed, so this is composition-by-duplication, not inheritance, +/// mirroring this test project's own "redefine, don't reach across" convention that Support file's +/// own header note documents — PLUS a fake . GET /api/status's +/// StatusController resolves the real Postgres-backed MediaRepository otherwise, which +/// does NOT share OnAirPersonaAccessor's/CachingScheduleResolver's own graceful +/// "unconfigured Station Postgres is a supported deployment shape" contract — it throws against this +/// factory's bogus ConnectionStrings:Library, 500ing the one scenario below that reads +/// /api/status. The shared, already-built tests/GenWave.Host.Tests/FakeMediaCatalog.cs +/// (STORY-084's own "for GET /api/status specs" fake) is the right tool here — no new fake invented +/// for this file. +/// +file sealed class TruthLaneStatusWebFactory(string llmEndpoint) : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Development"); + builder.UseSetting("ConnectionStrings:Library", "Host=nowhere;Database=test"); + builder.UseSetting("Admin:Password", LlmCompletionsWebFactory.Password); + builder.UseSetting("Llm:Endpoint", llmEndpoint); + builder.UseSetting("Llm:Model", LlmCompletionsWebFactory.Model); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(new FakeMediaCatalog(ready: null)); + }); + } +} + +public static class FeatureTruthLaneEndToEnd +{ + static async Task LoginAsync(HttpClient client, string password) + { + var login = await client.PostAsJsonAsync("/api/auth/login", new { password }); + Assert.Equal(HttpStatusCode.NoContent, login.StatusCode); + } + + /// A minimal, valid two-persona pair for — mirrors + /// Support/CrosstalkWorkerHarness.cs's own identically-named helper (redefined here rather than + /// shared: that harness's helper is a `file`-scoped method one project idiom over, and this fact + /// needs no other part of that harness). + static PersonaCard MakeCard(string name) => + new(1, name, "", $"{name}'s soul.", [], new VoiceSpec("kokoro", "af_heart", 1.0, "en"), + EnergyDisposition: 0, [], []); + + public static class ScenarioContextSegmentReasksThenAirsTheCleanReply + { + const string FactBlock = "Calgary: sunny, 18°C. Wind 10 km/h from the northwest."; + const string PoisonedCopy = "It's a blustery 45 degrees out there today."; + const string CleanCopy = "It's sunny at 18 degrees with wind at 10 kilometers per hour from the northwest."; + + [Fact] + public static async Task A_poisoned_digit_reasks_once_and_the_endpoint_shows_both_causes() + { + // Given a real production host wired to a real (stub) completions endpoint whose FIRST + // reply claims a digit (45) the fact block never supports, and whose re-ask reply is clean... + await using var stub = await LlmCompletionsStub.StartAsync(); + stub.QueueReplies(PoisonedCopy, CleanCopy); + await using var factory = new LlmCompletionsWebFactory(stub.BaseUri.ToString()); + var client = factory.CreateClient(); + await LoginAsync(client, LlmCompletionsWebFactory.Password); + + var writer = factory.Services.GetRequiredService(); + var request = new SegmentRequest( + SegmentKind.ContextSegment, "af_heart", "GenWave", Track: null, DateTimeOffset.UtcNow, + "test-station", PersonaName: null, CounterpartName: null, ContextFacts: FactBlock); + + // When it renders through the REAL DegradationGatedCopyWriter -> LlmCopyWriter chain + // the Orchestrator's own graph resolves (SPEC F34.1, F69.1)... + var result = await writer.WriteAsync(request, CancellationToken.None); + + // Then the gate re-asked exactly once, and the clean reply is what airs. + Assert.Equal(2, stub.Requests.Count); + Assert.Equal(CleanCopy, result.Text); + Assert.True(result.FreshPerAiring); + + // And the F138.5 guard line rode BOTH system prompts, read back off the real wire body + // the stub actually received — not merely asserted against in-process state. + Assert.All( + stub.Requests, + req => Assert.Contains("Never name another day or time of day.", req.SystemPrompt)); + + // When the real admin endpoint is read back over an authenticated request... + var response = await client.GetFromJsonAsync("/api/llm-calls"); + Assert.NotNull(response); + var ordered = response!.Calls.OrderBy(row => row.Seq).ToArray(); + + // Then the first (rejected) call and the second (accepted re-ask) each carry their own + // honest cause — never folded into one entry. + Assert.Equal(2, ordered.Length); + Assert.Equal("truthgatereject", ordered[0].Cause); + Assert.Equal("success", ordered[1].Cause); + Assert.All(ordered, row => Assert.Equal(LlmCompletionsWebFactory.Model, row.Model)); + + // ...and the 24h causeSummary counts both, per (cause, model, kind). + Assert.Contains( + response.CauseSummary, + row => row is { Cause: "truthgatereject", Model: LlmCompletionsWebFactory.Model, Kind: "copy", Count: 1 }); + Assert.Contains( + response.CauseSummary, + row => row is { Cause: "success", Model: LlmCompletionsWebFactory.Model, Kind: "copy", Count: 1 }); + } + } + + public static class ScenarioLeadInWrongWeekdayDegradesAndTheOperatorCanSeeWhy + { + [Fact] + public static async Task Both_replies_claiming_the_wrong_weekday_degrade_to_the_template_and_the_wire_names_the_cause() + { + // Given a real production host, and a wrong-weekday claim computed against the SAME real + // IStationClockProvider seam the writer itself reads (never a fixed fixture date) — so + // this fact is correct regardless which day it actually runs on... + await using var stub = await LlmCompletionsStub.StartAsync(); + await using var factory = new TruthLaneStatusWebFactory(stub.BaseUri.ToString()); + var client = factory.CreateClient(); + await LoginAsync(client, LlmCompletionsWebFactory.Password); + + var stationClock = factory.Services.GetRequiredService(); + var actualWeekday = stationClock.LocalNow.DayOfWeek; + var wrongWeekday = actualWeekday == DayOfWeek.Saturday ? DayOfWeek.Sunday : DayOfWeek.Saturday; + // ReplyContent (not QueueReplies): BOTH the first call and the re-ask must claim the + // SAME wrong weekday, so the ladder's re-ask still violates and the render exhausts it. + stub.ReplyContent = $"This {wrongWeekday} has been one for the books so let's keep it going."; + + var writer = factory.Services.GetRequiredService(); + var request = new SegmentRequest( + SegmentKind.LeadIn, "af_heart", "GenWave", + new MediaItem("m1", "/media/x.mp3", "Astral Plane", default, "Valerie June"), + DateTimeOffset.UtcNow, "test-station"); + + // When it renders through the real writer chain and the re-ask still violates... + var result = await writer.WriteAsync(request, CancellationToken.None); + + // Then it degrades to the deterministic LeadIn template floor — never the still-violating + // LLM text, and never silence (LeadIn carries no F107.6-style skip guard). + Assert.Equal("Coming up: Astral Plane by Valerie June.", result.Text); + Assert.False(result.FreshPerAiring); + Assert.Equal(2, stub.Requests.Count); + Assert.All( + stub.Requests, + req => Assert.Contains("Never name another day or time of day.", req.SystemPrompt)); + + // And the real admin endpoint shows both rejections in its ring. + var callsResponse = await client.GetFromJsonAsync("/api/llm-calls"); + Assert.NotNull(callsResponse); + Assert.Equal(2, callsResponse!.Calls.Count(row => row.Cause == "truthgatereject")); + + // And, after this render, GET /api/status names the SAME dominant cause + model an + // operator staring at a red LLM tile needs (SPEC F139.2) — no SSH, no Loki. + var status = await client.GetFromJsonAsync("/api/status"); + Assert.NotNull(status); + Assert.Equal("truthgatereject", status!.Llm.DominantCause); + Assert.True(status.Llm.DominantCauseCount >= 2); + Assert.Equal(LlmCompletionsWebFactory.Model, status.Llm.DominantCauseModel); + } + } + + public static class ScenarioCrosstalkTruthDiscardIsVisibleOnTheSurface + { + // Clears every SHAPE rule (3-8 alternating HOST:/NEIGHBOR: lines, both speakers present, no + // line over budget) but names a real-world FM frequency (SPEC F138.6) — never a weekday/ + // daypart/condition/date word, so this fact's outcome depends on nothing but the frequency + // shape, regardless of which TruthShapeChecks entry happens to run first. + const string FrequencyScript = + "HOST: Hey glad you could drop by the studio for a chat.\n" + + "NEIGHBOR: Always fun swinging by between tracks.\n" + + "HOST: Someone in the chat says we sound just like 101 FM.\n" + + "NEIGHBOR: Ha well we will take that as a compliment and keep the music going."; + + [Fact] + public static async Task A_real_world_frequency_discards_the_exchange_and_the_summary_names_the_crosstalk_lane() + { + // Given a real production host, and a reply naming a real-world FM frequency... + await using var stub = await LlmCompletionsStub.StartAsync(); + stub.ReplyContent = FrequencyScript; + await using var factory = new LlmCompletionsWebFactory(stub.BaseUri.ToString()); + var client = factory.CreateClient(); + await LoginAsync(client, LlmCompletionsWebFactory.Password); + + // The real CrosstalkScriptWriter singleton (TtsServiceCollectionExtensions' + // AddSingleton()) — reachable directly, no worker/harness needed. + var scriptWriter = factory.Services.GetRequiredService(); + var request = new CrosstalkExchangeRequest( + MakeCard("Host DJ"), MakeCard("Next DJ"), "GenWave", ShowName: null, Daypart: null, + StationLocalNow: DateTimeOffset.UtcNow); + + // When it renders through the real writer... + var result = await scriptWriter.WriteExchangeAsync(request, CancellationToken.None); + + // Then the exchange is discarded on the real F138.6 truth check — never a re-ask (F127.4 + // has none for crosstalk: a truth discard is silent, the stock worker just tries again). + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Single(stub.Requests); + Assert.Contains("Never name another day or time of day.", stub.Requests[0].SystemPrompt); + + // And the real admin endpoint's causeSummary carries kind=crosstalk for this discard — + // the T334 review's own open line item: the crosstalk lane's 24h aggregate is on the wire + // but was rendered nowhere until this proof. + var response = await client.GetFromJsonAsync("/api/llm-calls"); + Assert.NotNull(response); + Assert.Contains( + response!.CauseSummary, + row => row is { Cause: "truthgatereject", Kind: "crosstalk", Count: 1 }); + } + } +} diff --git a/tests/GenWave.Host.Tests/Specs/Story353_LlmCauseTaxonomy.cs b/tests/GenWave.Host.Tests/Specs/Story353_LlmCauseTaxonomy.cs index bb445cfb..05c270da 100644 --- a/tests/GenWave.Host.Tests/Specs/Story353_LlmCauseTaxonomy.cs +++ b/tests/GenWave.Host.Tests/Specs/Story353_LlmCauseTaxonomy.cs @@ -1,66 +1,387 @@ // STORY-353 — A red LLM tile names its cause (SPEC F139 · PLAN T330/T334/T335) // -// BDD specification — xUnit. PENDING until built (see the Tts Story350 header note). -// The admin-ui tile half rides admin-ui/__specs__/health-tile-llm-cause.spec.tsx. +// BDD specification — xUnit. T330's facts below drive the REAL resolution paths — a hand-rolled +// LlmCopyWriter (the house FakeHttpMessageHandler/SingleHandlerHttpClientFactory idiom, mirrors +// GenWave.Tts.Tests' Story189_LlmSingleFlightAndWarnDetail) for the Copy-kind causes, and the shared +// CrosstalkWorkerHarness (Support/) — the SAME real CrosstalkStockWorker/CrosstalkScriptWriter/ +// CrosstalkAssembler wiring Story328_CrosstalkStockWorker.cs itself drives — for the break-window +// abandon. ScenarioCountersRoll is the one PURE-level exception (LlmCallCauseCounters has no I/O of +// its own to drive through). ScenarioTheSurfaceServesTheTaxonomy (PLAN T334) drives the deployed +// GET /api/llm-calls endpoint itself, WebApplicationFactory end to end — mirrors +// Story196_LlmCallInspector.cs's own AC1 idiom (a Kestrel-backed completions stub, a real +// POST /api/personas/preview render, then read the admin endpoint back) via the SAME shared +// Support/LlmCompletionsStub.cs types that file uses (T334 review round 1, advisory a — the two +// files carried a verbatim ~90-line copy of this stub/factory each before the extraction). // -// gh-#365's acceptance is the dev-station case verbatim: a tile that flaps red every -// 1–2 hours on an external ollama (gemma-class on a 16GB 4090 laptop) explains itself -// from the admin UI — no SSH, no Loki, no darts at Llm settings. +// gh-#365's acceptance is the dev-station case verbatim: a tile that flaps red every 1–2 hours on an +// external ollama (gemma-class on a 16GB 4090 laptop) explains itself from the admin UI — no SSH, no +// Loki, no darts at Llm settings. + +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using GenWave.Core.Domain; +using GenWave.Host.Tests.Fakes; +using GenWave.Host.Tests.Support; +using GenWave.Tts; namespace GenWave.Host.Tests.Specs; +// ── Wire shapes for ScenarioTheSurfaceServesTheTaxonomy (PLAN T334) ──────────────────────────────── +// The completions stub + web factory themselves are Support/LlmCompletionsStub.cs's +// LlmCompletionsStub/LlmCompletionsWebFactory, shared with Story196_LlmCallInspector.cs (T334 +// review round 1, advisory a). + +/// Wire shape of one row from GET /api/llm-calls — only the two SPEC F139.1 fields +/// this scenario cares about (mirrors Story196_LlmCallInspector.cs's own narrower-than-the-DTO +/// LlmCallRow idiom). +file sealed record LlmCallCauseRow(string Cause, string Model); + +/// Wire shape of one causeSummary row (SPEC F139.2, PLAN T334) — mirrors +/// without depending on it directly. +file sealed record LlmCallCauseSummaryRow(string Cause, string Model, string Kind, int Count); + +/// Wire shape of GET /api/llm-calls itself (SPEC F139.2, PLAN T334) — mirrors +/// without depending on it directly. +file sealed record LlmCallsSurfaceResponse( + IReadOnlyList Calls, IReadOnlyList CauseSummary); + public static class FeatureLlmCauseTaxonomy { + // ── Shared arrange for the Copy-kind facts (mirrors GenWave.Tts.Tests' own BuildWriter idiom) ── + + static SegmentRequest LeadInRequest() => + new(SegmentKind.LeadIn, "af_heart", "GenWave", + new MediaItem("m1", "/media/x.mp3", "Astral Plane", default, "Valerie June"), + DateTimeOffset.UtcNow, "test-station"); + + /// Builds a REAL against a fake completions handler — the + /// one constructor arg list every fact in shares (except + /// the window-cancel fact, which uses instead). Hands back + /// the ring AND the counters (SPEC F139 review finding F2) — the SAME + /// feeds both, so a fact can prove either half moved, or both. + static (LlmCopyWriter Writer, LlmCallRing Ring, LlmCallCauseCounters Counters) BuildWriter( + Func> respond, + int timeoutSeconds = 5, int maxCopyChars = 450) + { + var ring = new LlmCallRing(new FakeOptionsMonitor(new LlmOptions())); + var counters = new LlmCallCauseCounters(TimeProvider.System); + var writer = new LlmCopyWriter( + new TemplateCopyWriter(new PatterTemplateRenderer()), + new SingleHandlerHttpClientFactory(new FakeHttpMessageHandler(respond)), + new FakeOptionsMonitor(new LlmOptions + { + Endpoint = "http://fake-llm.local", Model = "test-model", TimeoutSeconds = timeoutSeconds, + MaxCopyChars = maxCopyChars, + }), + new LlmCopyStatusHolder(), + new FakeActivePersonaAccessor(), + NullLogger.Instance, + TimeProvider.System, + new LlmCallRecorder(ring, counters), + new FakeDegradationModeReader()); + return (writer, ring, counters); + } + + static Task Ok(string content) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = CompletionsBody(content), + }); + + static StringContent CompletionsBody(string content) => new( + JsonSerializer.Serialize(new + { + choices = new[] { new { message = new { content } } }, + }), + Encoding.UTF8, "application/json"); + public static class ScenarioOutcomesAreTyped { - [Fact(Skip = "pending T330 — LlmCallOutcome does not exist yet")] - public static void A_successful_call_records_Success() => - Assert.Fail("pending T330: the F73 ring entry carries exactly one cause"); + [Fact] + public static async Task A_successful_call_records_Success() + { + // Given a completions reply that fits comfortably under Llm:MaxCopyChars... + var (writer, ring, _) = BuildWriter((_, _) => Ok("Great tune coming up, stay tuned.")); + + // When it airs through the real WriteAsync -> RequestCleanedCompletionAsync seam... + await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then the F73 ring entry it left behind carries exactly one cause: Success. + Assert.Equal(LlmCallCause.Success, Assert.Single(ring.Snapshot()).Cause); + } + + [Fact] + public static async Task A_timed_out_call_records_Timeout() + { + // Given a completions endpoint that never answers inside Llm:TimeoutSeconds... + var (writer, ring, _) = BuildWriter( + async (_, ct) => + { + await Task.Delay(TimeSpan.FromSeconds(3), ct); + return new HttpResponseMessage(HttpStatusCode.OK); + }, + timeoutSeconds: 1); + + // When the render's own timeout budget elapses (RequestCleanedCompletionAsync's own + // timeoutCts, not the caller's token — CancellationToken.None here proves that)... + await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then the ring records Timeout — never a generic connection failure. + Assert.Equal(LlmCallCause.Timeout, Assert.Single(ring.Snapshot()).Cause); + } + + [Fact] + public static async Task An_over_length_call_records_OverLength() + { + // Given a reply with no sentence terminator anywhere, well over a tiny Llm:MaxCopyChars — + // the gh-#277 shape: nothing survives TrimToLastCompleteSentence's own salvage, so CleanCopy + // rejects with WasOverLength: true (a candidate existed, none fit). + var overLengthNoTerminator = string.Concat(Enumerable.Repeat("word ", 40)); + var (writer, ring, _) = BuildWriter((_, _) => Ok(overLengthNoTerminator), maxCopyChars: 50); + + // When it resolves through the real MaxCopyChars rejection path... + await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then the ring names the gh-#277 family by its new name: OverLength. + Assert.Equal(LlmCallCause.OverLength, Assert.Single(ring.Snapshot()).Cause); + } + + // SPEC F139 review finding F2 (T330): mutation-proven, the Story326:430 precedent one file + // over — LlmCallRecorder folds the ring write and the counter write into ONE call, so there + // is no longer a "delete just the counter half" mutation to express; this fact still pins the + // counter side explicitly rather than trusting that structural guarantee alone. + [Fact] + public static async Task A_successful_call_moves_the_cause_counter() + { + // Given a REAL writer wired to a REAL LlmCallRecorder (ring + counters, one call)... + var (writer, _, counters) = BuildWriter((_, _) => Ok("Great tune coming up, stay tuned.")); + + // When it airs through the real WriteAsync -> RequestCleanedCompletionAsync seam... + await writer.WriteAsync(LeadInRequest(), CancellationToken.None); - [Fact(Skip = "pending T330")] - public static void A_timed_out_call_records_Timeout() => - Assert.Fail("pending T330: a Llm:TimeoutSeconds breach records Timeout, never a generic failure"); + // Then the rolling cause counter moved too — not just the ring. + Assert.Equal( + 1, counters.Snapshot().Single(row => + row is { Cause: LlmCallCause.Success, Model: "test-model", Kind: LlmCallKind.Copy }).Count); + } - [Fact(Skip = "pending T330")] - public static void An_over_length_call_records_OverLength() => - Assert.Fail("pending T330: a MaxCopyChars rejection records OverLength (the gh-#277 family gains a name)"); + /// Drives the REAL CrosstalkStockWorker/CrosstalkScriptWriter/ + /// CrosstalkAssembler wiring via — the SAME + /// arrangement Story328_CrosstalkStockWorker.An_in_flight_generation_is_cancelled_the_instant_the_window_reopens + /// drives, supplying this fact's OWN so it can read back what the + /// worker stamped (SPEC F139.1's own "reuse the signal, don't re-derive" — CanceledByWindow is + /// stamped by the worker, never by CrosstalkScriptWriter itself; see + /// CrosstalkStockWorker.RecordWindowCancellation's own remarks for why). + [Fact] + public static async Task A_window_cancelled_stock_call_records_CanceledByWindow() + { + const string ShowSlug = "night-shift"; + const string ShowName = "Night Shift"; - [Fact(Skip = "pending T330")] - public static void A_window_cancelled_stock_call_records_CanceledByWindow() => - Assert.Fail("pending T330: a crosstalk mid-flight abandon records CanceledByWindow"); + // Given a real stock-timer tick whose script generation completes (an Accepted exchange — + // its own ring entry lands as Success) but whose per-line synth blocks forever... + var ring = new LlmCallRing(new FakeOptionsMonitor(new LlmOptions())); + var now = new DateTimeOffset(2026, 1, 5, 12, 0, 0, TimeSpan.Zero); // a Monday noon + var (worker, gate, timeProvider, _, llmHandler, synthesizer) = + await CrosstalkWorkerHarness.BuildAsync(now, ShowSlug, ShowName, callRing: ring); + + var tickTask = worker.TickOnceAsync(CancellationToken.None); + + // ...and generation genuinely started (the positive control: the LLM was really called). + await synthesizer.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Single(llmHandler.Requests); + + // When a real on-air render starts mid-flight and the watchdog's next poll observes it... + gate.Enter(); + timeProvider.Advance(TimeSpan.FromSeconds(3)); // CrosstalkStockWorker's own WatchdogInterval + await tickTask.WaitAsync(TimeSpan.FromSeconds(5)); + + // Then the NEWEST ring entry (the abandoned synth, recorded after the earlier successful + // script generation) carries CanceledByWindow, under the Crosstalk kind. + var newest = ring.Snapshot()[0]; + Assert.Equal(LlmCallCause.CanceledByWindow, newest.Cause); + Assert.Equal(LlmCallKind.Crosstalk, newest.Kind); + } } + // ── PURE level: LlmCallCauseCounters has no I/O of its own to drive through a resolution path ── + public static class ScenarioCountersRoll { - [Fact(Skip = "pending T330")] - public static void Counts_group_per_cause_model_and_kind() => - Assert.Fail("pending T330: the 24h counters key on (cause, model, segment kind)"); + [Fact] + public static void Counts_group_per_cause_model_and_kind() + { + // Given a mix of resolved calls across two models and both segment kinds... + var counters = new LlmCallCauseCounters(new FakeTimeProvider(DateTimeOffset.UtcNow)); + counters.Record(LlmCallCause.Success, "model-a", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "model-a", LlmCallKind.Copy); + counters.Record(LlmCallCause.Timeout, "model-a", LlmCallKind.Copy); + counters.Record(LlmCallCause.Success, "model-b", LlmCallKind.Crosstalk); + + // When the rolling 24h counters are read... + var snapshot = counters.Snapshot(); - [Fact(Skip = "pending T330")] - public static void Entries_older_than_24h_stop_counting() => - Assert.Fail("pending T330: the rolling window forgets (TimeProvider-driven, testable)"); + // Then counts are grouped per (cause, model, kind) — never merged across a differing key. + Assert.Equal(2, snapshot.Single(row => row is { Cause: LlmCallCause.Success, Model: "model-a", Kind: LlmCallKind.Copy }).Count); + Assert.Equal(1, snapshot.Single(row => row is { Cause: LlmCallCause.Timeout, Model: "model-a", Kind: LlmCallKind.Copy }).Count); + Assert.Equal(1, snapshot.Single(row => row is { Cause: LlmCallCause.Success, Model: "model-b", Kind: LlmCallKind.Crosstalk }).Count); + } + + // Renamed (STORY-353 AC2, amended at T330 review) from Entries_older_than_24h_stop_counting — + // that name asserted a false 24h razor's edge the class body immediately below contradicts. + // The true, honest claim: the hourly-bucket band ages entries out somewhere between 24h and + // 25h, never under, so a 25h advance is the worst-case proof this window forgot the entry. + [Fact] + public static void Entries_age_out_on_the_hourly_bucket_band() + { + // Given one recorded call, counted at the START of its own hourly bucket (an + // hour-aligned clock, so the 25h advance below unambiguously clears LlmCallCauseCounters' + // own bucket-granularity slop — see that class's own remarks: a bucket can hold entries up + // to just under an hour newer than its own start, so the true retention window is "24h to + // 25h", never a razor's-edge 24h+1min)... + var hourAligned = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + var timeProvider = new FakeTimeProvider(hourAligned); + var counters = new LlmCallCauseCounters(timeProvider); + counters.Record(LlmCallCause.Timeout, "model-a", LlmCallKind.Copy); + Assert.Single(counters.Snapshot()); + + // When the rolling window's own clock (TimeProvider, never wall-clock) advances past the + // full 25h worst case... + timeProvider.Advance(TimeSpan.FromHours(25)); + + // Then it no longer counts — the window forgot it. + Assert.Empty(counters.Snapshot()); + } } public static class ScenarioTheSurfaceServesTheTaxonomy { - // The deployed entry point: /api/llm-calls through WebApplicationFactory. - [Fact(Skip = "pending T334 — surface not extended yet")] - public static void Each_call_row_carries_its_cause() => - Assert.Fail("pending T334: a real request through the production pipeline shows the cause per call"); - - [Fact(Skip = "pending T334")] - public static void The_counter_summary_rides_the_response() => - Assert.Fail("pending T334: the 24h by-cause summary is served alongside the ring"); + static async Task LoginAsync(HttpClient client, string password) + { + var login = await client.PostAsJsonAsync("/api/auth/login", new { password }); + Assert.Equal(HttpStatusCode.NoContent, login.StatusCode); + } + + static object DraftPreviewBody() => new + { + kind = "LeadIn", + name = "Neon Nightowl", + backstory = "Spins vinyl til dawn.", + style = "moody, late-night", + }; + + // The deployed entry point: /api/llm-calls through WebApplicationFactory — mirrors + // Story196_LlmCallInspector.cs's own AC1 idiom (POST /api/personas/preview drives the real + // IPersonaPreviewWriter -> LlmCopyWriter -> RequestCleanedCompletionAsync hand-off, SPEC F35.6). + [Fact] + public static async Task Each_call_row_carries_its_cause() + { + // Given a real persona preview render against a real (stub) completions endpoint... + await using var stub = await LlmCompletionsStub.StartAsync(); + await using var factory = new LlmCompletionsWebFactory(stub.BaseUri.ToString()); + var client = factory.CreateClient(); + await LoginAsync(client, LlmCompletionsWebFactory.Password); + + var preview = await client.PostAsJsonAsync("/api/personas/preview", DraftPreviewBody()); + Assert.Equal(HttpStatusCode.OK, preview.StatusCode); + + // When the inspector endpoint is read back... + var response = await client.GetFromJsonAsync("/api/llm-calls"); + Assert.NotNull(response); + var row = Assert.Single(response!.Calls); + + // Then the row itself carries the F139.1 cause and the model it resolved against — a + // clean completion against a well-formed reply is "success"/"test-model". + Assert.Equal("success", row.Cause); + Assert.Equal("test-model", row.Model); + } + + [Fact] + public static async Task The_counter_summary_rides_the_response() + { + // Given the SAME production render as above... + await using var stub = await LlmCompletionsStub.StartAsync(); + await using var factory = new LlmCompletionsWebFactory(stub.BaseUri.ToString()); + var client = factory.CreateClient(); + await LoginAsync(client, LlmCompletionsWebFactory.Password); + + var preview = await client.PostAsJsonAsync("/api/personas/preview", DraftPreviewBody()); + Assert.Equal(HttpStatusCode.OK, preview.StatusCode); + + // When the SAME response is read back... + var response = await client.GetFromJsonAsync("/api/llm-calls"); + Assert.NotNull(response); + + // Then the 24h by-cause summary rides alongside the ring rows in the SAME response — one + // request, not two (the gh-#558 "no new chatty poller" lesson) — grouped by cause/model/kind + // exactly as LlmCallCauseCounters.Snapshot() itself groups (ScenarioCountersRoll above). + var summaryRow = Assert.Single(response!.CauseSummary); + Assert.Equal("success", summaryRow.Cause); + Assert.Equal("test-model", summaryRow.Model); + Assert.Equal("copy", summaryRow.Kind); + Assert.Equal(1, summaryRow.Count); + } } public static class SadPathDiscipline { - [Fact(Skip = "pending T330")] - public static void Nothing_survives_a_restart() => - Assert.Fail("pending T330: fresh ring, fresh counters — F73.3 stands"); + [Fact] + public static async Task Nothing_survives_a_restart() + { + // Given a ring entry and a counter recorded on one process... + var (writer, ring, _) = BuildWriter((_, _) => Ok("Great tune coming up, stay tuned.")); + await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + Assert.Single(ring.Snapshot()); + + var counters = new LlmCallCauseCounters(TimeProvider.System); + counters.Record(LlmCallCause.Success, "test-model", LlmCallKind.Copy); + Assert.Single(counters.Snapshot()); + + // When a brand-new ring and counter store stand up — nothing about either type persists + // anything (F73.3/F139.3 stand): both constructors' only dependency is an options monitor + // or a TimeProvider, no store/repository/connection type in sight — a fresh instance is + // the strongest available proof at this level, mirroring Story196's own AC3 idiom. + var freshRing = new LlmCallRing(new FakeOptionsMonitor(new LlmOptions())); + var freshCounters = new LlmCallCauseCounters(TimeProvider.System); + + // Then both start empty. + Assert.Empty(freshRing.Snapshot()); + Assert.Empty(freshCounters.Snapshot()); + } + + [Fact] + public static async Task A_truth_gate_rejection_is_its_own_cause() + { + // Given a ContextSegment render whose first reply fabricates a claim the real fact + // block never supports (the gh-#434 exhibit shape), and a re-ask reply that finally + // supports it — driven through the real F138.2 gate at the LlmCopyWriter seam (PLAN T331) + const string factBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + const string poisonedCopy = + "It feels like 6 degrees below freezing with plenty of sunshine and today is saturday here in the studio."; + const string cleanCopy = "It's overcast today at 15 degrees with a high of 21 and a low of 12."; + var callCount = 0; + var (writer, ring, _) = BuildWriter((_, _) => + { + callCount++; + return Ok(callCount == 1 ? poisonedCopy : cleanCopy); + }); + var request = new SegmentRequest( + SegmentKind.ContextSegment, "af_heart", "GenWave", Track: null, DateTimeOffset.UtcNow, + "test-station", PersonaName: null, CounterpartName: null, ContextFacts: factBlock); + + // When it renders + await writer.WriteAsync(request, CancellationToken.None); - [Fact(Skip = "pending T331")] - public static void A_truth_gate_rejection_is_its_own_cause() => - Assert.Fail("pending T331: a F138 gate failure records TruthGateReject, distinct from every other cause"); + // Then the ring carries BOTH calls, and the rejected first one is stamped + // TruthGateReject — its own cause, distinct from the re-ask's own Success. + var records = ring.Snapshot(); + Assert.Equal(2, records.Count); + Assert.Contains(records, record => record.Cause == LlmCallCause.TruthGateReject); + Assert.Contains(records, record => record.Cause == LlmCallCause.Success); + } } } diff --git a/tests/GenWave.Host.Tests/Support/CrosstalkWorkerHarness.cs b/tests/GenWave.Host.Tests/Support/CrosstalkWorkerHarness.cs index 7d4357f2..7677ba6d 100644 --- a/tests/GenWave.Host.Tests/Support/CrosstalkWorkerHarness.cs +++ b/tests/GenWave.Host.Tests/Support/CrosstalkWorkerHarness.cs @@ -108,6 +108,42 @@ file sealed class FakeCrosstalkScopeProvider(IReadOnlyList enabledShows) /// internal static class CrosstalkWorkerHarness { + /// + /// Hygiene fix (round-N review — the leaked-temp-dir finding): every call + /// used to hand a FRESH Directory.CreateTempSubdirectory + /// root of its own, straight under the OS temp directory, with nothing ever deleting it — hundreds + /// of orphaned crosstalk-worker-test-* directories accumulate on a box that has run this + /// suite repeatedly (this file alone is called from three spec files, several times each), eventually + /// exhausting tmpfs inodes and silently redding unrelated facts across the whole test run. ONE + /// shared root for the WHOLE test process instead, created lazily on first use; each + /// call gets its own uniquely-named SUBdirectory underneath it, and + /// deletes the entire root, recursively, exactly once, when the + /// test host process itself ends — no per-call disposal for every one of the many call sites across + /// Story328/Story353/Story354 to thread through. + /// + static readonly string SharedTempRoot = CreateSharedTempRoot(); + + static string CreateSharedTempRoot() + { + var root = Directory.CreateTempSubdirectory("crosstalk-worker-tests-").FullName; + AppDomain.CurrentDomain.ProcessExit += (_, _) => + { + try + { + Directory.Delete(root, recursive: true); + } + catch (IOException) + { + // Best-effort cleanup — a stray open handle at process teardown never fails the run. + } + catch (UnauthorizedAccessException) + { + // Same — teardown ordering is not guaranteed, so this is advisory, not load-bearing. + } + }; + return root; + } + static readonly string WellFormedReply = string.Join('\n', new[] { "HOST: Hey, welcome back to the show.", @@ -131,12 +167,22 @@ static PersonaCard MakeCard(string name) => /// empty string here — 's own /// "Llm:Endpoint is not configured" short-circuit, discarding in milliseconds with NO generation /// ever attempted. + /// + /// SPEC F139.1 (STORY-353, PLAN T330): a caller that wants to assert on what + /// stamps into the ring (e.g. a window-cancellation's + /// ) passes its OWN instance here to read back + /// afterward. Defaults to a fresh, unobserved ring — every pre-T330 fact that never cared about + /// the ring at all keeps compiling and passing unchanged. + /// + /// The SPEC F139.2 sibling of — same + /// "supply your own to observe it, otherwise get a fresh unobserved one" shape. public static async Task<( CrosstalkStockWorker Worker, OnAirRenderGate Gate, FakeTimeProvider TimeProvider, NowPlayingService NowPlaying, FakeHttpMessageHandler LlmHandler, BlockingTtsSynthesizer Synthesizer)> BuildAsync( DateTimeOffset now, string showSlug, string showName, string? replyContent = null, - string llmEndpoint = "http://fake-llm.local") + string llmEndpoint = "http://fake-llm.local", LlmCallRing? callRing = null, + LlmCallCauseCounters? causeCounters = null) { var timeProvider = new FakeTimeProvider(now); var gate = new OnAirRenderGate(); @@ -169,20 +215,33 @@ static PersonaCard MakeCard(string name) => { Content = new StringContent(wireResponse, System.Text.Encoding.UTF8, "application/json"), })); + // SPEC F139.1 (PLAN T330): ONE shared LlmOptions monitor for both the script writer and the + // worker itself below — the worker's own CanceledByWindow ring stamp reads Model from this + // SAME monitor, so a test never sees two different "Llm:Model" answers depending on which + // collaborator it asks. + var llmOptionsMonitor = new FakeOptionsMonitor(new LlmOptions + { + Endpoint = llmEndpoint, Model = "test-model", TimeoutSeconds = 5, MaxCopyChars = 300, + }); + var ring = callRing ?? new LlmCallRing(new FakeOptionsMonitor(new LlmOptions())); + var counters = causeCounters ?? new LlmCallCauseCounters(timeProvider); + // SPEC F139.1/F139.2 (T330 review finding F2): ONE shared LlmCallRecorder for both the script + // writer and the worker below — same reasoning as llmOptionsMonitor immediately above, one + // seam over: a test that supplies its own ring/counters gets them fed by whichever collaborator + // records first, never two independently-wrapped recorders racing to write the same pair. + var recorder = new LlmCallRecorder(ring, counters); + var degradationModeReader = new FakeDegradationModeReader(); var scriptWriter = new CrosstalkScriptWriter( new SingleHandlerHttpClientFactory(llmHandler), - new FakeOptionsMonitor(new LlmOptions - { - Endpoint = llmEndpoint, Model = "test-model", TimeoutSeconds = 5, MaxCopyChars = 300, - }), + llmOptionsMonitor, new FakeOptionsMonitor(new CrosstalkOptions()), - new LlmCallRing(new FakeOptionsMonitor(new LlmOptions())), - new FakeDegradationModeReader(), + recorder, + degradationModeReader, NullLogger.Instance, timeProvider); var synthesizer = new BlockingTtsSynthesizer(); - var cacheRoot = Directory.CreateTempSubdirectory("crosstalk-worker-test-").FullName; + var cacheRoot = Directory.CreateDirectory(Path.Combine(SharedTempRoot, Guid.NewGuid().ToString("N"))).FullName; var ttsOptions = new FakeOptionsMonitor(new TtsOptions { CacheRoot = cacheRoot, RenderBudgetSeconds = 30 }); var assembler = new CrosstalkAssembler( synthesizer, @@ -201,7 +260,8 @@ static PersonaCard MakeCard(string name) => var worker = new CrosstalkStockWorker( planner, scriptWriter, assembler, scheduleResolver, nowPlayingService, identityProvider, stationClock, ttsOptions, gate, - NullLogger.Instance, timeProvider); + NullLogger.Instance, timeProvider, + llmOptionsMonitor, recorder, degradationModeReader); return (worker, gate, timeProvider, nowPlayingService, llmHandler, synthesizer); } diff --git a/tests/GenWave.Host.Tests/Support/LlmCompletionsStub.cs b/tests/GenWave.Host.Tests/Support/LlmCompletionsStub.cs new file mode 100644 index 00000000..7d181920 --- /dev/null +++ b/tests/GenWave.Host.Tests/Support/LlmCompletionsStub.cs @@ -0,0 +1,188 @@ +// Extracted from Story196_LlmCallInspector.cs and Story353_LlmCauseTaxonomy.cs (T334 review round +// 1, advisory a): both files carried their own verbatim ~90-line "a Kestrel-backed OpenAI- +// compatible completions stub, plus a WebApplicationFactory that boots the real host +// against it" — a `file`-scoped type genuinely cannot cross files, but a normal internal type in +// the test project's own Support/ folder can (mirrors CrosstalkWorkerHarness.cs's own identical +// precedent one file over: "T335 needs it a third time next task"). Both spec files now call these +// shared types instead of keeping their own copy. +// +// T335 (STORY-350/351/353, SPEC F138.2/F138.4/F138.5) extended this minimally, additively, for the +// wire-proof spec: QueueReplies scripts a reply PER CALL NUMBER (the re-ask ladder fires a SECOND +// completions call inside one WriteAsync — a scenario proving "poisoned, then clean" needs the stub +// itself to answer those two calls differently), and Requests captures each call's parsed +// system/user prompt so a fact can assert the F138.5 guard line rode the real wire body, not just +// that some text arrived. Neither addition changes a single existing caller: both default to +// "nothing queued, ReplyContent answers every call" / "captured but never read", exactly the shape +// Story196/Story353 already exercise unchanged. + +using System.Text.Json; +using GenWave.Host; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace GenWave.Host.Tests.Support; + +/// One POST /v1/chat/completions request parsed off +/// the wire (T335) — the "system"/"user" message content only (the two roles every real caller in +/// this codebase sends, LlmCopyWriter/CrosstalkScriptWriter alike), so a fact can assert on the +/// SAME F138.5 guard line the production prompt builders append, read back through a real HTTP +/// round-trip rather than a hand-rolled capturing handler. +internal sealed record CapturedCompletionsRequest(string SystemPrompt, string UserPrompt); + +/// +/// Minimal Kestrel-backed stub for an OpenAI-compatible POST /v1/chat/completions endpoint +/// — mirrors GenWave.Tts.Tests.MockCompletionsServer (STORY-119) in shape, redefined here +/// since this test project has no reference to that test project (the "redefine, don't +/// cross-reference across test PROJECTS" convention Story186_CorrectionsObservability's own header +/// note explains — this is that same posture applied within ONE test project's shared Support/ +/// folder instead of per spec-file duplication). Every request serves 200 with either the next +/// entry (call-sequenced) or, once that queue is empty, plain +/// — callers needing the fuller Serve/Fail/Delay repertoire should reach +/// for GenWave.Tts.Tests' own original instead of extending this one. +/// +internal sealed class LlmCompletionsStub : IAsyncDisposable +{ + readonly WebApplication app; + readonly object gate = new(); + readonly Queue queuedReplies = new(); + readonly List requests = []; + + public string ReplyContent { get; set; } = "Great tune coming up, stay tuned."; + public Uri BaseUri { get; } + + /// Every request this stub has served so far, in call order (T335) — see + /// 's own remarks. + public IReadOnlyList Requests + { + get + { + lock (gate) + return requests.ToArray(); + } + } + + LlmCompletionsStub(WebApplication app, Uri baseUri) + { + this.app = app; + BaseUri = baseUri; + } + + /// Scripts the reply for the NEXT calls, in order — the Nth queued reply answers the + /// Nth request from this point on (T335, SPEC F138.4's re-ask ladder: a scenario proving the + /// gate re-asks queues "poisoned, then clean" so the SAME stub instance answers both legs of one + /// render differently). Once exhausted, every further request falls back to + /// unchanged — the pre-T335 behavior every existing caller relies on. + public void QueueReplies(params string[] contents) + { + lock (gate) + { + foreach (var content in contents) + queuedReplies.Enqueue(content); + } + } + + public static async Task StartAsync() + { + var builder = WebApplication.CreateSlimBuilder(); + builder.Logging.ClearProviders(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + + var app = builder.Build(); + LlmCompletionsStub? stubRef = null; + + app.MapPost("/v1/chat/completions", async (HttpContext ctx) => + { + var stub = stubRef; + if (stub is null) + { + ctx.Response.StatusCode = StatusCodes.Status500InternalServerError; + return; + } + + var payload = await JsonSerializer.DeserializeAsync( + ctx.Request.Body, cancellationToken: ctx.RequestAborted); + var (systemPrompt, userPrompt) = ExtractPrompts(payload); + + string reply; + lock (stub.gate) + { + reply = stub.queuedReplies.Count > 0 ? stub.queuedReplies.Dequeue() : stub.ReplyContent; + stub.requests.Add(new CapturedCompletionsRequest(systemPrompt, userPrompt)); + } + + ctx.Response.StatusCode = StatusCodes.Status200OK; + await ctx.Response.WriteAsJsonAsync( + new { choices = new[] { new { message = new { content = reply } } } }, + ctx.RequestAborted); + }); + + await app.StartAsync(); + var stub = new LlmCompletionsStub(app, new Uri(app.Urls.First())); + stubRef = stub; + return stub; + } + + /// Pulls the "system"/"user" message content out of an OpenAI-shaped chat-completions + /// request body (T335) — the same messages: [{ role, content }, …] shape every real + /// caller here (LlmCopyWriter, CrosstalkScriptWriter) sends. Missing/malformed fields default to + /// "" rather than throwing — a capture helper must never be the reason a scenario's real request + /// fails. + static (string SystemPrompt, string UserPrompt) ExtractPrompts(JsonElement payload) + { + var systemPrompt = ""; + var userPrompt = ""; + + if (payload.TryGetProperty("messages", out var messages) && messages.ValueKind == JsonValueKind.Array) + { + foreach (var message in messages.EnumerateArray()) + { + var role = message.TryGetProperty("role", out var roleProp) ? roleProp.GetString() : null; + var content = message.TryGetProperty("content", out var contentProp) ? contentProp.GetString() ?? "" : ""; + + if (role == "system") + systemPrompt = content; + else if (role == "user") + userPrompt = content; + } + } + + return (systemPrompt, userPrompt); + } + + public async ValueTask DisposeAsync() => await app.DisposeAsync(); +} + +/// +/// Boots the real host with a real Llm:Endpoint (a genuine ) +/// so LlmCopyWriter/LlmCallRing/LlmCallCauseCounters are the exact production +/// singletons AddGenWaveTts wires — nothing about the LLM pipeline is faked. Only hosted +/// services are removed (no Liquidsoap/Postgres background work during a test); every +/// Postgres-backed controller dependency a caller's own render might touch (e.g. +/// PersonaController) is left as its REAL, Lazy-backed registration, since a draft-fields +/// preview never forces any of them to actually connect (see Story196_LlmCallInspector.cs's own +/// original header note for the full rationale this extraction carries forward unchanged). +/// Llm:Model is fixed to for every caller — no fact so far has needed a +/// second value, so this stays a constant rather than a second constructor parameter. +/// +internal sealed class LlmCompletionsWebFactory(string llmEndpoint) : WebApplicationFactory +{ + internal const string Password = "test-password-llm-completions"; + internal const string Model = "test-model"; + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Development"); + builder.UseSetting("ConnectionStrings:Library", "Host=nowhere;Database=test"); + builder.UseSetting("Admin:Password", Password); + builder.UseSetting("Llm:Endpoint", llmEndpoint); + builder.UseSetting("Llm:Model", Model); + builder.ConfigureTestServices(services => services.RemoveAll()); + } +} diff --git a/tests/GenWave.Tts.Tests/Fakes/SingleHandlerHttpClientFactory.cs b/tests/GenWave.Tts.Tests/Fakes/SingleHandlerHttpClientFactory.cs new file mode 100644 index 00000000..ea4f64fb --- /dev/null +++ b/tests/GenWave.Tts.Tests/Fakes/SingleHandlerHttpClientFactory.cs @@ -0,0 +1,13 @@ +namespace GenWave.Tts.Tests.Fakes; + +/// +/// Hands every named-client request to the same fake handler (never disposed by the client) — the +/// shared home for what had drifted into two verbatim per-file copies +/// (Story189_LlmSingleFlightAndWarnDetail, Story350_ContextFactGate; T331 review +/// finding F6). Mirrors GenWave.Host.Tests.Fakes.SingleHandlerHttpClientFactory's own shape +/// one project over. +/// +public sealed class SingleHandlerHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory +{ + public HttpClient CreateClient(string name) => new(handler, disposeHandler: false); +} diff --git a/tests/GenWave.Tts.Tests/Specs/Gh291_DislikeTasteColor.cs b/tests/GenWave.Tts.Tests/Specs/Gh291_DislikeTasteColor.cs index fe555a68..7c0deeb3 100644 --- a/tests/GenWave.Tts.Tests/Specs/Gh291_DislikeTasteColor.cs +++ b/tests/GenWave.Tts.Tests/Specs/Gh291_DislikeTasteColor.cs @@ -60,7 +60,9 @@ static LlmCopyWriter BuildWriter(string endpoint) => new FakeActivePersonaAccessor(), new CapturingLogger(), new FakeTimeProvider(FixedLocalNow), - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); static string ExtractMessageContent(string body, string role) diff --git a/tests/GenWave.Tts.Tests/Specs/Story119_LlmCopyWriter.cs b/tests/GenWave.Tts.Tests/Specs/Story119_LlmCopyWriter.cs index bfadef22..81d88d69 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story119_LlmCopyWriter.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story119_LlmCopyWriter.cs @@ -49,7 +49,9 @@ static SegmentRequest TimeDateRequest() => new FakeActivePersonaAccessor(), logger, TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); return (writer, holder, logger); } @@ -149,7 +151,9 @@ public async Task InitializeAsync() new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); var track = new MediaItem( @@ -400,7 +404,7 @@ public sealed class ScenarioCallRingRecordsThePersonaName : IAsyncLifetime new FakeActivePersonaAccessor { Persona = persona }, new CapturingLogger(), TimeProvider.System, - ring, + new LlmCallRecorder(ring, new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); return (writer, ring); } diff --git a/tests/GenWave.Tts.Tests/Specs/Story121_PersonaPromptSections.cs b/tests/GenWave.Tts.Tests/Specs/Story121_PersonaPromptSections.cs index 5c56bec1..f250be8f 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story121_PersonaPromptSections.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story121_PersonaPromptSections.cs @@ -42,7 +42,9 @@ static LlmCopyWriter BuildWriter(string endpoint, FakeActivePersonaAccessor acce accessor, new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); static string ExtractSystemContent(string body) diff --git a/tests/GenWave.Tts.Tests/Specs/Story123_PersonaPreviewWriter.cs b/tests/GenWave.Tts.Tests/Specs/Story123_PersonaPreviewWriter.cs index b1e55ec8..9c1cee11 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story123_PersonaPreviewWriter.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story123_PersonaPreviewWriter.cs @@ -43,7 +43,9 @@ static SegmentRequest StationIdRequest() => new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); return (writer, holder); } diff --git a/tests/GenWave.Tts.Tests/Specs/Story124_EndpointLiveRepoint.cs b/tests/GenWave.Tts.Tests/Specs/Story124_EndpointLiveRepoint.cs index 9165fe6e..699e700d 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story124_EndpointLiveRepoint.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story124_EndpointLiveRepoint.cs @@ -115,7 +115,9 @@ public async Task LlmRepointRoutesTheNextBlurbToTheNewEndpointAndModel() new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); var before = await writer.WriteAsync(LeadInRequest(), CancellationToken.None); diff --git a/tests/GenWave.Tts.Tests/Specs/Story188_LlmDegradationModes.cs b/tests/GenWave.Tts.Tests/Specs/Story188_LlmDegradationModes.cs index f23b2a65..3b85dddf 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story188_LlmDegradationModes.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story188_LlmDegradationModes.cs @@ -93,7 +93,8 @@ static SegmentRequest BackAnnounceRequest() => var template = new TemplateCopyWriter(new PatterTemplateRenderer()); var llmWriter = new LlmCopyWriter( template, new FakeHttpClientFactory(), llmOptions, holder, new FakeActivePersonaAccessor(), - new CapturingLogger(), clock, new LlmCallRing(llmOptions), controller); + new CapturingLogger(), clock, + new LlmCallRecorder(new LlmCallRing(llmOptions), new LlmCallCauseCounters(clock)), controller); var writer = new DegradationGatedCopyWriter(controller, llmWriter, template, degradationOptions, clock); return (writer, template, controller, holder, health, clock, llmOptions); } @@ -435,7 +436,7 @@ public static async Task Explicit_operator_render_is_attempted_even_in_hard_mode IPersonaPreviewWriter previewWriter = new LlmCopyWriter( new TemplateCopyWriter(new PatterTemplateRenderer()), new FakeHttpClientFactory(), llmOptions, holder, new FakeActivePersonaAccessor(), new CapturingLogger(), clock, - new LlmCallRing(llmOptions), controller); + new LlmCallRecorder(new LlmCallRing(llmOptions), new LlmCallCauseCounters(clock)), controller); // When an operator triggers an explicit preview/test render var result = await previewWriter.WritePreviewAsync(LeadInRequest(), personaOverride: null, CancellationToken.None); diff --git a/tests/GenWave.Tts.Tests/Specs/Story189_LlmSingleFlightAndWarnDetail.cs b/tests/GenWave.Tts.Tests/Specs/Story189_LlmSingleFlightAndWarnDetail.cs index fc2c4828..26613293 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story189_LlmSingleFlightAndWarnDetail.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story189_LlmSingleFlightAndWarnDetail.cs @@ -56,7 +56,9 @@ public static async Task Concurrent_copy_renders_execute_sequentially() new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); // When their backend calls are traced (the handler counts overlapping SendAsync calls @@ -92,7 +94,9 @@ public async Task Failure_warning_includes_exception_status_and_context() new FakeActivePersonaAccessor(), logger, TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); var request = LeadInRequest(); @@ -153,11 +157,4 @@ protected override async Task SendAsync( } } } - - /// Hands every client the SAME shared handler (never disposed by the client) so - /// 's counters observe every call this writer makes. - sealed class SingleHandlerHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory - { - public HttpClient CreateClient(string name) => new(handler, disposeHandler: false); - } } diff --git a/tests/GenWave.Tts.Tests/Specs/Story193_PersonaPromptAssemblyAndClock.cs b/tests/GenWave.Tts.Tests/Specs/Story193_PersonaPromptAssemblyAndClock.cs index 7b0b2fc1..18635cde 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story193_PersonaPromptAssemblyAndClock.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story193_PersonaPromptAssemblyAndClock.cs @@ -69,7 +69,9 @@ static LlmCopyWriter BuildWriter( accessor, new CapturingLogger(), timeProvider ?? TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader(), stationClock); diff --git a/tests/GenWave.Tts.Tests/Specs/Story214_TasteBecomesAudible.cs b/tests/GenWave.Tts.Tests/Specs/Story214_TasteBecomesAudible.cs index e87731e5..3c1bca2c 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story214_TasteBecomesAudible.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story214_TasteBecomesAudible.cs @@ -58,7 +58,9 @@ static LlmCopyWriter BuildWriter(string endpoint) => new FakeActivePersonaAccessor(), new CapturingLogger(), new FakeTimeProvider(FixedLocalNow), - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); static string ExtractMessageContent(string body, string role) diff --git a/tests/GenWave.Tts.Tests/Specs/Story243_DjsHandOffAudibly.cs b/tests/GenWave.Tts.Tests/Specs/Story243_DjsHandOffAudibly.cs index 82972c44..33e7c17c 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story243_DjsHandOffAudibly.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story243_DjsHandOffAudibly.cs @@ -292,7 +292,9 @@ public async Task HardDegradationModeHandoffRendersNull() new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), controller); var gated = new DegradationGatedCopyWriter( controller, llmWriter, template, new TestOptionsMonitor(new DegradationOptions()), @@ -339,7 +341,9 @@ static LlmCopyWriter BuildWriter(string endpoint) => new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); [Fact] diff --git a/tests/GenWave.Tts.Tests/Specs/Story297_ContextSegmentsAir.cs b/tests/GenWave.Tts.Tests/Specs/Story297_ContextSegmentsAir.cs index 0fe7776b..70275178 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story297_ContextSegmentsAir.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story297_ContextSegmentsAir.cs @@ -252,7 +252,9 @@ public async Task HardDegradationModeContextSegmentRendersNull() new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), controller); var gated = new DegradationGatedCopyWriter( controller, llmWriter, template, new TestOptionsMonitor(new DegradationOptions()), @@ -366,7 +368,9 @@ static LlmCopyWriter BuildWriter(string endpoint) => new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); [Fact] diff --git a/tests/GenWave.Tts.Tests/Specs/Story298_OneFactPatterLane.cs b/tests/GenWave.Tts.Tests/Specs/Story298_OneFactPatterLane.cs index 0e22d6b9..02996855 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story298_OneFactPatterLane.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story298_OneFactPatterLane.cs @@ -57,7 +57,9 @@ static LlmCopyWriter BuildWriter( new FakeActivePersonaAccessor(), new CapturingLogger(), timeProvider ?? TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader(), stationClock: null, patterFactSource: patterFactSource); diff --git a/tests/GenWave.Tts.Tests/Specs/Story308_FlavorLineSharedSlot.cs b/tests/GenWave.Tts.Tests/Specs/Story308_FlavorLineSharedSlot.cs index 852646dd..3fed0665 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story308_FlavorLineSharedSlot.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story308_FlavorLineSharedSlot.cs @@ -59,7 +59,9 @@ static LlmCopyWriter BuildWriter( new FakeActivePersonaAccessor(), new CapturingLogger(), timeProvider ?? TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader(), stationClock: null, patterFactSource: patterFactSource, diff --git a/tests/GenWave.Tts.Tests/Specs/Story319_CopyFitsItsBreak.cs b/tests/GenWave.Tts.Tests/Specs/Story319_CopyFitsItsBreak.cs index 4aa97b32..609efa86 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story319_CopyFitsItsBreak.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story319_CopyFitsItsBreak.cs @@ -63,7 +63,7 @@ static LlmCopyWriter BuildWriter(string endpoint, int maxCopyChars) => new FakeActivePersonaAccessor { Persona = persona }, logger, TimeProvider.System, - ring, + new LlmCallRecorder(ring, new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader()); return (writer, ring, logger); } diff --git a/tests/GenWave.Tts.Tests/Specs/Story326_BoothWritesForTwo.cs b/tests/GenWave.Tts.Tests/Specs/Story326_BoothWritesForTwo.cs index 1644cef4..ba761c48 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story326_BoothWritesForTwo.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story326_BoothWritesForTwo.cs @@ -62,7 +62,7 @@ static CrosstalkExchangeRequest Request() => MaxCopyChars = maxCopyChars, }), crosstalkMonitor, - ring, + new LlmCallRecorder(ring, new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader(), logger, TimeProvider.System); @@ -371,9 +371,13 @@ public sealed class ScenarioTheExchangeFitsItsMoment : IAsyncLifetime [Fact] public async Task A_script_under_the_duration_target_is_accepted() { - // Given a validated script well under the 25s default (three short lines)... + // Given a validated script well under the shipped 50s default (three short lines) — + // built EXPLICITLY off CrosstalkOptions()'s own default (T333 review advisory A5), never + // this file's own BuildWriter convenience parameter default (25, an unrelated fixed test + // value several OTHER facts in this file use purely to prove the cap/word-budget SCALE + // with whatever target is configured) — so "the default" means one thing in this scenario. mock.ReplyContent = WellFormedReply; - var writer = BuildWriter(mock.BaseUri.ToString()); + var writer = BuildWriter(mock.BaseUri.ToString(), durationTargetSeconds: new CrosstalkOptions().DurationTargetSeconds); // When the spoken-duration estimate is computed... var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); @@ -382,11 +386,12 @@ public async Task A_script_under_the_duration_target_is_accepted() } [Fact] - public async Task The_duration_target_is_live_editable_with_a_25s_default() + public async Task The_duration_target_is_live_editable_with_the_shipped_default() { - // Given the shipped default (SPEC F127.4) — 200 chars / 15 chars-per-sec ~= 13.3s, which - // fits comfortably under it. - Assert.Equal(25, new CrosstalkOptions().DurationTargetSeconds); + // Given the shipped default (SPEC F127.4 as amended, PLAN T333) — 200 chars / 15 + // chars-per-sec ~= 13.3s, which fits comfortably under it. + var shippedDefault = new CrosstalkOptions().DurationTargetSeconds; + Assert.Equal(50, shippedDefault); mock.ReplyContent = string.Join('\n', new[] { @@ -394,7 +399,12 @@ public async Task The_duration_target_is_live_editable_with_a_25s_default() $"{CrosstalkScriptParser.NeighborTag}: {new string('b', 70)}", $"{CrosstalkScriptParser.HostTag}: {new string('c', 60)}", }); - var (writer, _, _, crosstalkMonitor) = BuildWriterWithRingAndLogger(mock.BaseUri.ToString()); + // Threads the SAME shippedDefault value read above (T333 review advisory A5) — never + // this file's own BuildWriter convenience default (still 25 elsewhere in this file), so + // the fact's own "shipped default" assertion and the writer it builds provably agree on + // what "the default" means. + var (writer, _, _, crosstalkMonitor) = BuildWriterWithRingAndLogger( + mock.BaseUri.ToString(), durationTargetSeconds: shippedDefault); var underDefault = await writer.WriteExchangeAsync(Request(), CancellationToken.None); Assert.IsType(underDefault); @@ -496,6 +506,9 @@ public async Task An_over_budget_line_rejects_the_whole_exchange() var discarded = Assert.IsType(result); Assert.Contains("per-line budget", discarded.Reason, StringComparison.Ordinal); + // SPEC F139.1 (PLAN T334 doc pickup): the reply came back and fit no length constraint — + // OverLength, the same bucket LlmCopyWriter's own gh-#277 family lands in. + Assert.Equal(LlmCallCause.OverLength, discarded.Cause); } [Fact] @@ -513,6 +526,8 @@ public async Task An_over_duration_script_rejects_whole() var discarded = Assert.IsType(result); Assert.Contains("exceeds", discarded.Reason, StringComparison.Ordinal); + // SPEC F139.1 (PLAN T334 doc pickup): an over-target duration estimate is OverLength too. + Assert.Equal(LlmCallCause.OverLength, discarded.Cause); } // T282 review finding (F2a): mutation-proven — deleting the both-speakers-present guards @@ -536,6 +551,25 @@ public async Task A_single_speaker_all_interjection_reply_is_discarded() var discarded = Assert.IsType(result); Assert.Contains(CrosstalkScriptParser.NeighborTag, discarded.Reason, StringComparison.Ordinal); + // SPEC F139.1 (PLAN T334 doc pickup): a missing required speaker turn is a shape problem + // — content arrived, it just never took the required shape — so this is MalformedResponse. + Assert.Equal(LlmCallCause.MalformedResponse, discarded.Cause); + } + + // SPEC F139.1 amendment (T330 review round 1, 2026-08-20 — the F135.5 precedent): the + // reviewer's own exhibit — a reply carrying MORE than MaxLines is not "empty" by any honest + // reading, so the whole parser-shape family (this branch included) moved off EmptyCompletion + // onto its own MalformedResponse bucket. + [Fact] + public async Task A_twelve_line_reply_is_a_malformed_response_not_an_empty_one() + { + mock.ReplyContent = string.Join('\n', Enumerable.Range(1, 12).Select(i => + $"{(i % 2 == 1 ? CrosstalkScriptParser.HostTag : CrosstalkScriptParser.NeighborTag)}: Line {i}.")); + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + Assert.Equal(LlmCallCause.MalformedResponse, Assert.IsType(result).Cause); } } @@ -565,6 +599,9 @@ public async Task A_completion_capped_by_max_tokens_is_discarded_even_though_it_ // Then the whole exchange is discarded — never aired truncated. var discarded = Assert.IsType(result); Assert.Contains("length", discarded.Reason, StringComparison.Ordinal); + // SPEC F139.1 (PLAN T334 doc pickup): a finish_reason: length truncation is OverLength — + // the reply came back but did not fit, the same family as a per-line/duration overrun. + Assert.Equal(LlmCallCause.OverLength, discarded.Cause); } [Fact] diff --git a/tests/GenWave.Tts.Tests/Specs/Story329_CrosstalkSupersedesTheGatedLanes.cs b/tests/GenWave.Tts.Tests/Specs/Story329_CrosstalkSupersedesTheGatedLanes.cs index 0d1fb466..c6c60480 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story329_CrosstalkSupersedesTheGatedLanes.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story329_CrosstalkSupersedesTheGatedLanes.cs @@ -51,7 +51,9 @@ static LlmCopyWriter BuildWriter( new FakeActivePersonaAccessor(), new CapturingLogger(), TimeProvider.System, - new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallRecorder( + new LlmCallRing(new TestOptionsMonitor(new LlmOptions())), + new LlmCallCauseCounters(TimeProvider.System)), new FakeDegradationModeReader(), stationClock: null, patterFactSource: patterFactSource, diff --git a/tests/GenWave.Tts.Tests/Specs/Story350_ContextFactGate.cs b/tests/GenWave.Tts.Tests/Specs/Story350_ContextFactGate.cs index 65357662..9d591977 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story350_ContextFactGate.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story350_ContextFactGate.cs @@ -10,67 +10,577 @@ // armor at the LlmCopyWriter seam: prompt asks (F138.5), checker enforces (F138.2), // ladder degrades re-ask-once → template (F138.4), never silence (F107.6). +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Text; +using System.Text.Json; +using GenWave.Core.Domain; +using GenWave.Tts; +using GenWave.Tts.Tests.Fakes; +using Xunit; + namespace GenWave.Tts.Tests.Specs; public static class FeatureContextFactGate { + // ── Shared fixture for the HTTP-driven ladder scenarios below (mirrors GenWave.Host.Tests' + // own Story353 BuildWriter idiom — the ONE constructor arg list every fact in + // ScenarioTheLadderDegrades/ScenarioGh434ExhibitEndToEnd/ScenarioNonContextKindIsNeverGated/ + // ScenarioEmptyFactBlockNeverGates/SadPathCheckerDiscipline shares) — drives the REAL + // LlmCopyWriter through a scripted FakeHttpMessageHandler rather than asserting on CopyClaims + // in isolation, so the ladder wiring at the LlmCopyWriter seam itself is what is under test. + + const string GhFactBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + + // gh-#434's own aired exhibit, unchanged: three fabrications in one line, all three F138.1 + // claim classes at once — a digit run ("6"), a condition word ("sunshine"), and a weekday + // ("saturday") — none of them supported by GhFactBlock. + const string PoisonedCopy = + "It feels like 6 degrees below freezing with plenty of sunshine and today is saturday here in the studio."; + + const string CleanCopy = "It's overcast today at 15 degrees with a high of 21 and a low of 12."; + + // 2026-08-15, station-local — a Saturday morning, so LlmPromptBuilder.BuildClockGuardLine's own + // output is a known, assertable literal ("It is Saturday morning...") rather than whatever day + // the machine running the test happens to land on. + static readonly DateTimeOffset FixedStationLocalNow = new(2026, 8, 15, 9, 0, 0, TimeSpan.Zero); + + static SegmentRequest ContextRequest(string? facts) => + new(SegmentKind.ContextSegment, "af_heart", "GenWave", Track: null, FixedStationLocalNow, "test-station", + PersonaName: null, CounterpartName: null, ContextFacts: facts); + + static SegmentRequest LeadInRequest() => + new(SegmentKind.LeadIn, "af_heart", "GenWave", + new MediaItem("m1", "/media/x.mp3", "Astral Plane", default, "Valerie June"), + FixedStationLocalNow, "test-station"); + + /// Builds a REAL against a fake completions handler that + /// scripts its reply BY CALL NUMBER (1-based) — also sees each call's + /// raw request body via the returned RequestBodies list, so a fact can inspect exactly + /// what the re-ask's own prompt said. pins the station + /// clock to so the F138.5 guard line is a known literal. + /// Logger is a real a fact can inspect for the T331 + /// review finding F3 WARN pin, rather than a value every caller must construct and discard. + static (LlmCopyWriter Writer, LlmCallRing Ring, List RequestBodies, CapturingLogger Logger) BuildWriter( + Func> respond, int timeoutSeconds = 5) + { + var bodies = new List(); + var handler = new FakeHttpMessageHandler(async (request, ct) => + { + var body = request.Content is null ? "" : await request.Content.ReadAsStringAsync(ct); + bodies.Add(body); + return await respond(bodies.Count, ct); + }); + var ring = new LlmCallRing(new TestOptionsMonitor(new LlmOptions())); + var logger = new CapturingLogger(); + var writer = new LlmCopyWriter( + new TemplateCopyWriter(new PatterTemplateRenderer()), + new SingleHandlerHttpClientFactory(handler), + new TestOptionsMonitor(new LlmOptions + { + Endpoint = "http://fake-llm.local", Model = "test-model", TimeoutSeconds = timeoutSeconds, + MaxCopyChars = 450, + }), + new LlmCopyStatusHolder(), + new FakeActivePersonaAccessor(), + logger, + TimeProvider.System, + new LlmCallRecorder(ring, new LlmCallCauseCounters(TimeProvider.System)), + new FakeDegradationModeReader(), + new FakeStationClockProvider(FixedStationLocalNow)); + return (writer, ring, bodies, logger); + } + + static Task Ok(string content) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = CompletionsBody(content), + }); + + static async Task DelayThenOk(TimeSpan delay, CancellationToken ct, string content = CleanCopy) + { + await Task.Delay(delay, ct); + return await Ok(content); + } + + static StringContent CompletionsBody(string content) => new( + JsonSerializer.Serialize(new { choices = new[] { new { message = new { content } } } }), + Encoding.UTF8, "application/json"); + + // Role-keyed, not positional (T331 review finding F6 — the Story119/121/123 precedent): looks + // up the message BY its own "role" field rather than trusting messages[0]/messages[1] to stay + // system-then-user forever. + static string ExtractSystemContent(string requestBodyJson) => ExtractMessageContent(requestBodyJson, "system"); + + static string ExtractUserContent(string requestBodyJson) => ExtractMessageContent(requestBodyJson, "user"); + + static string ExtractMessageContent(string requestBodyJson, string role) + { + using var doc = JsonDocument.Parse(requestBodyJson); + foreach (var message in doc.RootElement.GetProperty("messages").EnumerateArray()) + { + if (message.GetProperty("role").GetString() == role) + return message.GetProperty("content").GetString() ?? ""; + } + + return ""; + } + public static class ScenarioSupportedCopyPassesUntouched { - // Given the gh-#434 fact block / When copy claims only overcast, 15, 21, or 12 - [Fact(Skip = "pending T329 — CopyClaims checker does not exist yet")] - public static void Copy_with_only_supported_claims_passes_unchanged() => - Assert.Fail("pending T329: supported digits/conditions pass the checker with zero violations"); - - [Fact(Skip = "pending T329 — CopyClaims checker does not exist yet")] - public static void A_supported_claim_is_matched_case_insensitively() => - Assert.Fail("pending T329: 'Overcast' in copy matches 'overcast' in facts"); + // Given the gh-#434 fact block + const string FactBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + + // When copy claims only overcast, 15, 21, or 12 + [Fact] + public static void Copy_with_only_supported_claims_passes_unchanged() + { + const string copy = "It's overcast today at 15 degrees, with a high of 21 and a low of 12."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then supported digits/conditions pass the checker with zero violations + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_supported_claim_is_matched_case_insensitively() + { + const string copy = "Overcast conditions expected all day."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then 'Overcast' in copy matches 'overcast' in facts + Assert.Empty(result.Violations); + } } public static class ScenarioInventedClaimsAreCaught { - // Given the same fact block / When the copy fabricates - [Fact(Skip = "pending T329")] - public static void An_unsupported_digit_run_is_reported() => - Assert.Fail("pending T329: '6 degrees below' against 15/21/12 facts yields a digit violation naming '6'"); - - [Fact(Skip = "pending T329")] - public static void An_unsupported_condition_word_is_reported() => - Assert.Fail("pending T329: 'sunshine' against overcast facts yields a condition violation"); - - [Fact(Skip = "pending T329")] - public static void An_unsupported_weekday_is_reported() => - Assert.Fail("pending T329: 'today is saturday' with no weekday in facts yields a weekday violation"); + // Given the same fact block + const string FactBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + + // When the copy fabricates + [Fact] + public static void An_unsupported_digit_run_is_reported() + { + const string copy = "It feels like 6 degrees below freezing out there today."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then '6 degrees below' against 15/21/12 facts yields a digit violation naming '6' + Assert.Contains(result.Violations, v => v.Class == ClaimClass.DigitRun && v.Token == "6"); + } + + [Fact] + public static void An_unsupported_condition_word_is_reported() + { + const string copy = "Expect plenty of sunshine out there this afternoon."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then 'sunshine' against overcast facts yields a condition violation + Assert.Contains(result.Violations, v => v.Class == ClaimClass.ConditionWord && v.Token == "sunshine"); + } + + [Fact] + public static void An_unsupported_weekday_is_reported() + { + const string copy = "Today is saturday here in the studio."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then 'today is saturday' with no weekday in facts yields a weekday violation + Assert.Contains(result.Violations, v => v.Class == ClaimClass.Weekday && v.Token == "saturday"); + } + } + + // The present-frame narrowing (SPEC F138.3, amended T329 review round 1) governs CheckFacts's own + // weekday class exactly as it governs CheckClock's: only a weekday ASSERTED as the present frame + // is a claim at all. A displaced/recall/anticipatory reference is never extracted, so it can never + // be reported "unsupported" — the F138 when-in-doubt-PASS posture doing its job, not a fact-block + // leniency of its own. + public static class ScenarioWeekdayPresentFrameNarrowingAppliesToFacts + { + [Fact] + public static void A_song_title_naming_a_weekday_is_never_a_claim() + { + // Given a fact block that names no weekday at all + const string factBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + // When copy names a weekday only inside a song title, under no present-frame marker + const string copy = "Next up: Manic Monday."; + + var result = CopyClaims.CheckFacts(copy, factBlock); + + // Then "Monday" is never extracted (no "this/today is/it's/happy {weekday}" marker + // precedes it), so there is nothing to check for support — it passes + Assert.Empty(result.Violations); + } + } + + // T329 review round 3 regression pin: same curly-apostrophe fix as Story351's own pin, exercised + // through CheckFacts's own weekday class (F138.2) — see Story351's own remarks for why a model + // reaches this checker with a curly U+2019 apostrophe intact, not the SpeechText-folded straight + // one. + public static class ScenarioCurlyApostropheMarksAFactClaimToo + { + [Fact] + public static void A_curly_apostrophe_its_weekday_marker_is_checked_for_support() + { + // Given a fact block that names no weekday at all + const string factBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + // When copy asserts a weekday under the curly-quoted "it's" marker + const string copy = "It\u2019s saturday here in the studio."; + + var result = CopyClaims.CheckFacts(copy, factBlock); + + // Then the curly apostrophe still marks "saturday" as a present-frame claim, and with + // no weekday anywhere in the facts, it is unsupported + Assert.Contains(result.Violations, v => v.Class == ClaimClass.Weekday && v.Token == "saturday"); + } } public static class ScenarioTheLadderDegrades { - // Given a first completion that fails the gate (stub LLM serving poisoned copy - // through the production LlmCopyWriter seam — the entry-point scenario) - [Fact(Skip = "pending T331 — gate not wired at the LlmCopyWriter seam yet")] - public static void Exactly_one_reask_is_issued() => - Assert.Fail("pending T331: the writer retries once, never more"); - - [Fact(Skip = "pending T331")] - public static void The_reask_prompt_names_the_violating_claim() => - Assert.Fail("pending T331: the retry prompt contains the rejected claim text"); - - [Fact(Skip = "pending T331")] - public static void A_failing_reask_lands_on_the_template() => - Assert.Fail("pending T331: second violation airs the deterministic template line (F107.6 — never silence)"); - - [Fact(Skip = "pending T331")] - public static void The_guard_line_rides_the_prompt() => - Assert.Fail("pending T331: the system prompt carries the comma-free weekday/daypart guard line (F138.5)"); + [Fact] + public static async Task Exactly_one_reask_is_issued() + { + // Given a first completion that fails the gate (the gh-#434 exhibit) and a second that + // finally supports the facts — driven through the real production LlmCopyWriter seam + var (writer, _, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? PoisonedCopy : CleanCopy)); + + // When the render goes through WriteAsync -> RequestCleanedCompletionAsync + await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + + // Then exactly two completion calls were made — the rejected first, and ONE re-ask, never more + Assert.Equal(2, bodies.Count); + } + + [Fact] + public static async Task The_reask_prompt_names_the_violating_claim() + { + // Given the same poisoned-then-clean pair + var (writer, ring, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? PoisonedCopy : CleanCopy)); + + // When the render resolves + await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + + // Then the SECOND call's own user prompt names one of the rejected claims — the retry + // prompt contains the rejected claim text, not a bare "try again" — and it opens with + // plain declarative English, never a machine-looking "Re-ask:" label a model could echo + // straight back into its own reply (T331 review advisory F5). + var reaskPrompt = ExtractUserContent(bodies[1]); + Assert.Contains("sunshine", reaskPrompt, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("Re-ask:", reaskPrompt, StringComparison.Ordinal); + + // And the RING's own re-ask entry (T331 review finding F4a) — not just the wire — carries + // that same re-ask prompt: the newest ring record is the re-ask's own honest entry. + var newest = ring.Snapshot()[0]; + Assert.Contains("sunshine", newest.PromptUser ?? "", StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public static async Task A_failing_reask_lands_on_the_f107_floor() + { + // Given a first AND second completion that both violate the facts + var (writer, _, bodies, logger) = BuildWriter((_, _) => Ok(PoisonedCopy)); + + // When the render exhausts the ladder + var result = await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + + // Then it degrades to the EXISTING context-lane floor (SPEC F107.6's skip-never-silence + // posture) — the same template PatterTemplateRenderer already produces for a + // ContextSegment writer that degraded for any other reason, never a new floor invented + // for the gate — and never the still-violating LLM text. Still exactly one re-ask, never + // a retry storm. + Assert.Equal("Here's something worth knowing.", result.Text); + Assert.False(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + + // And the failure WARN names the REAL cause (T331 review finding F3, generalized wording + // PLAN T332) — the truth gate, and the still-unsupported claim — never the wrong-lever + // "empty or exceeded Llm:MaxCopyChars" wording a hygiene reject carries (that message + // sends an operator at settings this failure has nothing to do with). + Assert.Contains( + logger.Warnings, + warning => warning.Contains("truth gate", StringComparison.OrdinalIgnoreCase) + && warning.Contains("sunshine", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + logger.Warnings, warning => warning.Contains("empty or exceeded", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public static async Task The_guard_line_rides_the_prompt() + { + // Given an ordinary completion that never trips the gate at all + var (writer, _, bodies, _) = BuildWriter((_, _) => Ok(CleanCopy)); + + // When any patter prompt renders + await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + + // Then the system prompt carries the F138.5 guard line verbatim (weekday/daypart + // substituted for the pinned station clock) — comma-free prompt hardening on every + // render, gate or not. + var systemPrompt = ExtractSystemContent(bodies[0]); + Assert.Contains(LlmPromptBuilder.BuildClockGuardLine(FixedStationLocalNow), systemPrompt); + } + } + + public static class ScenarioGh434ExhibitEndToEnd + { + [Fact] + public static async Task The_pinned_exhibit_recovers_through_the_reask() + { + // Given the gh-#434 aired exhibit's own poisoned first reply against the real fact + // block, and a clean second reply + var (writer, _, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? PoisonedCopy : CleanCopy)); + + // When the render goes through the real ladder end to end + var result = await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + + // Then the clean re-ask airs — genuinely LLM-authored, never the invented first reply + Assert.Equal(CleanCopy, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + } + + [Fact] + public static async Task Both_calls_failing_the_gate_still_lands_on_the_floor() + { + // Given the SAME exhibit poisoning both the first reply and the re-ask + var (writer, ring, bodies, _) = BuildWriter((_, _) => Ok(PoisonedCopy)); + + // When the render exhausts the ladder + var result = await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + + // Then the fabricated copy never airs (the F107.6 floor), and BOTH calls left their own + // honest ring entry — the rejected first, and the re-ask that violated again — never one + // entry standing in for two calls. + Assert.False(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + Assert.Equal(2, ring.Snapshot().Count); + Assert.All(ring.Snapshot(), record => Assert.Equal(LlmCallCause.TruthGateReject, record.Cause)); + } + } + + public static class ScenarioNonContextKindIsNeverGated + { + [Fact] + public static async Task A_lead_in_with_fabricated_claims_is_never_fact_checked() + { + // Given a LeadIn request (not a context segment) whose only reply fabricates a claim + // that would trip CheckFacts if this kind were ever gated + var (writer, _, bodies, _) = BuildWriter((_, _) => Ok(PoisonedCopy)); + + // When it renders + var result = await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then the copy airs exactly as the model wrote it — F138.2 gates ContextSegment only, + // so no re-ask is even attempted for any other kind (the scope pin). + Assert.Equal(PoisonedCopy, result.Text); + Assert.Single(bodies); + + // And the narrowing to ContextSegment-only is pinned on the REAL LeadIn call's own + // system prompt (T331 review finding F2 — the reviewer's own mutation: narrowing + // production to context-only survived every fact here because none of them ever looked + // at what a non-gated kind's prompt actually carries) — the F138.5 guard line still rides + // it regardless, since that line is unconditional across every LLM-authored kind. + var systemPrompt = ExtractSystemContent(bodies[0]); + Assert.Contains(LlmPromptBuilder.BuildClockGuardLine(FixedStationLocalNow), systemPrompt); + } + } + + public static class ScenarioEmptyFactBlockNeverGates + { + [Fact] + public static async Task A_context_segment_with_no_fact_block_is_never_fact_checked() + { + // Given a ContextSegment request whose own ContextFacts is blank (an admin preview's + // typical case, per LlmPromptBuilder.BuildContextFactsLine's own remarks) and a reply + // fabricating a claim + var (writer, _, bodies, _) = BuildWriter((_, _) => Ok(PoisonedCopy)); + + // When it renders + var result = await writer.WriteAsync(ContextRequest(facts: null), CancellationToken.None); + + // Then CheckFacts is never even invoked — an empty fact block skips the gate entirely, + // so the copy airs unchecked with no re-ask. + Assert.Equal(PoisonedCopy, result.Text); + Assert.Single(bodies); + } } public static class SadPathCheckerDiscipline { - [Fact(Skip = "pending T329")] - public static void The_checker_is_pure() => - Assert.Fail("pending T329: reflection shows a static class, no instance state, no I/O (the F68.6 posture)"); + [Fact] + public static void The_checker_is_pure() + { + // Given the CopyClaims implementation + var type = typeof(CopyClaims); + + // When its shape is inspected (reflection): a static class with no instance state, + // exactly the SpeechText purity posture (F68.6) SPEC F138.1 names by name + var isStaticClass = type is { IsAbstract: true, IsSealed: true }; + var hasNoInstanceConstructors = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).Length == 0; + var hasNoInstanceFields = type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).Length == 0; + var checkFacts = type.GetMethod(nameof(CopyClaims.CheckFacts), BindingFlags.Public | BindingFlags.Static); + var checkClock = type.GetMethod(nameof(CopyClaims.CheckClock), BindingFlags.Public | BindingFlags.Static); + + // Then no instance state anywhere on the type, and both entry points are public + // static functions of their own parameters only — no I/O, no settings read + Assert.True(isStaticClass && hasNoInstanceConstructors && hasNoInstanceFields + && checkFacts is not null && checkClock is not null); + } + + [Fact] + public static async Task Budget_exhaustion_degrades_to_template_not_a_longer_hold() + { + // Given a first reply that BURNS MOST of this render's Llm:TimeoutSeconds budget before + // violating the facts (T331 review finding F1 — an instantly-answering first call left + // the shared budget entirely unconsumed, so this fact previously could not tell a + // correctly-SHARED clock apart from a re-ask that wrongly got its own fresh one: both + // shapes finish in about the same wall-clock time when call 1 is instant), and a re-ask + // endpoint that would take 10s regardless of which clock ends up bounding it. + var (writer, ring, bodies, _) = BuildWriter( + (call, ct) => call == 1 + ? DelayThenOk(TimeSpan.FromMilliseconds(1500), ct, PoisonedCopy) + : DelayThenOk(TimeSpan.FromSeconds(10), ct), + timeoutSeconds: 2); + var stopwatch = Stopwatch.StartNew(); + + // When the render's own timeout budget elapses mid-reask — RequestCleanedCompletionAsync's + // own timeoutCts, shared by BOTH calls, never a fresh clock for the re-ask + var result = await writer.WriteAsync(ContextRequest(GhFactBlock), CancellationToken.None); + stopwatch.Stop(); + + // Then the render degrades to the template rung — never a longer feeder hold than this + // render's own single 2s budget, ~1.5s of which the first call already spent, leaving + // only ~0.5s for the re-ask before the SHARED clock fires. Sharing correctly lands at + // ~2s total; the reviewer's own mutation (a fresh CreateLinkedTokenSource + CancelAfter + // for the re-ask, starting its OWN 2s from ~1.5s in) would run to ~3.5s instead — the + // bound is pinned at the MIDPOINT of the two (T331 pickup, PLAN T332: do NOT widen this + // toward 3.5s, and do NOT change either delay above — both would weaken the discriminant), + // so it reds under that mutation with room to spare on either side. + Assert.Equal("Here's something worth knowing.", result.Text); + Assert.False(result.FreshPerAiring); + Assert.True( + stopwatch.Elapsed < TimeSpan.FromSeconds(2.75), + $"took {stopwatch.Elapsed} - a re-ask given its OWN fresh timeout clock (not this " + + "render's one shared budget) would run past 2.75s (the midpoint of the correctly-shared " + + "~2s and the wrongly-fresh ~3.5s)"); + + // And the ring shows exactly what happened: the rejected first call, then the re-ask's + // own honest Timeout — TWO entries with TWO distinct dispatch times (T331 review finding + // F4b: the re-ask's own ring entry must carry its OWN StartedAt, never the first call's — + // a catch-all that reused the first call's timing would leave both entries stamped alike). + Assert.Equal(2, ring.Snapshot().Count); + var rejected = Assert.Single(ring.Snapshot(), record => record.Cause == LlmCallCause.TruthGateReject); + var timedOut = Assert.Single(ring.Snapshot(), record => record.Cause == LlmCallCause.Timeout); + Assert.NotEqual(rejected.StartedAt, timedOut.StartedAt); + } + } + + // Further pure-level pins (PLAN T329) — digit-run tokenization the design constraints called + // out explicitly: decimals, and range endpoints vs. an in-between value. Not gh-#434 exhibit + // text; a second fact block exercises the shapes the exhibit itself doesn't cover. + public static class ScenarioDigitRunTokenization + { + // Given a fact block with a decimal figure and a hyphenated range + const string FactBlock = "Ocean depth today: 108.8 meters. Coastal range 12-15°C."; + + [Fact] + public static void A_decimal_claim_matches_the_full_token() + { + const string copy = "Depth reads 108.8 meters this morning."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then "108.8" is one token (never split into "108" and "8") and it is literally + // present, so it is supported + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_truncated_digit_run_is_supported_by_the_decimal_prefix_rule() + { + const string copy = "Depth reads about 108 meters this morning."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then "108" is not its own token in the fact block, but "108.8" — a fact token — starts + // with "108." (amended T329 review round 1: the deliberate decimal-prefix allowance, + // kept explicitly; this is NOT the old, removed literal-substring rule) + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_range_endpoint_is_supported() + { + const string copy = "Highs near 15 along the coast."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then "15" is the range's own printed endpoint token, equal to a fact token + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_value_strictly_inside_a_stated_range_is_flagged() + { + const string copy = "Highs near 13 along the coast."; + + var result = CopyClaims.CheckFacts(copy, FactBlock); + + // Then "13" is never printed literally in "12-15°C" — the checker does not interpolate + // ranges, so this is reported (a documented, accepted conservative gap) + Assert.Contains(result.Violations, v => v.Class == ClaimClass.DigitRun && v.Token == "13"); + } + + [Fact] + public static void A_short_digit_claim_is_not_satisfied_by_a_containing_digit_run() + { + // Given a fact block whose only "6" lives embedded inside a larger number (gh-#434 + // hardened: the removed literal-substring rule let "6" hide inside "16") + const string factBlock = "Edmonton: overcast, 16°C. Today's high 21°C, low 12°C."; + const string copy = "It feels like 6 degrees below freezing out there today."; + + var result = CopyClaims.CheckFacts(copy, factBlock); + + // Then "6" is never its OWN digit-run token in the fact block (only "16", "21", "12" are), + // so whole-token matching still reports it — the robust #434 regression + Assert.Contains(result.Violations, v => v.Class == ClaimClass.DigitRun && v.Token == "6"); + } + + [Fact] + public static void A_short_digit_claim_is_not_satisfied_by_a_date_or_timestamp_block() + { + // Given a fact block carrying a full date/timestamp + const string factBlock = "Issued 2026-08-20 14:37 station-local."; + const string copy = "Just 1 more track before the break."; + + var result = CopyClaims.CheckFacts(copy, factBlock); + + // Then "1" is never its own digit-run token — it only ever appears embedded inside "14" + // — so whole-token matching reports it rather than falsely passing it + Assert.Contains(result.Violations, v => v.Class == ClaimClass.DigitRun && v.Token == "1"); + } + } + + public static class ScenarioMultiWordConditionPhrase + { + [Fact] + public static void A_condition_word_inside_a_multiword_phrase_still_matches() + { + // Given facts naming only the single word "cloudy" (no compound-phrase entry exists) + const string factBlock = "Forecast: cloudy, light breeze."; + // When copy uses it inside a larger phrase + const string copy = "Skies look partly cloudy this afternoon."; + + var result = CopyClaims.CheckFacts(copy, factBlock); - [Fact(Skip = "pending T331")] - public static void Budget_exhaustion_degrades_to_template_not_a_longer_hold() => - Assert.Fail("pending T331: an exhausted render budget skips the re-ask and airs the template"); + // Then extraction is word-by-word, so "cloudy" alone still matches + Assert.Empty(result.Violations); + } } } diff --git a/tests/GenWave.Tts.Tests/Specs/Story351_ClockClaimsGate.cs b/tests/GenWave.Tts.Tests/Specs/Story351_ClockClaimsGate.cs index 9e96c52c..dda8e08b 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story351_ClockClaimsGate.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story351_ClockClaimsGate.cs @@ -1,51 +1,666 @@ // STORY-351 — Patter can't lie about the clock (SPEC F138.3, F138.5 · PLAN T329/T332) // -// BDD specification — xUnit. PENDING until built (see Story350's header note). +// BDD specification — xUnit. The pure-checker-level pins below (PLAN T329) were built first; +// PLAN T332 wires CopyClaims.CheckClock into the real LlmCopyWriter seam for every LLM patter +// kind, so ScenarioClockLiesAreCaught and everything below it drive the REAL writer, not the +// pure checker in isolation — see Story350's own BuildWriter idiom, mirrored here. // // The gh-#438 aired exhibit is the pinned regression: "We're diving into a neon dusk on // this Saturday morning... Tonight we flip..." aired at Sunday 11:50 AM while the F117 // clock line named the correct instant in the prompt. The model isn't missing the // information; it ignores it — so the check is mechanical, on EVERY patter kind. +using System.Net; +using System.Text; +using System.Text.Json; +using GenWave.Core.Domain; +using GenWave.Tts; +using GenWave.Tts.Tests.Fakes; +using Xunit; + namespace GenWave.Tts.Tests.Specs; public static class FeatureClockClaimsGate { + // ── Shared fixture for the HTTP-driven wiring scenarios below (PLAN T332) — mirrors + // Story350_ContextFactGate's own BuildWriter idiom: drives the REAL LlmCopyWriter through a + // scripted FakeHttpMessageHandler rather than asserting on CopyClaims in isolation, so the + // T332 wiring at the LlmCopyWriter seam itself is what is under test, for kinds beyond + // ContextSegment. Every wiring scenario shares ONE clock — Sunday, 11:50 AM station-local — + // the exact instant the gh-#438 aired exhibit was rejected against, so "Saturday" and + // "tonight" are both known, assertable violations rather than whatever day the machine + // running the test happens to land on. + + static readonly DateTimeOffset FixedStationLocalNow = new(2026, 8, 16, 11, 50, 0, TimeSpan.Zero); + + static SegmentRequest LeadInRequest(string trackTitle = "Astral Plane") => + new(SegmentKind.LeadIn, "af_heart", "GenWave", + new MediaItem("m1", "/media/x.mp3", trackTitle, default, "Valerie June"), + FixedStationLocalNow, "test-station"); + + static SegmentRequest BackAnnounceRequest() => + new(SegmentKind.BackAnnounce, "af_heart", "GenWave", + new MediaItem("m1", "/media/x.mp3", "Astral Plane", default, "Valerie June"), + FixedStationLocalNow, "test-station"); + + /// + /// SPEC F107.3 fact block, or for a factless ContextSegment request — the + /// ONLY shape that reaches this method from PersonaController.Preview (SPEC F138.2's structural + /// exemption is for the FACTS half alone, review round-2 finding F1 — see + /// LlmCopyWriter.RequestCleanedCompletionAsync's own remarks; the clock half still applies). + /// + static SegmentRequest ContextRequest(string? facts) => + new(SegmentKind.ContextSegment, "af_heart", "GenWave", Track: null, FixedStationLocalNow, "test-station", + PersonaName: null, CounterpartName: null, ContextFacts: facts); + + /// Builds a REAL against a fake completions handler that + /// scripts its reply BY CALL NUMBER (1-based) — see Story350_ContextFactGate's own BuildWriter + /// for the full idiom this mirrors. rides along (review round-2 + /// finding F4) so a fact can pin the exact WARN wording produces on an + /// exhausted ladder, not just the airable outcome. + static (LlmCopyWriter Writer, List RequestBodies, CapturingLogger Logger) BuildWriter( + Func> respond) + { + var bodies = new List(); + var handler = new FakeHttpMessageHandler(async (request, ct) => + { + var body = request.Content is null ? "" : await request.Content.ReadAsStringAsync(ct); + bodies.Add(body); + return await respond(bodies.Count, ct); + }); + var ring = new LlmCallRing(new TestOptionsMonitor(new LlmOptions())); + var logger = new CapturingLogger(); + var writer = new LlmCopyWriter( + new TemplateCopyWriter(new PatterTemplateRenderer()), + new SingleHandlerHttpClientFactory(handler), + new TestOptionsMonitor(new LlmOptions + { + Endpoint = "http://fake-llm.local", Model = "test-model", TimeoutSeconds = 5, MaxCopyChars = 450, + }), + new LlmCopyStatusHolder(), + new FakeActivePersonaAccessor(), + logger, + TimeProvider.System, + new LlmCallRecorder(ring, new LlmCallCauseCounters(TimeProvider.System)), + new FakeDegradationModeReader(), + new FakeStationClockProvider(FixedStationLocalNow)); + return (writer, bodies, logger); + } + + static Task Ok(string content) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = CompletionsBody(content), + }); + + static StringContent CompletionsBody(string content) => new( + JsonSerializer.Serialize(new { choices = new[] { new { message = new { content } } } }), + Encoding.UTF8, "application/json"); + + static string ExtractUserContent(string requestBodyJson) + { + using var doc = JsonDocument.Parse(requestBodyJson); + foreach (var message in doc.RootElement.GetProperty("messages").EnumerateArray()) + { + if (message.GetProperty("role").GetString() == "user") + return message.GetProperty("content").GetString() ?? ""; + } + + return ""; + } + public static class ScenarioConsistentClaimsPass { // Given a clock line of Sunday 11:50 AM - [Fact(Skip = "pending T329 — clock predicate does not exist yet")] - public static void A_matching_weekday_claim_passes() => - Assert.Fail("pending T329: copy naming Sunday passes"); + static readonly DateTimeOffset Clock = new(2026, 8, 16, 11, 0, 0, TimeSpan.Zero); + + [Fact] + public static void A_matching_weekday_claim_passes() + { + const string copy = "Happy Sunday to everyone tuning in."; + + var result = CopyClaims.CheckClock(copy, Clock); + + // Then copy naming Sunday passes + Assert.Empty(result.Violations); + } - [Fact(Skip = "pending T329")] - public static void A_matching_daypart_claim_passes() => - Assert.Fail("pending T329: 'this morning' at 11:50 AM passes"); + [Fact] + public static void A_matching_daypart_claim_passes() + { + const string copy = "It's morning here in the studio."; + + var result = CopyClaims.CheckClock(copy, Clock); + + // Then 'it's morning' at 11:00 AM (the 05-11 morning window) passes + Assert.Empty(result.Violations); + } } public static class ScenarioClockLiesAreCaught { - [Fact(Skip = "pending T332 — check not wired across patter kinds yet")] - public static void A_wrong_weekday_in_a_lead_in_is_rejected() => - Assert.Fail("pending T332: 'this Saturday morning' against a Sunday clock line rejects with the weekday violation"); + // Wrong-weekday and wrong-daypart replies, both under a present-frame marker (SPEC + // F138.3's own closed marker set) so they are genuine claims, not displaced/recall + // mentions the checker deliberately lets pass. Sunday 11:50 AM (FixedStationLocalNow) + // makes "this Saturday" a weekday violation and "it's tonight" a daypart violation + // (tonight's own "night" category window is 21:00-04:59, nowhere near 11:00). + const string WrongWeekdayCopy = "This Saturday has been one for the books so let's keep it going."; + const string WrongDaypartCopy = "It's tonight and this one is going to hit just right."; + const string CleanReply = "Coming up next a classic from the vault."; + + [Fact] + public static async Task A_wrong_weekday_in_a_lead_in_is_rejected() + { + // Given a first lead-in reply asserting the wrong weekday and a clean second reply, + // driven through the real production LlmCopyWriter seam + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? WrongWeekdayCopy : CleanReply)); + + // When the render goes through WriteAsync -> RequestCleanedCompletionAsync + var result = await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then the clean re-ask airs, and exactly one re-ask fired — never the wrong-weekday text + Assert.Equal(CleanReply, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + } + + [Fact] + public static async Task A_wrong_daypart_is_rejected() + { + // Given a first lead-in reply asserting the wrong daypart and a clean second reply + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? WrongDaypartCopy : CleanReply)); + + var result = await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + Assert.Equal(CleanReply, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + } + + [Fact] + public static async Task A_back_announce_is_checked_like_a_lead_in() + { + // Given the SAME wrong-weekday shape, but for BackAnnounce instead of LeadIn — the + // gate applies to every LLM patter kind (F138.3), not the context lane or LeadIn alone + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? WrongWeekdayCopy : CleanReply)); + + var result = await writer.WriteAsync(BackAnnounceRequest(), CancellationToken.None); + + Assert.Equal(CleanReply, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + } + } + + // The truth-gate ladder is reachable from WritePreviewAsync too (review round-2 findings F1-F3, + // PLAN T332) — RequestCleanedCompletionAsync is the ONE seam both WriteAsync and + // WritePreviewAsync call, and CheckTruthGate/RunTruthGateLadderAsync gate on request.Kind alone, + // never on which caller reached them. These facts pin that reachability directly rather than + // leaving it as an inference from the production seam's own doc comments. They live HERE, not in + // Story123_PersonaPreviewWriter, because they are specifically about the T332 ladder's OWN + // preview reachability (a brand-new code path as of this task) — Story123 already owns the + // broader, pre-existing "preview never templates" contract (SPEC F35.6) and has no reason to grow + // clock/fact-claim fixtures of its own. Reuses this file's own call-scripted BuildWriter fixture + // rather than Story123's MockCompletionsServer idiom, since a poisoned-then-clean re-ask needs a + // reply that differs BY CALL NUMBER — exactly what BuildWriter already scripts and + // MockCompletionsServer's single mutable ReplyContent field does not. + public static class ScenarioPreviewReachesTheLadderToo + { + const string WrongWeekdayCopy = "This Saturday has been one for the books so let's keep it going."; + const string CleanReply = "Coming up next a classic from the vault."; + + [Fact] + public static async Task A_poisoned_lead_in_preview_reasks_once_and_returns_the_clean_text() + { + // Given a first preview reply asserting the wrong weekday and a clean second reply + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? WrongWeekdayCopy : CleanReply)); + + // When the preview goes through WritePreviewAsync -> RequestCleanedCompletionAsync + var result = await writer.WritePreviewAsync(LeadInRequest(), personaOverride: null, CancellationToken.None); + + // Then the clean re-ask airs as a Success, exactly one re-ask fired — the SAME ladder + // WriteAsync exercises, reachable from the preview seam too + var success = Assert.IsType(result); + Assert.Equal(CleanReply, success.Text); + Assert.Equal(2, bodies.Count); + } + + [Fact] + public static async Task An_exhausted_ladder_preview_names_the_truth_gate_not_empty_or_over_length() + { + // Given BOTH the first reply AND the re-ask asserting the wrong weekday + var (writer, bodies, _) = BuildWriter((_, _) => Ok(WrongWeekdayCopy)); + + // When the preview exhausts the ladder + var result = await writer.WritePreviewAsync(LeadInRequest(), personaOverride: null, CancellationToken.None); - [Fact(Skip = "pending T332")] - public static void A_wrong_daypart_is_rejected() => - Assert.Fail("pending T332: 'Tonight' at 11:50 AM rejects with the daypart violation"); + // Then Failed.Detail names the truth gate (review round-2 finding F2 — DescribeNullTextReason + // reused here), never the wrong-lever hygiene wording a preview used to report + // unconditionally for ANY null TextOf result before this fix + var failed = Assert.IsType(result); + Assert.Contains("truth gate", failed.Detail, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("empty or over-length", failed.Detail, StringComparison.OrdinalIgnoreCase); + Assert.Equal(2, bodies.Count); + } - [Fact(Skip = "pending T332")] - public static void A_back_announce_is_checked_like_a_lead_in() => - Assert.Fail("pending T332: the gate applies to every LLM patter kind, not the context lane alone"); + [Fact] + public static async Task A_factless_context_segment_preview_is_still_clock_checked() + { + // Given a ContextSegment preview with NO fact block at all — the ONLY shape that ever + // reaches this seam with ContextFacts null (PersonaController.Preview never supplies + // one; the AIR path never builds a factless ContextSegment request at all — + // Orchestrator.BuildContextSegmentRequestAsync's own blank-facts guard) — and a first + // reply asserting the wrong weekday, which a Kind-based whole-gate exemption (deleted, + // review round-2 finding F1) used to let straight through unchecked + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? WrongWeekdayCopy : CleanReply)); + + // When the preview renders + var result = await writer.WritePreviewAsync( + ContextRequest(facts: null), personaOverride: null, CancellationToken.None); + + // Then the clock half still gates it — one re-ask, clean text airs — proving the FACTS + // half's own "never even ask" (CheckTruthGate's factBlock-is-null guard) is scoped to the + // facts half alone, never the whole gate + var success = Assert.IsType(result); + Assert.Equal(CleanReply, success.Text); + Assert.Equal(2, bodies.Count); + } } public static class SadPathExemptionsHold { - [Fact(Skip = "pending T329")] - public static void A_track_title_naming_a_day_is_exempt() => - Assert.Fail("pending T329: 'Saturday Night Fever' on a Sunday does not trip the gate"); + [Fact] + public static void A_track_title_naming_a_day_is_exempt() + { + // Given a track title that is itself a present-frame-marked weekday mention + const string copy = "Coming up next, it's Saturday Night Fever."; + + // When checked against a Sunday clock, with that title supplied + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 16, 11, 0, 0, TimeSpan.Zero), trackTitle: "Saturday Night Fever"); + + // Then "it's Saturday" is a present-frame marker match that would otherwise violate + // (Saturday != Sunday), but it falls entirely inside the exempt title span, so + // the title mention never becomes a claim + Assert.Empty(result.Violations); + } + + [Fact] + public static void Copy_with_no_clock_claims_records_zero_rejections() + { + const string copy = "That was a great track, right off a classic album."; + + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 19, 15, 0, 0, TimeSpan.Zero)); + + // Then claim-free copy passes with no violations recorded + Assert.Empty(result.Violations); + } + } + + // Further pure-level pins (PLAN T329) — the hour->daypart boundary CheckClock derives its + // "expected" value from (SPEC F138.3), and a genuine mismatch's own shape (Expected carries the + // fix), pinned at the pure-checker level ahead of T332's wiring-level equivalents above. + public static class ScenarioDaypartBoundaries + { + [Fact] + public static void Hour_four_is_still_night() + { + // Given a present-frame-marked "night" claim + const string copy = "It's night out there."; + + // When checked at Monday 04:00 (the small-hours edge of the night window) + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 17, 4, 0, 0, TimeSpan.Zero)); + + // Then 04:00 is still inside night's own 21:00-04:59 window, so it passes + Assert.Empty(result.Violations); + } + + [Fact] + public static void Hour_five_becomes_morning() + { + // Given a present-frame-marked "morning" claim + const string copy = "Good morning, early risers."; + + // When checked at Monday 05:00 (the first hour of the morning window) + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 17, 5, 0, 0, TimeSpan.Zero)); + + // Then 05:00 is inside morning's own 05:00-11:59 window, so it passes + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_night_claim_at_a_morning_hour_is_rejected_naming_the_correct_daypart() + { + const string copy = "Good night, everybody."; + + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 17, 9, 0, 0, TimeSpan.Zero)); + + // Then 09:00 falls in no window "night" names (21:00-04:59), so this is a genuine + // violation naming the hour's own correct daypart + Assert.Contains(result.Violations, + v => v.Class == ClaimClass.Daypart && v.Token == "night" && v.Expected == "morning"); + } + } + + public static class ScenarioWeekdayMismatch + { + [Fact] + public static void A_wrong_weekday_claim_names_the_correct_weekday_as_expected() + { + const string copy = "This Saturday has been a wild ride."; + + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 16, 11, 0, 0, TimeSpan.Zero)); + + Assert.Contains(result.Violations, + v => v.Class == ClaimClass.Weekday && v.Token == "Saturday" && v.Expected == "Sunday"); + } + } + + // The full T329 review round 1 acceptance set for the amended F138.3 present-frame rule — every + // line here is realistic DJ patter shape, not a synthetic probe (the finding: bare-token matching + // rejected 5/10 of exactly these lines, all correct copy). Each PASS below would have wrongly + // violated under the pre-amendment bare-token rule; each VIOLATE still correctly fires under the + // narrowed one — both gh-#438 aired exhibits are among the VIOLATEs. + public static class ScenarioRealisticPatterAcceptanceSet + { + static readonly DateTimeOffset Sun11 = new(2026, 8, 16, 11, 0, 0, TimeSpan.Zero); + static readonly DateTimeOffset Mon9 = new(2026, 8, 17, 9, 0, 0, TimeSpan.Zero); + static readonly DateTimeOffset Mon21 = new(2026, 8, 17, 21, 0, 0, TimeSpan.Zero); + static readonly DateTimeOffset Sat21 = new(2026, 8, 15, 21, 0, 0, TimeSpan.Zero); + + [Fact] + public static void Anticipation_of_a_future_weekday_passes() + { + // "next {weekday}" is anticipation, never present-frame + var result = CopyClaims.CheckClock("Join us next Friday for the countdown.", Sun11); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void Recall_of_a_past_weekday_with_a_possessive_passes() + { + // "last {weekday}'s" is recall, and the possessive besides + var result = CopyClaims.CheckClock("Last Saturday's show was a wild ride.", Sun11); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_bare_daypart_mention_with_no_greeting_marker_passes() + { + // "coming up tonight" — no greeting/copula marker precedes "tonight" + var result = CopyClaims.CheckClock("Coming up tonight: two hours of soul.", Mon9); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void Recall_using_this_daypart_is_not_a_present_frame_claim() + { + // "this morning" is recall of an earlier hour, not a claim about the current one — the + // whole reason "this {daypart}" was deliberately excluded from the daypart marker set + var result = CopyClaims.CheckClock("We opened this morning with a classic.", Mon21); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void A_greeting_daypart_within_its_overlapping_window_passes() + { + // "good evening" at 21:00 — inside evening's own 17:00-22:59 window, not a lie just + // because 21:00 also falls in night's window + var result = CopyClaims.CheckClock("Good evening and welcome in.", Sat21); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void Tomorrow_prefixed_daypart_is_not_a_present_frame_claim() + { + // "tomorrow morning" — no greeting/copula marker precedes "morning" + var result = CopyClaims.CheckClock("Tomorrow morning we do it all again.", Sat21); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void Every_prefixed_weekday_is_never_a_claim() + { + // "every Saturday" — a recurring reference, explicitly outside the marker set + var result = CopyClaims.CheckClock("That track owned every Saturday night in 1978.", Sun11); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void On_a_prefixed_weekday_is_never_a_claim() + { + // "on a Tuesday" — a generic, non-present reference, explicitly outside the marker set + var result = CopyClaims.CheckClock("This one topped the charts on a Tuesday back in 1983.", Sun11); + + Assert.Empty(result.Violations); + } + + [Fact] + public static void This_weekday_still_violates_under_the_narrowed_rule() + { + // "this Saturday" IS in the marker set — a gh-#438 aired exhibit, still caught + var result = CopyClaims.CheckClock("We're diving into a neon dusk on this Saturday morning", Sun11); + + Assert.Contains(result.Violations, v => v.Class == ClaimClass.Weekday && v.Token == "Saturday"); + } + + [Fact] + public static void Today_is_weekday_still_violates_under_the_narrowed_rule() + { + // "today is {weekday}" IS in the marker set — the other gh-#438-family exhibit shape + var result = CopyClaims.CheckClock("Today is saturday", Sun11); + + Assert.Contains(result.Violations, v => v.Class == ClaimClass.Weekday && v.Token == "saturday"); + } + } + + // T329 review round 3 regression pin: a curly apostrophe (U+2019, RIGHT SINGLE QUOTATION MARK) + // must mark a present-frame "it's" claim exactly like the straight one (U+0027) does. + // SpeechText's own curly->straight fold runs AFTER this checker by design (this checker sees + // LlmCopyWriter's POST-hygiene, PRE-Normalize text), and LlmCopyWriter already treats U+2019 as + // an apostrophe elsewhere, so a model emitting "It’s Saturday" reaches this checker with the + // curly form intact — the marker regex must recognize it, not silently wave the claim through. + public static class ScenarioCurlyApostropheMarksAClaimToo + { + [Fact] + public static void A_curly_apostrophe_its_weekday_marker_still_violates() + { + const string copy = "It\u2019s Saturday, folks."; + + var result = CopyClaims.CheckClock(copy, new DateTimeOffset(2026, 8, 16, 11, 0, 0, TimeSpan.Zero)); + + // Then the curly-quoted "It\u2019s Saturday" marks a present-frame weekday claim exactly + // like "It's Saturday" would, and Saturday != Sunday still violates + Assert.Contains(result.Violations, v => v.Class == ClaimClass.Weekday && v.Token == "Saturday"); + } + } + + // T329 review round 3 advisory: ClaimVocabulary encodes the hour->daypart boundaries twice — + // CategoryForHour's own non-overlapping partition (used only to fill ClaimViolation.Expected) + // and HourIsInCategory's overlapping windows (used for the actual pass/fail decision). This pin + // holds the two in agreement across every hour of the day, so an edit to one boundary set and + // not the other fails loudly here rather than drifting silently apart. + public static class ScenarioHourCategoryAgreement + { + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + [InlineData(8)] + [InlineData(9)] + [InlineData(10)] + [InlineData(11)] + [InlineData(12)] + [InlineData(13)] + [InlineData(14)] + [InlineData(15)] + [InlineData(16)] + [InlineData(17)] + [InlineData(18)] + [InlineData(19)] + [InlineData(20)] + [InlineData(21)] + [InlineData(22)] + [InlineData(23)] + public static void Every_hours_own_canonical_category_is_inside_its_own_overlapping_window(int hour) + { + // Given the hour's single canonical category (the one Expected would carry on a mismatch) + var category = ClaimVocabulary.CategoryForHour(hour); + + // Then that same category's own overlapping window always includes the hour it was + // derived from — the two structures can never disagree about this hour + Assert.True(ClaimVocabulary.HourIsInCategory(category, hour)); + } + } + + // The gh-#438 aired exhibit, end to end, through the real production LlmCopyWriter seam (PLAN + // T332) — not the pure-checker-level pin ScenarioRealisticPatterAcceptanceSet already holds. + public static class ScenarioGh438ExhibitEndToEnd + { + // The exhibit's own two lines, unchanged: "this Saturday morning" still violates under the + // amended present-frame rule (a weekday claim); the bare "Tonight we flip" mention does not + // (no greeting/copula marker precedes it — ScenarioRealisticPatterAcceptanceSet's own + // A_bare_daypart_mention_with_no_greeting_marker_passes pins that half separately), so this + // exhibit's ladder trip is the weekday claim alone. + const string PoisonedExhibit = + "We're diving into a neon dusk on this Saturday morning. Tonight we flip the switch and keep it going."; + const string CleanReply = + "We're diving into a neon dusk this evening. Let's flip the switch and keep it going."; + + [Fact] + public static async Task The_pinned_exhibit_recovers_through_the_reask() + { + // Given the gh-#438 exhibit's own poisoned first reply, aired at Sunday 11:50 AM while + // the F117 clock line named the correct instant, and a clean second reply + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? PoisonedExhibit : CleanReply)); + + // When the render goes through the real ladder end to end + var result = await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then the clean re-ask airs — genuinely LLM-authored — never the exhibit's own + // invented "this Saturday morning" + Assert.Equal(CleanReply, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + } + } + + // The composite check (SPEC F138.2 + F138.3, PLAN T332): a ContextSegment reply violating BOTH + // claim families gets exactly ONE re-ask naming both — never two chained ladder runs (the T331 + // reviewer ruling; see LlmCopyWriter.CheckTruthGate's own remarks). + public static class ScenarioCompositeContextChecksBothFamilies + { + const string FactBlock = "Edmonton: overcast, 15°C. Today's high 21°C, low 12°C."; + + // gh-#434's own exhibit shape, unchanged: a digit run ("6") and a condition word + // ("sunshine") the fact block never supports, PLUS — at this file's Sunday clock — "today + // is saturday" is now ALSO a clock violation, not merely a fact-block one: the SAME token + // trips BOTH CheckFacts (no weekday anywhere in the fact block) and CheckClock (the actual + // day is Sunday, not Saturday). + const string PoisonedCopy = + "It feels like 6 degrees below freezing with plenty of sunshine and today is saturday here in the studio."; + const string CleanReply = "It's overcast today at 15 degrees with a high of 21 and a low of 12."; + + [Fact] + public static async Task A_context_reply_violating_both_families_gets_one_reask_naming_both() + { + // Given the composite poisoned reply and a clean second reply + var (writer, bodies, _) = BuildWriter((call, _) => Ok(call == 1 ? PoisonedCopy : CleanReply)); + + // When the render goes through the real ladder end to end + var result = await writer.WriteAsync(ContextRequest(FactBlock), CancellationToken.None); + + // Then the clean re-ask airs, and exactly ONE re-ask fired for BOTH claim families — + // never a second, chained ladder run + Assert.Equal(CleanReply, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); + + // And that single re-ask's own prompt names a violation from EACH family: the facts + // half ("sunshine", never in the fact block) and the clock half (the correct weekday + // named as the FIX, "actually Sunday" — the clock-violation clause shape, + // LlmPromptBuilder.DescribeViolationForReask's own Expected-is-set branch, distinct + // from a bare "Sunday" mention, which the ambient F71.8 clock line + // (LlmPromptBuilder.BuildStationClockLine) would already put in every prompt regardless + // of any violation at all) — proof the two checks composed into one gate/re-ask cycle + // rather than needing two. + var reaskPrompt = ExtractUserContent(bodies[1]); + Assert.Contains("sunshine", reaskPrompt, StringComparison.OrdinalIgnoreCase); + Assert.Contains("actually Sunday", reaskPrompt, StringComparison.Ordinal); + } + } + + // The track-title exemption (SPEC F138.3), through the real writer rather than CopyClaims in + // isolation (SadPathExemptionsHold above already pins the pure-checker level). + public static class SadPathTrackTitleExemptionThroughTheRealWriter + { + const string TrackTitleMention = "Coming up next, it's Saturday Night Fever."; + + [Fact] + public static async Task A_lead_in_naming_its_own_track_title_is_not_rejected() + { + // Given a lead-in whose track IS "Saturday Night Fever", and a reply that names it + // under a present-frame marker ("it's Saturday Night Fever") — a claim that would + // otherwise violate this file's Sunday clock + var (writer, bodies, _) = BuildWriter((_, _) => Ok(TrackTitleMention)); + + // When it renders through the real writer + var result = await writer.WriteAsync(LeadInRequest(trackTitle: "Saturday Night Fever"), CancellationToken.None); + + // Then the title mention is exempt — the copy airs on the FIRST call, no re-ask ever fired + Assert.Equal(TrackTitleMention, result.Text); + Assert.True(result.FreshPerAiring); + Assert.Single(bodies); + } + } + + // The per-kind floor (PLAN T332 investigation): LeadIn/BackAnnounce have no F107.6-style + // skip-never-silence guard the way ContextSegment/SignOff/SignOn do (TtsSegmentSource's own + // non-fresh-copy guard names only those three) — a still-violating re-ask degrades to + // PatterTemplateRenderer's deterministic template instead, and that template DOES reach air. + public static class SadPathReaskStillViolatingLandsOnTheTemplate + { + const string WrongWeekdayCopy = "This Saturday has been one for the books so let's keep it going."; + + [Fact] + public static async Task A_lead_in_whose_reask_still_violates_lands_on_the_template() + { + // Given BOTH the first reply AND the re-ask asserting the wrong weekday + var (writer, bodies, logger) = BuildWriter((_, _) => Ok(WrongWeekdayCopy)); + + // When the render exhausts the ladder + var result = await writer.WriteAsync(LeadInRequest(), CancellationToken.None); + + // Then it degrades to the LeadIn template floor (PatterTemplateRenderer.Expand's own + // arm, which renders fixed prose with no weekday/daypart word in it for THIS request — + // it interpolates the track's own title/artist verbatim, so it is the template's fixed + // wording, not a guarantee about arbitrary track metadata, that the floor actually relies + // on) — never the still-violating LLM text, and never silence either: unlike + // ContextSegment/SignOff/SignOn, this template DOES reach air for LeadIn. Still exactly + // one re-ask, never a retry storm. + Assert.Equal("Coming up: Astral Plane by Valerie June.", result.Text); + Assert.False(result.FreshPerAiring); + Assert.Equal(2, bodies.Count); - [Fact(Skip = "pending T329")] - public static void Copy_with_no_clock_claims_records_zero_rejections() => - Assert.Fail("pending T329: claim-free copy passes with no violations recorded"); + // And the failure WARN names this as a WRONG-DAY claim, never an "unsupported claim" + // (review round-2 finding F4 — LlmCopyWriter.DescribeViolationForLog's three-way split + // was unpinned: a clock violation carries ClaimViolation.Expected, a fact-block violation + // never does, and only THIS fact proves the Expected-set branch actually fires rather + // than every violation reading as a generic "unsupported claim"). + Assert.Contains( + logger.Warnings, warning => warning.Contains("wrong-day claim", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + logger.Warnings, warning => warning.Contains("unsupported claim", StringComparison.OrdinalIgnoreCase)); + } } } diff --git a/tests/GenWave.Tts.Tests/Specs/Story352_BanterTruth.cs b/tests/GenWave.Tts.Tests/Specs/Story352_BanterTruth.cs index 19d1fded..e095eb0a 100644 --- a/tests/GenWave.Tts.Tests/Specs/Story352_BanterTruth.cs +++ b/tests/GenWave.Tts.Tests/Specs/Story352_BanterTruth.cs @@ -1,6 +1,8 @@ // STORY-352 — Banter stays fictional, never false (SPEC F138.6, F127.4 as amended · PLAN T333) // -// BDD specification — xUnit. PENDING until built (see Story350's header note). +// BDD specification — xUnit, LIVE as of T333. Drives the REAL CrosstalkScriptWriter through a +// scripted completions server (Story326_BoothWritesForTwo's own writer-harness idiom), never +// CopyClaims/CrosstalkScriptParser in isolation, so the actual T333 wiring is what is under test. // // The ruling (Dean, 2026-08-20): real-world verifiables are forbidden — frequency/call-sign // shapes, dates, weather words, clock lies — and mechanically enforced through F127.4's @@ -11,42 +13,349 @@ namespace GenWave.Tts.Tests.Specs; +using GenWave.Core.Domain; +using GenWave.Tts.Tests.Fakes; + public static class FeatureBanterTruth { + // ── Shared fixtures ───────────────────────────────────────────────────── + + static readonly PersonaCard HostCard = MakeCard( + "Neon Nightowl", "Neon Nightowl spins moody late-night sets deep into the small hours."); + + static readonly PersonaCard NeighborCard = MakeCard( + "Daybreak Dana", "Daybreak Dana brings bright upbeat energy straight off the morning show."); + + // A fixed Monday noon (verified: 2026-08-17 is a Monday) — every clock-claim fixture below is + // written against this ONE known instant, mirroring Story351_ClockClaimsGate's own + // FixedStationLocalNow idiom, so "Friday" is a known, assertable violation rather than + // whatever day the machine running the test happens to land on. + static readonly DateTimeOffset FixedStationLocalNow = new(2026, 8, 17, 12, 0, 0, TimeSpan.Zero); + + static readonly string FrequencyShapeReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: Did you catch us at 98.7 FM last night.", + $"{CrosstalkScriptParser.NeighborTag}: Every single time I tune in.", + $"{CrosstalkScriptParser.HostTag}: That is the spirit right there.", + }); + + // The task-pinned edge (PLAN T333 review round 1, probe-proven F1): a real FM frequency spoken + // as a bare INTEGER, no decimal point — real FM frequencies are commonly said this way, and a + // decimal-required rule let this class of claim through as a false PASS (the exact F138.6 harm: + // an airable fabricated broadcast fact). + static readonly string IntegerFrequencyShapeReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: Radio 101 FM keeps us on the air every day.", + $"{CrosstalkScriptParser.NeighborTag}: That is the frequency that never lets me down.", + $"{CrosstalkScriptParser.HostTag}: Long may it broadcast.", + }); + + static readonly string CallSignShapeReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: Tune into KXRT for more of this energy.", + $"{CrosstalkScriptParser.NeighborTag}: That station never lets me down.", + $"{CrosstalkScriptParser.HostTag}: KXRT is the name everyone remembers.", + }); + + static readonly string WeatherShapeReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: It is so sunny outside for the drive today.", + $"{CrosstalkScriptParser.NeighborTag}: Perfect for a road trip playlist.", + $"{CrosstalkScriptParser.HostTag}: Let us keep the good vibes rolling.", + }); + + static readonly string DateShapeReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: Mark your calendars for August 20 because it is a big one.", + $"{CrosstalkScriptParser.NeighborTag}: I already circled it in red.", + $"{CrosstalkScriptParser.HostTag}: See you all right here when it comes around.", + }); + + static readonly string ClockLieReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: Happy Friday to everyone listening in.", + $"{CrosstalkScriptParser.NeighborTag}: Best day of the week hands down.", + $"{CrosstalkScriptParser.HostTag}: Let us make it count together.", + }); + + // A recurring invented character plus a running gag — no real frequency, call sign, place, + // weather, or date anywhere, and no clock claim either (SPEC F138.6's "fictional lore passes + // untouched" half). + static readonly string FictionalLoreReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: Is Gary the Ghost DJ still haunting the night shift again.", + $"{CrosstalkScriptParser.NeighborTag}: Gary never skips his overnight howl before sign off.", + $"{CrosstalkScriptParser.HostTag}: Somewhere out there Gary is smiling right now.", + }); + + // The task-pinned edge: a TIME reference shaped just like a frequency claim ("digit space AM") + // must never trip the frequency shape — only a 3-4 digit AM frequency does (see + // CrosstalkScriptParser.FrequencyRx's own remarks). + static readonly string NineAmReply = string.Join('\n', new[] + { + $"{CrosstalkScriptParser.HostTag}: We're back on the air again at 9 AM sharp.", + $"{CrosstalkScriptParser.NeighborTag}: Set an alarm because I will be listening.", + $"{CrosstalkScriptParser.HostTag}: That is exactly the plan for both of us.", + }); + + static PersonaCard MakeCard(string name, string soul) => + new(PersonaCard.CurrentSchemaVersion, name, Tagline: "", soul, Quirks: [], + new VoiceSpec("kokoro", "af_heart", 1.0, "en"), EnergyDisposition: 0, Lore: [], Corrections: []); + + static CrosstalkExchangeRequest Request() => + new(HostCard, NeighborCard, "GenWave", ShowName: "Night Shift", Daypart: "late night", + StationLocalNow: FixedStationLocalNow); + + /// Mirrors Story326_BoothWritesForTwo's own BuildWriterWithRingAndLogger idiom — the + /// house crosstalk-writer spec harness, driving the REAL CrosstalkScriptWriter end to end. + static CrosstalkScriptWriter BuildWriter(string endpoint) + { + var ring = new LlmCallRing(new TestOptionsMonitor(new LlmOptions())); + return new CrosstalkScriptWriter( + new FakeHttpClientFactory(), + new TestOptionsMonitor(new LlmOptions + { + Endpoint = endpoint, + Model = "test-model", + TimeoutSeconds = 5, + MaxCopyChars = 300, + }), + new TestOptionsMonitor(new CrosstalkOptions()), + new LlmCallRecorder(ring, new LlmCallCauseCounters(TimeProvider.System)), + new FakeDegradationModeReader(), + new CapturingLogger(), + TimeProvider.System); + } + + static string ExtractSystemPrompt(string body) + { + using var doc = System.Text.Json.JsonDocument.Parse(body); + foreach (var message in doc.RootElement.GetProperty("messages").EnumerateArray()) + { + if (message.GetProperty("role").GetString() == "system") + return message.GetProperty("content").GetString() ?? ""; + } + + return ""; + } + + // ── Verifiables discard the script ───────────────────────────────────── + public static class ScenarioVerifiablesDiscardTheScript { - [Fact(Skip = "pending T333 — F127.4 truth discard reasons not built yet")] - public static void A_frequency_shape_discards_with_the_verifiable_reason() => - Assert.Fail("pending T333: a line claiming '98.7 FM' discards the exchange (fail-closed skip, no salvage)"); + [Fact] + public static async Task A_frequency_shape_discards_with_the_verifiable_reason() + { + // Given a script that names a real-shaped FM frequency ("98.7 FM")... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = FrequencyShapeReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + // When the writer validates it... + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); - [Fact(Skip = "pending T333")] - public static void A_call_sign_shape_discards() => - Assert.Fail("pending T333: a K/W-prefixed call sign claim discards the exchange"); + // Then the whole exchange discards (fail-closed skip, no salvage), stamped as a truth + // rejection — never a shape/malformed one, since the script parsed cleanly. + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Contains("98.7 FM", discarded.Reason, StringComparison.Ordinal); + } - [Fact(Skip = "pending T333")] - public static void A_weather_claim_discards() => - Assert.Fail("pending T333: a condition-word claim discards the exchange"); + [Fact] + public static async Task An_integer_FM_frequency_shape_also_discards() + { + // Given a script naming a real FM frequency spoken WITHOUT a decimal ("Radio 101 FM") — + // requiring a decimal would let this class of real, commonly-spoken frequency claim + // through as a false PASS (SPEC F138.6's own harm: an airable fabricated broadcast fact). + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = IntegerFrequencyShapeReply; + var writer = BuildWriter(mock.BaseUri.ToString()); - [Fact(Skip = "pending T333")] - public static void A_clock_lie_discards() => - Assert.Fail("pending T333: a wrong-weekday line against the clock context discards with the clock reason"); + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Contains("101 FM", discarded.Reason, StringComparison.Ordinal); + } + + [Fact] + public static async Task A_call_sign_shape_discards() + { + // Given a script that names a K/W-prefixed call sign shape... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = CallSignShapeReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Contains("KXRT", discarded.Reason, StringComparison.Ordinal); + } + + [Fact] + public static async Task A_weather_claim_discards() + { + // Given a script that names a real weather-condition word... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = WeatherShapeReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Contains("sunny", discarded.Reason, StringComparison.Ordinal); + } + + [Fact] + public static async Task A_date_claim_discards() + { + // Given a script that names a real-shaped date ("August 20")... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = DateShapeReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Contains("August 20", discarded.Reason, StringComparison.Ordinal); + } + + [Fact] + public static async Task A_clock_lie_discards() + { + // Given a wrong-weekday line against the clock context (station-local is a Monday, the + // script claims "Happy Friday")... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = ClockLieReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + // Then it discards with the clock reason — CopyClaims.CheckClock, the SAME T329 + // predicate every other patter kind shares, judged against the SAME generation-time + // instant the prompt's own clock line stated. + var discarded = Assert.IsType(result); + Assert.Equal(LlmCallCause.TruthGateReject, discarded.Cause); + Assert.Contains("Friday", discarded.Reason, StringComparison.Ordinal); + Assert.Contains("Monday", discarded.Reason, StringComparison.Ordinal); + } } + // ── Fictional lore passes ─────────────────────────────────────────────── + public static class ScenarioFictionalLorePasses { - [Fact(Skip = "pending T333")] - public static void An_invented_recurring_character_passes() => - Assert.Fail("pending T333: no lore-shaped rejection exists — invented characters validate clean"); + [Fact] + public static async Task An_invented_recurring_character_passes() + { + // Given a script built entirely from invented lore — a recurring character and a + // running gag, no real-world verifiable of any kind... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = FictionalLoreReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + // When the writer validates it... + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + // Then it validates clean — no lore-shaped rejection exists. + Assert.IsType(result); + } - [Fact(Skip = "pending T333")] - public static void The_narrow_clause_rides_the_banter_prompt() => - Assert.Fail("pending T333: the prompt forbids real-world verifiables and explicitly allows fictional lore"); + [Fact] + public static async Task The_narrow_clause_rides_the_banter_prompt() + { + // Given any generation attempt... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = FictionalLoreReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + // Then the system prompt forbids real-world verifiables and explicitly allows + // fictional lore, beside the banter prompt's other style rules. + var prompt = ExtractSystemPrompt(mock.Requests[0].Body); + Assert.Contains("Never mention a real radio frequency.", prompt, StringComparison.Ordinal); + Assert.Contains("Never mention a real call sign.", prompt, StringComparison.Ordinal); + Assert.Contains("Never mention a real place name.", prompt, StringComparison.Ordinal); + Assert.Contains("Never mention a real weather condition.", prompt, StringComparison.Ordinal); + Assert.Contains("Never mention a real date.", prompt, StringComparison.Ordinal); + Assert.Contains( + "Invented recurring characters running gags and station mythology are welcome.", + prompt, StringComparison.Ordinal); + } + + [Fact] + public static async Task The_f138_5_clock_guard_line_also_rides_the_banter_prompt() + { + // Given any generation attempt (PLAN T333 review round 1, probe-proven F2) — crosstalk + // was the only patter kind whose F138.3 clock check ran with no prompt-side guard at + // all, silently discarding a clock lie the model was never told not to make... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = FictionalLoreReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + // Then the SAME F138.5 guard line every other patter prompt already carries rides the + // crosstalk prompt too, stated against the SAME station-local instant this request + // carries (FixedStationLocalNow: Monday, 12:00 -> "afternoon"). + var prompt = ExtractSystemPrompt(mock.Requests[0].Body); + Assert.Contains( + "It is Monday afternoon. Never name another day or time of day.", + prompt, StringComparison.Ordinal); + } } + // ── The ratified target ───────────────────────────────────────────────── + public static class ScenarioTheRatifiedTarget { - [Fact(Skip = "pending T333")] + [Fact] public static void The_duration_default_is_fifty_seconds() => - Assert.Fail("pending T333: with no override, Crosstalk:DurationTargetSeconds reads 50 (F127.4 as amended)"); + // With no override, Crosstalk:DurationTargetSeconds reads 50 (F127.4 as amended, + // ratified from two days of live convergence — the 25s paper-audition posture retires). + Assert.Equal(50, new CrosstalkOptions().DurationTargetSeconds); + } + + // ── Edge pins (PLAN T333 review guidance) ─────────────────────────────── + + public static class ScenarioEdgeCasesArePinned + { + [Fact] + public static async Task A_time_like_9_AM_does_not_trip_the_frequency_shape() + { + // Given a script that mentions a CLOCK TIME shaped like a frequency claim ("9 AM") — + // the exact edge the frequency shape must NOT catch, since a time-of-day mention is + // not a station's dial position. + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = NineAmReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + // Then it validates clean — "9 AM" is never mistaken for "98.7 FM". + Assert.IsType(result); + } + + [Fact] + public static async Task The_discard_reason_names_what_was_wrong_not_an_internal_code_path() + { + // Given any truth-gate discard... + await using var mock = await MockCompletionsServer.StartAsync(); + mock.ReplyContent = FrequencyShapeReply; + var writer = BuildWriter(mock.BaseUri.ToString()); + + var result = await writer.WriteExchangeAsync(Request(), CancellationToken.None); + + // Then the reason is operator-honest: it names the offending text in plain words, never + // an internal enum or type name an operator reading /api/llm-calls would not recognize. + var discarded = Assert.IsType(result); + Assert.Contains("real-world radio frequency", discarded.Reason, StringComparison.Ordinal); + Assert.DoesNotContain("TruthGateReject", discarded.Reason, StringComparison.Ordinal); + Assert.DoesNotContain(nameof(CrosstalkScriptParser), discarded.Reason, StringComparison.Ordinal); + } } }