diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index be6f9a8..0d15288 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,14 +8,14 @@ }, "metadata": { "description": "Plugins from classifier.dev: zero-shot text classification with a calibrated confidence per answer, no API key.", - "version": "1.0.0" + "version": "1.0.1" }, "plugins": [ { "name": "classifier", "source": "./plugins/classifier", "description": "Sort up to 1,000 texts into your own labels in one call, with a calibrated confidence per answer. No API key.", - "version": "1.0.0", + "version": "1.0.1", "author": { "name": "Michael Ryaboy", "email": "contact@classifier.dev", "url": "https://classifier.dev" }, "homepage": "https://classifier.dev/mcp-setup", "repository": "https://github.com/mrmps/classifier-dev", diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index eb2a1c1..f16bf3d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,6 +7,13 @@ permissions: jobs: check: runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: spending-test + ports: ["5432:5432"] + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 @@ -23,6 +30,14 @@ jobs: NEON_MIGRATION_TEST_URL: ${{ secrets.NEON_MIGRATION_TEST_URL }} - run: node --test working-directory: cli + - name: Verify paid reservations with concurrent PostgreSQL connections + run: bun test tests/spending.e2e.test.ts + env: + POSTGRES_TEST_URL: postgres://postgres:spending-test@localhost:5432/postgres + - name: Verify document admission under concurrent PostgreSQL requests + run: bun e2e/document-admission.ts + env: + POSTGRES_TEST_URL: postgres://postgres:spending-test@localhost:5432/postgres - name: Render secret-free build configuration env: CLOUDFLARE_ACCOUNT_ID: "00000000000000000000000000000000" @@ -31,3 +46,56 @@ jobs: run: node .github/render-wrangler.mjs - name: Build application run: npm run build + - name: Verify chat usage and authenticated analytics + run: npm run test:e2e:chat-analytics + - name: Save chat analytics evidence + uses: actions/upload-artifact@v4 + with: + name: chat-analytics-e2e + path: captures/chat-analytics.json + - name: Verify public chat and classification tools + run: npm run test:e2e:chat + - name: Save chat verification evidence + uses: actions/upload-artifact@v4 + with: + name: chat-e2e + path: captures/chat-e2e.json + - name: Verify spending through the built Worker and real Durable Objects + run: npm run test:e2e:spending + - name: Verify classification usage responses + run: npm run test:e2e:usage + - name: Verify long-context screening and judgment + run: npm run test:e2e:long-context + - name: Verify URL scraping and billing + run: npm run test:e2e:url + - name: Save URL classification evidence + uses: actions/upload-artifact@v4 + with: + name: url-classification-e2e + path: captures/url-classification.json + - name: Verify long-context workspace billing + run: npm run test:e2e:long-context-billing + - name: Verify ten-million-token job and settlement + run: npm run test:e2e:long-context-job + - name: Verify one whole ten-million-token document upload + run: npm run test:e2e:whole-document + - name: Save long-context verification evidence + uses: actions/upload-artifact@v4 + with: + name: long-context-e2e + path: | + captures/long-context-e2e.json + captures/long-context-billing.json + captures/long-context-job.json + captures/whole-document.json + captures/document-admission.json + - name: Save classification usage evidence + uses: actions/upload-artifact@v4 + with: + name: usage-e2e + path: captures/usage-e2e.json + - name: Save spending verification evidence + uses: actions/upload-artifact@v4 + with: + name: spending-e2e + path: captures/spending-e2e.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ea171c6..fb4f12c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -92,6 +92,51 @@ jobs: # maintainer deploys with locally. - name: Build application run: npm run build + - name: Verify spending through the built Worker and real Durable Objects + run: npm run test:e2e:spending + - name: Verify classification usage responses + run: npm run test:e2e:usage + - name: Verify long-context screening and judgment + run: npm run test:e2e:long-context + - name: Verify URL scraping and billing + run: npm run test:e2e:url + - name: Save URL classification evidence + uses: actions/upload-artifact@v4 + with: + name: url-classification-e2e + path: captures/url-classification.json + - name: Verify long-context workspace billing + run: npm run test:e2e:long-context-billing + - name: Verify complimentary Pro lifecycle + run: bun e2e/complimentary-pro.ts + - name: Verify ten-million-token job and settlement + run: npm run test:e2e:long-context-job + - name: Verify one whole ten-million-token document upload + run: npm run test:e2e:whole-document + - name: Save long-context verification evidence + uses: actions/upload-artifact@v4 + with: + name: long-context-e2e + path: | + captures/long-context-e2e.json + captures/long-context-billing.json + captures/long-context-job.json + captures/whole-document.json + - name: Save complimentary Pro verification evidence + uses: actions/upload-artifact@v4 + with: + name: complimentary-pro-e2e + path: captures/complimentary-pro.json + - name: Save classification usage evidence + uses: actions/upload-artifact@v4 + with: + name: usage-e2e + path: captures/usage-e2e.json + - name: Save spending verification evidence + uses: actions/upload-artifact@v4 + with: + name: spending-e2e + path: captures/spending-e2e.json - name: Migrate account database env: @@ -111,15 +156,25 @@ jobs: DATABASE_URL: ${{ secrets.DATABASE_URL }} run: bun scripts/check-newsletter-cutover.ts --activate + # The chunklaya pair is optional and travels together: with both set as + # repository secrets explicit chunklaya requests route there; with either + # missing they are left out of the file, so an existing value survives + # and unconfigured explicit requests answer chunklaya_unavailable. The + # dgemma pair (the image-capable System One service) works the same way. - name: Deploy env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} DATABASE_URL: ${{ secrets.DATABASE_URL }} + CHUNKLAYA_URL: ${{ secrets.CHUNKLAYA_URL }} + CHUNKLAYA_TOKEN: ${{ secrets.CHUNKLAYA_TOKEN }} + DGEMMA_URL: ${{ secrets.DGEMMA_URL }} + DGEMMA_TOKEN: ${{ secrets.DGEMMA_TOKEN }} run: | node --input-type=module -e ' import { writeFileSync } from "node:fs"; + const { DATABASE_URL, CHUNKLAYA_URL, CHUNKLAYA_TOKEN, DGEMMA_URL, DGEMMA_TOKEN } = process.env; writeFileSync(process.env.RUNNER_TEMP + "/classifier-secrets.json", - JSON.stringify({ DATABASE_URL: process.env.DATABASE_URL }), { mode: 0o600 }); + JSON.stringify({ DATABASE_URL, ...(CHUNKLAYA_URL && CHUNKLAYA_TOKEN ? { CHUNKLAYA_URL, CHUNKLAYA_TOKEN } : {}), ...(DGEMMA_URL && DGEMMA_TOKEN ? { DGEMMA_URL, DGEMMA_TOKEN } : {}) }), { mode: 0o600 }); ' npx wrangler deploy --secrets-file "$RUNNER_TEMP/classifier-secrets.json" @@ -132,9 +187,20 @@ jobs: --retry-all-errors https://classifier.dev/v1/health) echo "$health" echo "$health" | grep -q '"ok": true' - label=$(curl --fail --silent --show-error https://classifier.dev/spam,not+spam/Win+a+free+iPhone) - echo "classify -> $label" - test "$label" = "spam" + # The anonymous probe runs from a hosted runner, whose IP the free-traffic + # reputation gate may refuse (403 proxy_requires_payment). That is the + # Worker answering with its own gate, not a broken deploy, so it passes + # with a notice; every other non-200 fails. The body is always printed, + # so a failure here says what was answered rather than only that it was. + code=$(curl --silent --show-error -o classify.json -w '%{http_code}' -H 'accept: application/json' \ + https://classifier.dev/spam,not+spam/Win+a+free+iPhone) + echo "classify -> HTTP $code"; cat classify.json; echo + case "$code" in + 200) grep -Eq '"label": *"spam"' classify.json ;; + 403) grep -Eq '"code": *"proxy_requires_payment"' classify.json && + echo "::notice::the anonymous probe was refused by the reputation gate for this runner's IP; the Worker is live" ;; + *) echo "::error::anonymous classification answered HTTP $code"; exit 1 ;; + esac - name: Check multidimensional classification end to end env: CLASSIFIER_BASE_URL: https://classifier.dev diff --git a/AGENTS.md b/AGENTS.md index 4f42934..e0cacd2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # AGENTS.md — working in this repository -classifier.dev is one Cloudflare Worker (`src/index.ts`) with no runtime -dependencies, a one-file CLI (`cli/classify.js`) and a Python eval harness +classifier.dev is one Cloudflare Worker (`src/index.ts`), +a one-file CLI (`cli/classify.js`) and a Python eval harness (`eval/`). Everything the site says is generated from constants in `src/`, so the plain text (`curl classifier.dev`), the HTML and the Markdown never drift. @@ -24,6 +24,7 @@ the plain text (`curl classifier.dev`), the HTML and the Markdown never drift. - Docs are plain text with UPPERCASE headings (`src/docs.ts`, `src/pages.ts`); `renderDoc` turns them into HTML and `toMarkdown` into Markdown. - Discovery files (`/.well-known/*`, sitemap, robots, auth.md) are generated in `src/wellknown.ts` from `SITE` and the MCP tool table — edit the source, never a served file. - The MCP servers (`src/mcp.ts`) are stateless Streamable HTTP; tools call the API through `worker.fetch` so limits and logging are shared. +- Whole-document uploads use `POST /v1/classify`: `src/document-upload.ts` stream-parses JSON or UTF-8 text up to 10M tokens/100 MB, preserving exact cl100k counts across internal fragment boundaries. `src/long-context-job.ts` stores source privately while queued, deletes it as SQLite Durable Object alarms screen it, then judges and settles automatically. Reserve the actual document price after upload. Delete remaining source/evidence on failure, cancellation or 24-hour expiry. The old manual-part API remains supported. Keep generated docs and `e2e/whole-document.mjs` in sync; test through the built Worker, real Durable Objects and PostgreSQL ledger. - Never commit secrets; `.secrets.env`, `.dev.vars` are ignored. `eval/data/` is ignored except the summary copied to `src/vs-jev.json`. - Measured numbers on the site come from `eval/`; do not type numbers in by hand. - Jev is asked through Vercel's AI Gateway first when `AI_GATEWAY_API_KEY` @@ -32,6 +33,52 @@ the plain text (`curl classifier.dev`), the HTML and the Markdown never drift. when the gateway refuses; both transports and the translation between them live in `src/jev.ts`. Nothing downstream should know which door answered beyond the `model` label. +- Laya (`jev/laya`) and Kev (`jev/kev`) are hosted by Beam, which speaks the + same System One protocol, so they are a third transport in `src/jev.ts` + rather than a second client: one packer, one retry policy, one validator, + one meter, selected by a `Backend` descriptor. `src/laya.ts` owns only the + product contract — lanes, caller limits, quota cost. `BEAM_API_KEY` is the + single credential; there is no deployment of ours and nothing on Modal. + Beam refuses more than 32 named questions per request and rejects an + oversized context rather than truncating, so a context refusal is + translated to `max_tokens_exceeded` and the batch halves and retries. + Unlike Jev, a Beam request is never retried: a lane quota counts attempts. +- Default/explicit Jev inputs over 32,000 characters use long-context Jev: + Chonkie RecursiveChunker with 600 cl100k_base tokens, parallel relevance/ + uncertainty screening that preserves opposing evidence and exceptions, + then whole eligible chunks in source order for final Jev within 20,000 + cl100k_base tokens and a safe provider estimate. Eligible evidence can be + omitted when the budget fills; disclose selection in usage.long_context. + Limits: 250,000 original context tokens summed once across inputs, 20 + documents, 32 decisions (documents × dimensions or multi-label categories), + 1 MB request body. Fast only. Require a workspace + with paid balance or active paid subscription; signup credit is insufficient. + Retail is original context tokens × 2 × $0.042/M, independent of dimensions + and actual screening/final usage. No evidence returns 422 + long_context_no_evidence without charge. Never claim universal accuracy or + that the final call reads the full original document. + Preserve existing trusted dedicated enterprise/operator access. Aggregate + long-context counts go to classifier_long_context_events and appended + account analytics fields; never record document text or caller identifiers. +- Explicit chunklaya (`chunklaya/multilingual`) is our legacy opt-in service: Laya + behind a chunk-and-index harness, github.com/myxamediyar/chunklaya under + `serve/`, on a RunPod pod. It speaks System One too, so it is a fourth + transport in `src/jev.ts`, reached through `CHUNKLAYA_URL` and + `CHUNKLAYA_TOKEN` (Worker secrets). It requires explicit model: "chunklaya", + `CHUNKLAYA_ENABLED` set to `"true"` and both secrets; it is never selected + automatically for long text. One document is one + request; the service refuses rather than truncates, its 4xx become + `chunklaya_input`, and there is no fallback to Jev or the LLM chain. It is + billed by the hour, so no per-token provider cost is metered. +- dgemma (`dgemma`) is DiffusionGemma 26B-A4B in vLLM's structured-read mode + (vllm-project/vllm#57250), on a RunPod pod, behind the PR's `/v1/systemone` + interposer, which speaks System One and reads an `images` array. It is the + image door of POST /v1/systemone: `src/dgemma.ts` sends a body that names + model "dgemma" or carries images there, with `DGEMMA_URL` and `DGEMMA_TOKEN` + (Worker secrets) when `DGEMMA_ENABLED` is `"true"`; every other body stays + TypeSafe's. Nothing answers for the other: a refused body is `dgemma_input`, + a busy service `dgemma_busy`, a down or unconfigured one `dgemma_unavailable`, + and images under another model `images_unsupported`. Billed by the hour. - The updates roadmap is one constant, `ROADMAP` in `src/newsletter.ts`; the plain text, the signup form and the Markdown all render from it. Addresses go to the `subscriber` table in the shared application Neon database. Preserve consent, diff --git a/README.md b/README.md index bdbe801..9b6cdfa 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,10 @@ acknowledged as a duplicate but not emailed again, so one looping agent cannot empty itself into the inbox; the hourly budget is 100 per IP and the remainder comes back on every receipt. +Testimonials use `signal.category: "testimonial"`. They require +`reporter.agent_type` and `reporter.agent_description`, so praise arrives with +enough context to understand what kind of agent benefited and how it works. + `quality_score` is a deterministic function of how complete the report is — an agent can read the rule and write a better one next time. Nothing here calls the classifier or Analytics Engine: this is where reports arrive saying those @@ -210,10 +214,55 @@ The copy is one constant — `ROADMAP` in `src/newsletter.ts`. The plain text at `curl classifier.dev`, the form on the rendered page and `index.md` all read it, so a change to the roadmap changes all three or none. +## Jev long context + +Default Jev and explicit `model: "jev"` inputs over 32,000 characters use +Chonkie RecursiveChunker with 600-token `cl100k_base` chunks, parallel Jev +screening for relevant or uncertain evidence (including opposing evidence and +exceptions), then final Jev over whole eligible chunks in source order. +The final evidence budget is 20,000 `cl100k_base` tokens plus a conservative +provider context check. Eligible chunks can be omitted when that budget fills; +`usage.long_context` reports selection counts, tokens, calls and timing. +This does not guarantee full-document final reading or universal accuracy. + +Use Fast with a workspace key backed by paid balance or an active paid +subscription; anonymous access and signup credit do not qualify. Synchronous limits: +250,000 original context tokens summed across inputs, 20 documents, 32 decisions, +and a 1 MB body. Retail is $0.084/M original `cl100k_base` context tokens, +counted once across inputs regardless of dimensions and actual inference usage. +Each continuous whitespace or non-whitespace run is limited to 8,192 UTF-16 +code units. Longer runs return `400 long_context_input` before document +tokenization or chunking, bounding tokenizer work on pathological inputs. +No eligible evidence returns `422 long_context_no_evidence` without charge. +Explicit `model: "chunklaya"` keeps its separate legacy opt-in behavior. + +Send one whole document of up to 10 million input tokens (100 MB) to +`POST /v1/classify` with a funded workspace key. Use the normal JSON input and +labels, or send `text/plain` with a `labels` query parameter. Large uploads +return `202` with a `status_url`; `Prefer: respond-async` also works for smaller +documents. The server splits, screens, retries and finishes automatically. +The repository CLI handles upload and polling with +`node cli/classify.js a,b --document document.txt --json`. + +The exact whole-document token count sets the reservation and $0.084/M charge. +Source text is stored privately while queued and deleted as it is screened. +Final Jev reads at most 20,000 selected evidence tokens. Completion, failure, +cancellation and 24-hour expiry delete remaining source and evidence; failed +or canceled work is refunded. Results remain available until expiry. + ## Analytics Every request writes one Analytics Engine datapoint (tier, label-set fingerprint, -country, status, count, latency). No request text is ever stored. +country, status, count, latency). No request text is written to analytics. + +Long-context aggregates also go to `classifier_long_context_events`, with +context/document totals, chunk screening/selection/omission counts, phase input +tokens (including unknown-count flags), answered-call counts and phase timings. +Account events append the corresponding fields; their ordered column contract +is in [`src/server/analytics/schema.ts`](src/server/analytics/schema.ts). +Documents and original context tokens count once; chunk and phase counters +accumulate across dimension passes. No document text or caller identifiers are +written to the dedicated dataset. Every request also records what it cost us: OpenRouter returns the charge for a call when asked, and Jev is billed on the input tokens it reports, at the @@ -262,7 +311,11 @@ The report includes status, reason, count and mean latency for each provider; `/alerts` shows current incidents even when their notification is suppressed. The existing 15-minute alert check warns when at least three attempts fail and failures exceed 5% for either provider. Counts account for Analytics Engine -sampling. No input text, labels, caller identifiers or upstream messages are stored. +sampling. No input text, caller identifiers or upstream messages are stored. +Successful simple and multi-label classifier names are retained for 90 days in +a separate aggregate KV record with no caller or source-text fields. Its keyed +fingerprint also appears in per-request analytics, allowing operators to resolve +label names in those pseudonymous records. `AI_GATEWAY_DISABLED = "true"` in `wrangler.example.toml` keeps production on TypeSafe directly after the gateway repeatedly returned 429 on September 19. @@ -385,8 +438,11 @@ persist when flipped, so this likely needs a plan-level change or support. ## Operator agent access For operator-owned bulk agent work, set a dedicated `AGENT_API_KEY` Worker secret -and send it as `Authorization: Bearer ...`. It uses the existing unmetered -classification path without replacing `ENTERPRISE_API_KEY`. The CLI accepts it +and send it as `Authorization: Bearer ...`. Classification quotas are unmetered, +but inference uses a separate $2/day operator allowance (`OPERATOR_DAILY_USD`), +shared across IPs, with the same request and concurrency limits as free usage. +It neither consumes nor expands the anonymous allowance and does not replace +`ENTERPRISE_API_KEY`. The CLI accepts it through `CLASSIFY_API_KEY` or `CLASSIFIER_API_KEY`. It does not authorize private reports or admin access. Keep it in an ignored secret file; never give it to public clients. Anonymous quotas continue to apply to unauthenticated traffic. diff --git a/cli/README.md b/cli/README.md index 74773c8..d7d6697 100644 --- a/cli/README.md +++ b/cli/README.md @@ -23,6 +23,19 @@ One line per input, in input order: `label`, `confidence`, `text`, tab-separated everything the API returns; `--quiet` gives labels only; `--count` gives a histogram. +## One whole document + +With a funded workspace key in `CLASSIFY_API_KEY`, upload a UTF-8 document of +up to 10 million tokens (100 MB): + + classify renewal,cancellation --document agreement.txt --json + +The CLI streams the file once and waits for the result. The server handles +chunking, screening, retries and final judgment. The price is $0.084 per +million original tokens; 10M tokens costs $0.84. Failed jobs are refunded. +`CLASSIFY_JOB_TIMEOUT` controls how long to poll (seconds, default 86400). +Interrupting the CLI does not cancel accepted work; stderr includes its job ID. + ## Built for agents The confidence is calibrated (on a six-way emotion set, answers at ≥ 0.9 were @@ -50,6 +63,7 @@ resumes; a daily quota stops immediately. Errors go to stderr with exit code 1. -k, --max at most n labels (implies --multi) -s, --smart re-ask uncertain answers of a reasoning model -i, --instructions extra criteria + --document one whole UTF-8 document (funded workspace) -r, --review print only inputs with confidence below t -c, --count label histogram instead of rows -j, --json NDJSON output diff --git a/cli/classify.js b/cli/classify.js index 47e7e04..c1472aa 100755 --- a/cli/classify.js +++ b/cli/classify.js @@ -5,7 +5,8 @@ // tab-separated and greppable, or NDJSON with --json. Errors go to stderr with // exit 1; nothing else ever does. -import { readFileSync, writeFileSync, mkdirSync, statSync, realpathSync } from "node:fs"; +import { createReadStream, readFileSync, writeFileSync, mkdirSync, statSync, realpathSync } from "node:fs"; +import { randomUUID } from "node:crypto"; import { homedir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -23,6 +24,7 @@ const HELP = `classify ${PKG.version} — sort text into your own labels, with a USAGE classify "" one text, prints the label + classify --document file.txt one whole document, uploads and waits classify < items.txt one line per input, in order cat items.jsonl | classify --field title @@ -42,8 +44,10 @@ OPTIONS -k, --max at most n labels (implies --multi) -s, --smart re-ask uncertain answers of a reasoning model (slower) --model laya opt into the automatically routed Laya trial (default: jev) + --model chunklaya opt into the legacy long-document model --processing bulk Laya bulk lane; default fast is one decision per call -i, --instructions extra criteria: "judge only the service, ignore the food" + --document upload one UTF-8 document, up to 10M tokens (paid workspace) -r, --review print only inputs with confidence below t -c, --count print a label histogram instead of rows -j, --json NDJSON output with every field the API returns @@ -85,7 +89,7 @@ Docs: https://classifier.dev Agent skill: npx skills add https://classifier.de function parseArgs(argv) { const o = { labels: null, text: null, multi: false, max: null, smart: false, instructions: "", review: null, count: false, json: false, quiet: false, field: "text", id: null, - endpoint: ENDPOINT, model: "jev", processing: "fast", apiKey: process.env.CLASSIFY_API_KEY || process.env.CLASSIFIER_API_KEY || "", help: false, version: false }; + endpoint: ENDPOINT, model: "jev", processing: "fast", document: null, apiKey: process.env.CLASSIFY_API_KEY || process.env.CLASSIFIER_API_KEY || "", help: false, version: false }; const positional = []; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -108,6 +112,7 @@ function parseArgs(argv) { else if (a === "--id") o.id = next(); else if (a === "--endpoint") o.endpoint = next(); else if (a === "--api-key") o.apiKey = next(); + else if (a === "--document") o.document = next(); else if (a === "--model") o.model = next(); else if (a === "--processing") o.processing = next(); else if (a.startsWith("--") && a.includes("=")) { argv.splice(i + 1, 0, a.slice(a.indexOf("=") + 1)); argv[i] = a.slice(0, a.indexOf("=")); i--; } @@ -121,8 +126,8 @@ function parseArgs(argv) { if (!(Number.isInteger(o.max) && o.max > 0)) fail("--max takes a whole number above 0, e.g. --max 3"); o.multi = true; // the API reads max_labels as multi-label; a single label cannot be capped } - if (!["jev", "laya"].includes(o.model)) fail("--model must be jev or laya"); - if (!["fast", "bulk"].includes(o.processing) || o.model !== "laya" && o.processing !== "fast") fail("--processing bulk requires --model laya"); + if (!["jev", "laya", "kev", "chunklaya"].includes(o.model)) fail("--model must be jev, laya, kev or chunklaya"); + if (!["fast", "bulk"].includes(o.processing) || (o.model === "jev" || o.model === "chunklaya") && o.processing !== "fast") fail("--processing bulk requires --model laya or --model kev"); return o; } @@ -175,7 +180,7 @@ async function readStdin() { async function post(o, inputs) { const body = { inputs, labels: o.labels }; - if (o.model === "laya") { body.model = "laya"; body.processing = o.processing; } + if (o.model !== "jev") { body.model = o.model; if (o.model !== "chunklaya") body.processing = o.processing; } if (o.multi) body.multi = true; if (o.max) body.max_labels = o.max; if (o.smart) body.tier = "smart"; @@ -184,8 +189,8 @@ async function post(o, inputs) { if (o.apiKey) headers.authorization = `Bearer ${o.apiKey}`; let last = ""; - const attempts = o.model === "laya" ? 20 : ATTEMPTS; - const deadline = o.model === "laya" ? Date.now() + 180_000 : Infinity; + const attempts = o.model !== "jev" ? 20 : ATTEMPTS; + const deadline = o.model !== "jev" ? Date.now() + 180_000 : Infinity; for (let attempt = 0; attempt < attempts && Date.now() < deadline; attempt++) { let res; try { @@ -204,6 +209,7 @@ async function post(o, inputs) { continue; } const payload = await res.json().catch(() => ({})); + if (res.status === 202) return checkResults(o, await waitDocument(o, payload), inputs.length); if (res.ok) return checkResults(o, payload, inputs.length); last = payload.error || `HTTP ${res.status}`; // A daily quota cannot recover during a normal CLI run. Keep the API's @@ -213,7 +219,7 @@ async function post(o, inputs) { // The API says exactly how long; a batch that trips the minute window // resumes on its own rather than dying at item 7,400. The first wait // also says, once, that the wait is optional. - if (o.model !== "laya" && !o.apiKey && !hinted) { hinted = true; process.stderr.write(`classify: free limits are per IP; Pro lifts them 10x for $20/month: ${payload.upgrade || "https://classifier.dev/pro"}\n`); } + if (o.model === "jev" && !o.apiKey && !hinted) { hinted = true; process.stderr.write(`classify: free limits are per IP; Pro lifts them 10x for $20/month: ${payload.upgrade || "https://classifier.dev/pro"}\n`); } await retry("rate limited", Number(res.headers.get("retry-after")) || 5, attempt, attempts, deadline); continue; } @@ -223,6 +229,62 @@ async function post(o, inputs) { throw new Error(last); } +async function waitDocument(o, accepted) { + const url = new URL(accepted.status_url, o.endpoint); + if (url.origin !== new URL(o.endpoint).origin) throw new Error("The API returned an invalid job URL."); + if (!process.env.CLASSIFY_NO_PROGRESS) process.stderr.write(`classify: document accepted; waiting for ${accepted.id}\n`); + const deadline = Date.now() + (Number(process.env.CLASSIFY_JOB_TIMEOUT) || 86400) * 1000; + let failures = 0; + while (Date.now() < deadline) { + let response; + try { + response = await fetch(url, { headers: { authorization: `Bearer ${o.apiKey}` }, signal: AbortSignal.timeout(TIMEOUT_MS) }); + } catch { + if (++failures >= 5) throw new Error(`Polling interrupted. Your job continues at ${url}`); + await sleep(2000); continue; + } + const job = await response.json().catch(() => ({})); + if (response.ok && job.status === "finished") return job.result; + if (job.status === "failed") throw new Error(job.error?.message || "Document classification failed; the reservation was refunded."); + if (!response.ok && response.status !== 429 && response.status < 500) + throw new Error(job.error || `Unable to read job: HTTP ${response.status}`); + if (!response.ok && ++failures >= 5) throw new Error(`Polling interrupted. Your job continues at ${url}`); + if (response.ok) failures = 0; + await sleep(Math.min(30, Math.max(1, Number(response.headers.get("retry-after")) || 2)) * 1000); + } + throw new Error(`Stopped waiting. Check your job at ${url}`); +} + +async function classifyDocument(o) { + if (!o.apiKey) throw new Error("--document requires a funded workspace API key (CLASSIFY_API_KEY)."); + if (o.text !== null || o.smart || o.model !== "jev" || o.max !== null) + throw new Error("--document accepts one file with Jev fast, optional --multi and --instructions."); + if (!statSync(o.document).isFile()) throw new Error("--document must name a UTF-8 text file."); + const url = new URL(o.endpoint); + for (const label of o.labels) url.searchParams.append("label", label); + if (o.instructions) url.searchParams.set("instructions", o.instructions); + if (o.multi) url.searchParams.set("multi", "true"); + const id = randomUUID(); + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + const stream = createReadStream(o.document); + try { + const response = await fetch(url, { method: "POST", duplex: "half", body: stream, + headers: { authorization: `Bearer ${o.apiKey}`, "content-type": "text/plain; charset=utf-8", + "idempotency-key": id, prefer: "respond-async" }, signal: AbortSignal.timeout(TIMEOUT_MS) }); + const payload = await response.json().catch(() => ({})); + if (response.status === 202) return await waitDocument(o, payload); + if (response.status < 500 && response.status !== 409 && response.status !== 429) + throw new Error(payload.error || `Document upload failed: HTTP ${response.status}`); + if (attempt + 1 === ATTEMPTS) throw new Error(payload.error || "Document upload is unavailable."); + } catch (error) { + if (error instanceof TypeError || error.name === "TimeoutError") { + if (attempt + 1 === ATTEMPTS) throw new Error(`Upload interrupted. Retry with Idempotency-Key ${id}, or check /v1/long-context/jobs/${id}/status.`); + } else throw error; + } finally { stream.destroy(); } + await sleep(1000 * 2 ** attempt); + } +} + const ATTEMPTS = 5; let hinted = false; @@ -244,7 +306,8 @@ function configuredBatch() { } function batchSize(o) { - if (o.model === "laya") return Math.min(configuredBatch(), o.processing === "fast" ? 1 : Math.floor(1000 / (o.multi ? o.labels.length : 1)), o.smart ? SMART_BATCH : MAX_BATCH); + if (o.model === "chunklaya") return Math.min(configuredBatch(), 20); // one document is one upstream request there + if (o.model !== "jev") return Math.min(configuredBatch(), o.processing === "fast" ? 1 : Math.floor(1000 / (o.multi ? o.labels.length : 1)), o.smart ? SMART_BATCH : MAX_BATCH); return Math.min(configuredBatch(), o.smart && !o.apiKey ? SMART_BATCH : MAX_BATCH); } @@ -294,7 +357,7 @@ export async function classifyAll(o, items, onReady, onProgress) { // Public smart traffic is limited to 200 classifications/minute. One // worker keeps concurrent batches from spending that whole window at once; // partner keys can use the normal four workers. - const concurrency = o.model === "laya" || o.smart && !o.apiKey ? 1 : CONCURRENCY; + const concurrency = o.model !== "jev" || o.smart && !o.apiKey ? 1 : CONCURRENCY; await Promise.all( Array.from({ length: Math.min(concurrency, batches.length) }, async () => { while (next < batches.length) { @@ -437,6 +500,16 @@ export async function main(argv) { if (o.labels.length > 100) fail("at most 100 labels"); try { batchSize(o); } catch (e) { fail(e.message); } + if (o.document) { + const payload = await classifyDocument(o); + const results = checkResults(o, payload, 1); + if (o.count) process.stdout.write(formatCount(o, results).join("\n") + "\n"); + else if (kept(o, results[0])) process.stdout.write((o.json + ? JSON.stringify({ file: o.document, ...results[0], usage: payload.usage, pricing: payload.pricing }) + : labelOf(o, results[0])) + "\n"); + return 0; + } + const hint = updateHint(); let items; diff --git a/docs/autumn-integration.md b/docs/autumn-integration.md index 1277cd8..0aee7a9 100644 --- a/docs/autumn-integration.md +++ b/docs/autumn-integration.md @@ -9,15 +9,15 @@ Autumn controls subscription checkout and portal access. Neon is the authoritati - `AUTUMN_PRO_PLAN_ID`: the configured $20 monthly Pro plan. - `APP_ORIGIN`: trusted HTTPS application origin for checkout/portal return URLs; defaults to `https://classifier.dev`. Set an isolated preview origin for sandbox checkout. -Configure `billing.updated` delivery. Scheduled reconciliation repairs missed events with at most ten accounts per invocation and a global maximum of sixty provider calls per UTC day. The budget counts failed attempts; webhook calls are separate. This repair budget is deliberately small and must be reviewed as the paid customer count grows. +Configure `billing.updated` delivery. Scheduled reconciliation repairs missed events with at most ten accounts per invocation and a global maximum of sixty provider calls per UTC day. The budget counts failed attempts; webhook calls are separate. Customers rotate by last attempted reconciliation, including provider failures, so unavailable customers cannot starve the rest of the queue. This repair budget is deliberately small and must be reviewed as the paid customer count grows. ## Payment proof and scope -Activation is not proof of payment. The reconciler fetches canonical customer state and `invoices.list`, then matches a paid invoice's base-plan line to the active subscription's current billing period. Invoice ID and account/period constraints prevent a second grant. New allowances wait for pending requests to settle and replace the previous included allowance; they do not stack. +Activation is not proof of payment. The reconciler fetches canonical customer state and `invoices.list`, then matches a paid invoice's base-plan line to the active subscription's current billing period. Invoice ID and account/period constraints prevent a second grant. New allowances wait for pending requests to settle and replace the previous included allowance; they do not stack. An overdue-payment flag does not cancel an active subscription or erase a verified paid allowance: the current period still requires its matching paid invoice. The initial implementation supports **exactly $20 USD paid monthly Pro invoices**, with one $20 base-plan line and no refunded amount. Discounts, taxes, prorations, bundled invoices and historical invoices without recorded line items require explicit reconciliation; they do not silently grant credits. Only the first 100 paid invoices are inspected. Unsupported cases return a retryable synchronization failure and retain `reconciliation_required`. -Refunds of a previously granted current-period invoice place the account on a billing hold immediately, preventing new reservations. Once pending work settles, the remaining included allowance is removed. The grant is marked revoked and cannot be reissued for the same period. Already consumed usage is not reverse-charged. A verified new paid period clears the hold. Pending requests are not retroactively interrupted. +Refunds of a previously granted current-period invoice place the account on a billing hold immediately, preventing new reservations. Once pending work settles, the remaining included allowance is removed. The grant is marked revoked and cannot be reissued for the same period. Already consumed usage is not reverse-charged. A verified new paid period clears the hold. Pending requests are not retroactively interrupted. Refund webhooks remain retryable until those requests settle and allowance removal completes. Superseded reconciliations cannot acknowledge completion or mutate refund state. Existing Autumn customers require an explicit workspace/customer mapping before checkout so no second customer is created. Live migration, sandbox checkout and live webhook delivery remain unverified until provider credentials are configured. diff --git a/docs/postgres-setup.md b/docs/postgres-setup.md index 78b6ddd..2097aff 100644 --- a/docs/postgres-setup.md +++ b/docs/postgres-setup.md @@ -101,12 +101,11 @@ connection strings into commands that will remain in shell history. Migrations are transactional, ordered, and checksum-verified; editing an already-applied migration is rejected. -The production template targets `aws:us-west-2`, supported by Wrangler 4.122's -placement schema. Cloudflare runs fetch handlers in a nearby Cloudflare data -center, not inside AWS itself. Placement does not move an existing Neon project -or Durable Object, and is not a residency guarantee. Confirm the Neon primary -and upstream latency before activating the production migration. See -[Cloudflare placement](https://developers.cloudflare.com/workers/configuration/placement/). +The production Worker targets Oregon (`aws:us-west-2`) for fetch execution. +This is not a data-residency guarantee and does not move an existing Neon +database or Durable Object. Verify the active database's region separately; +Worker placement alone does not establish full colocation. +See [Cloudflare placement](https://developers.cloudflare.com/workers/configuration/placement/). Do not enable the new account offering until retail token rates, Autumn events, legacy Pro handling, reconciliation, and end-to-end production checks are ready. diff --git a/docs/pricing-economics.md b/docs/pricing-economics.md index 2d499c0..3416c87 100644 --- a/docs/pricing-economics.md +++ b/docs/pricing-economics.md @@ -34,6 +34,22 @@ remainders, so splitting a batch does not multiply rounding charges. Every reservation must pin its rate version and a proven maximum exposure before calling a provider. +## Jev long-context pricing + +The automatic default/explicit Jev path above 32,000 characters charges +original input context counted with `cl100k_base`, summed once across inputs, +at 2 × $0.042/M = $0.084/M. Dimensions do not multiply the context charge. +Screening and final-call provider usage remain separate cost measurements and +do not set the retail charge. At the 250,000-token request ceiling, the retail +charge is $0.021. This is a price calculation, not a measured provider margin. + +Only workspaces with paid balance or an active paid subscription qualify; +anonymous access and free signup credit do not. The path is Fast-only, with +20 documents, 32 decisions and a 1 MB body maximum. No eligible evidence returns +`422 long_context_no_evidence` without charge. The existing explicit chunklaya +trial is separate. The economics below describe the original launch rate card, +not the margin of this screening-and-selection pipeline. + ## Economics of a fully consumed $20 plan Assumptions: standard US domestic Stripe payments (2.9% + $0.30), Stripe Billing diff --git a/drizzle/schema.ts b/drizzle/schema.ts index 25d5e92..e7ddac6 100644 --- a/drizzle/schema.ts +++ b/drizzle/schema.ts @@ -108,6 +108,8 @@ export const app_usage = pgTable("app_usage", { actual_nano: bigint({ mode: "bigint" }), rate_version: text(), metering_mode: text().default('credits').notNull(), + classifications: integer(), + escalations: integer(), reporting_status: text().default('not_ready').notNull(), }, (table) => [ index("app_usage_account").using("btree", table.account_id.asc().nullsLast().op("text_ops"), table.created_at.asc().nullsLast().op("text_ops")), @@ -124,6 +126,8 @@ export const app_usage = pgTable("app_usage", { name: "app_usage_agent_id_fkey" }), check("app_usage_actual_nano_check", sql`actual_nano >= 0`), + check("app_usage_classifications_check", sql`classifications >= 0`), + check("app_usage_escalations_check", sql`escalations >= 0`), check("app_usage_metering_mode_check", sql`metering_mode = ANY (ARRAY['credits'::text, 'tokens'::text, 'legacy'::text])`), check("app_usage_reporting_status_check", sql`reporting_status = ANY (ARRAY['not_ready'::text, 'review'::text, 'pending'::text, 'reported'::text, 'exempt'::text])`), ]); diff --git a/e2e/admin-browser.ts b/e2e/admin-browser.ts new file mode 100644 index 0000000..2032095 --- /dev/null +++ b/e2e/admin-browser.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dashboard } from "../src/admin"; +import { adminFixture } from "../test/admin-fixture"; + +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + const url = new URL(req.url); + if (url.pathname.startsWith("/admin-assets/")) + return new Response(Bun.file("public" + url.pathname)); + const data = adminFixture(); + if (url.searchParams.has("empty")) { + data.chatOutcomes = []; + data.chatSeries = []; + data.chatCallers = 0; + } + if (url.searchParams.has("unavailable")) { + data.unavailable = ["chatOutcomes", "chatSeries", "chatCallers"]; + data.errors = ["Synthetic unavailable analytics"]; + } + if (url.searchParams.has("unknown")) + data.chatOutcomes = [ + { + outcome: "stopped", + turns: 1, + modelCalls: 1, + unknownTokenCalls: 1, + unknownCostCalls: 1, + }, + ]; + return new Response(dashboard("24h", data, "fixture-nonce"), { + headers: { "content-type": "text/html" }, + }); + }, +}); +const run = promisify(execFile); +const browser = async (...args: string[]) => + (await run("agent-browser", ["--session", "chat-analytics-e2e", ...args])) + .stdout; +const check = async (condition: string) => + browser( + "eval", + `if (!(${condition})) throw new Error(${JSON.stringify(condition)}); true`, + ); +const url = new URL("/admin", server.url).href; +const results: string[] = []; +try { + await mkdir("captures", { recursive: true }); + await browser("open", url); + await browser("set", "viewport", "1366", "900"); + await browser("wait", "--text", "Chat model accounting"); + await browser("eval", "document.fonts.ready.then(()=>true)"); + await check('document.querySelectorAll(".analytics-section").length === 6'); + await check('!document.querySelector("[role=tab]")'); + await check( + 'document.querySelector("#Dimensions").getBoundingClientRect().height > 0', + ); + await check( + 'Array.from(document.querySelectorAll(".section-nav a")).every(a => { const id = decodeURIComponent(a.hash.slice(1)); return !/\\s/.test(id) && document.getElementById(id); })', + ); + await browser("screenshot", "captures/desktop-after.png"); + await browser("click", 'a[href="#Chat"]'); + await browser("mouse", "move", "0", "0"); + await browser("screenshot", "captures/chat-desktop.png"); + await browser("download", "button.control", "captures/admin-export.json"); + const exported = JSON.parse(await readFile("captures/admin-export.json", "utf8")); + assert.equal(exported.range, "24h"); + assert.deepEqual(exported.chatOutcomes, adminFixture().chatOutcomes); + results.push("JSON export downloads the selected range and chat accounting"); + for (const term of [ + "Model economics", + "Dimension outcomes", + "ls_9f3c7a8d", + "Chat model accounting", + ]) { + await browser("eval", "getSelection()?.removeAllRanges(); scrollTo(0,0)"); + await check(`window.find(${JSON.stringify(term)})`); + } + results.push( + "all six sections rendered together; browser Find reaches models, dimensions, classifier fingerprints and chat", + ); + await browser("open", url + "?unknown"); + await browser("wait", "--text", "Chat model accounting"); + await check( + 'document.querySelector("#Chat").innerText.includes("0 / 1 calls returned cost")', + ); + await check( + 'document.querySelector("#Chat .metrics").innerText.includes("Model spend reported\\n—")', + ); + results.push( + "unknown model spend shown as unavailable with explicit coverage, never zero", + ); + await browser("open", url + "?empty"); + await browser("wait", "--text", "Chat model accounting"); + await check( + 'document.querySelector("#Chat").innerText.includes("No activity recorded")', + ); + await browser("open", url + "?unavailable"); + await browser("wait", "--text", "Some analytics are unavailable."); + await check( + 'document.querySelector("#Chat").innerText.includes("Data unavailable")', + ); + results.push( + "empty traffic and unavailable analytics remain distinguishable", + ); + await browser("open", url); + await browser("set", "viewport", "390", "844"); + await browser("wait", "--text", "Chat model accounting"); + await browser("screenshot", "captures/mobile-after.png"); + await browser("click", 'a[href="#Chat"]'); + await browser("mouse", "move", "0", "0"); + await browser("screenshot", "captures/chat-mobile.png"); + await check("document.documentElement.scrollWidth <= innerWidth"); + results.push( + "mobile layout has no page overflow; chat and section links remain usable", + ); + assert.equal(results.length, 5); +} finally { + await browser("close"); + server.stop(true); + await writeFile( + "captures/admin-browser.json", + JSON.stringify( + { + runtime: + "real browser, Vite-built admin bundle and synthetic analytics snapshot", + results, + }, + null, + 2, + ), + ); +} +console.log( + `${results.length} dashboard browser scenarios passed; captures/admin-browser.json`, +); diff --git a/e2e/chat-analytics.ts b/e2e/chat-analytics.ts new file mode 100644 index 0000000..28aeae8 --- /dev/null +++ b/e2e/chat-analytics.ts @@ -0,0 +1,295 @@ +import assert from "node:assert/strict"; +import { Database } from "bun:sqlite"; +import { mkdir, writeFile } from "node:fs/promises"; +import worker, { type Env } from "../src/index"; +import { adminResponse } from "../src/admin"; + +const points: { indexes: string[]; blobs: string[]; doubles: number[] }[] = []; +const pending: Promise[] = []; +const ctx = { + waitUntil: (promise: Promise) => pending.push(promise), + passThroughOnException() {}, +} as ExecutionContext; +let mode = "success", + limited = false, + calls = 0, + toolName = "current_time"; +const db = new Database(":memory:"); +for (const name of ["classifier_events", "classifier_chat_events"]) + db.exec( + `CREATE TABLE ${name} (timestamp INTEGER, _sample_interval REAL, index1 TEXT, ${Array.from({ length: 20 }, (_, i) => `blob${i + 1} TEXT DEFAULT ''`).join(",")}, ${Array.from({ length: 20 }, (_, i) => `double${i + 1} REAL DEFAULT 0`).join(",")})`, + ); +const now = Math.floor(Date.now() / 1000); +const provider = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const body = (await req.json()) as { + messages: { role: string; content: string }[]; + }; + calls++; + if (mode === "failure") + return new Response("synthetic upstream failure", { status: 503 }); + if (mode === "slow") await Bun.sleep(1000); + const tool = body.messages.at(-1)?.role === "user"; + const delta = tool + ? { + tool_calls: [ + { + index: 0, + id: "clock", + function: { + name: toolName, + arguments: + mode === "tool-error" ? '{"timezone":"invalid-zone"}' : "{}", + }, + }, + ], + } + : { content: "The clock answered." }; + const usage = + mode === "unknown" + ? "" + : `data: ${JSON.stringify({ choices: [], usage: { prompt_tokens: 100, completion_tokens: 10, cost: mode === "zero-cost" ? 0 : 0.002 } })}\n\n`; + return new Response( + `data: ${JSON.stringify({ choices: [{ delta }] })}\n\n${usage}data: [DONE]\n\n`, + { headers: { "content-type": "text/event-stream" } }, + ); + }, +}); +const originalFetch = globalThis.fetch; +globalThis.fetch = (async (input, init) => { + const url = String(input); + if (url.includes("openrouter.ai")) return originalFetch(provider.url, init); + if (url.includes("/analytics_engine/sql")) { + const query = String(init?.body) + .replace( + /toStartOfInterval\(timestamp, INTERVAL '(\d+)' (HOUR|DAY)\)/g, + (_, n, unit) => + `CAST(timestamp / ${Number(n) * (unit === "DAY" ? 86400 : 3600)} AS INTEGER)*${Number(n) * (unit === "DAY" ? 86400 : 3600)}`, + ) + .replace(/toDateTime\(now\(\)\) - INTERVAL '(\d+)' HOUR/g, (_, n) => + String(now - Number(n) * 3600), + ); + return Response.json({ data: db.query(query).all() }); + } + throw new Error("Unexpected upstream"); +}) as typeof fetch; +const env = { + OPENROUTER_API_KEY: "fixture", + PRIVACY_SALT: "private-fixture", + ADMIN_PASSWORD: "fixture", + ADMIN_SIGNING_KEY: "fixture", + LIMITER: { + idFromName: (s: string) => s, + get: () => ({ + fetch: async () => Response.json({ limited, remaining: 10, resetIn: 60 }), + }), + }, + CHAT_AE: { + writeDataPoint(point: (typeof points)[number]) { + points.push(structuredClone(point)); + const values = [ + now, + 2, + point.indexes[0], + ...point.blobs, + ...point.doubles, + ]; + const columns = [ + "timestamp", + "_sample_interval", + "index1", + ...point.blobs.map((_, i) => `blob${i + 1}`), + ...point.doubles.map((_, i) => `double${i + 1}`), + ]; + db.query( + `INSERT INTO classifier_chat_events (${columns.join(",")}) VALUES (${values.map(() => "?").join(",")})`, + ).run(...values); + }, + }, +} as unknown as Env; +const drain = async () => { + while (pending.length) await Promise.all(pending.splice(0)); +}; +const send = (messages: unknown) => + worker.fetch( + new Request("https://classifier.dev/v1/chat", { + method: "POST", + headers: { + "content-type": "application/json", + "cf-connecting-ip": "203.0.113.49", + }, + body: JSON.stringify({ messages }), + }), + env, + ctx, + ); +const message = [{ role: "user", content: "PRIVATE PROMPT DO NOT STORE" }]; +const report: { + scenarios: string[]; + points: typeof points; + dashboard?: unknown; +} = { scenarios: [], points }; +try { + const response = await send(message); + assert.equal(response.status, 200); + assert.match(await response.text(), /The clock answered/); + await drain(); + assert.equal(points.length, 1); + assert.equal(points[0].blobs[1], "completed"); + assert.deepEqual(points[0].doubles.slice(2, 8), [2, 200, 20, 0.004, 0, 0]); + assert.equal(points[0].doubles[12], 1); + assert.equal(points[0].doubles[14], 1); + report.scenarios.push( + "one event after a two-model-call tool turn; measured tokens and spend", + ); + mode = "unknown"; + await ( + await send([...message, { role: "assistant", content: "ok" }, ...message]) + ).text(); + await drain(); + assert.deepEqual(points[1].doubles.slice(2, 8), [2, 0, 0, 0, 2, 2]); + assert.equal(points[1].doubles[14], 0); + report.scenarios.push( + "missing provider accounting stays unknown; follow-ups distinguished", + ); + mode = "failure"; + await (await send(message)).text(); + await drain(); + assert.equal(points[2].blobs[1], "failed"); + const before = calls; + limited = true; + assert.equal((await send(message)).status, 429); + await drain(); + assert.equal(calls, before); + assert.equal(points[3].blobs[1], "rate_limited"); + limited = false; + assert.equal((await send([])).status, 400); + await drain(); + assert.equal(points[4].blobs[1], "invalid_request"); + report.scenarios.push( + "upstream failures, refused turns and invalid bodies counted without inference", + ); + mode = "slow"; + const stopped = await send(message); + await stopped.body!.cancel(); + await drain(); + assert.equal(points[5].blobs[1], "stopped"); + assert.equal(points.length, 6); + report.scenarios.push( + "cancelled streams finish once and retain unknown usage", + ); + const serialized = JSON.stringify(points); + assert.ok(!serialized.includes("PRIVATE PROMPT")); + assert.ok(!serialized.includes("203.0.113.49")); + assert.ok(!serialized.includes("The clock answered")); + const login = (await adminResponse( + new Request("https://classifier.dev/admin", { + method: "POST", + headers: { origin: "https://classifier.dev" }, + body: new URLSearchParams({ password: "fixture" }), + }), + env, + "admin", + "test", + ))!; + const cookie = login.headers.getSetCookie()[0].split(";")[0]; + const page = (await adminResponse( + new Request("https://classifier.dev/admin", { headers: { cookie } }), + env, + "admin", + "test", + ))!; + const html = await page.text(); + const data = JSON.parse( + html + .match(/data-admin="([^"]+)"/)![1] + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/&/g, "&"), + ).data; + assert.deepEqual(data.unavailable, []); + assert.equal(data.chatCallers, 1); + assert.equal( + data.chatOutcomes.find( + (r: { outcome: string }) => r.outcome === "completed", + ).turns, + 4, + ); + assert.equal( + data.chatOutcomes.find( + (r: { outcome: string }) => r.outcome === "completed", + ).usd, + 0.008, + ); + assert.equal( + data.chatOutcomes.find( + (r: { outcome: string }) => r.outcome === "completed", + ).unknownCostCalls, + 4, + ); + report.dashboard = data.chatOutcomes; + report.scenarios.push( + "authenticated dashboard queries sampling-weighted usage; no prompt or raw IP retained", + ); + mode = "tool-error"; + await (await send(message)).text(); + await drain(); + assert.equal(points.at(-1)!.doubles[13], 1); + assert.equal(points.at(-1)!.blobs[1], "completed"); + mode = "zero-cost"; + await (await send(message)).text(); + await drain(); + assert.equal(points.at(-1)!.doubles[5], 0); + assert.equal(points.at(-1)!.doubles[7], 0); + report.scenarios.push( + "tool errors counted even when the assistant completes; measured zero cost is known", + ); + mode = "success"; + for (const name of [ + "classify_texts", + "classify_dimensions", + "classify_multi_label", + "count_labels", + "review_uncertain", + ]) { + toolName = name; + await (await send(message)).text(); + await drain(); + assert.equal( + points.at(-1)!.doubles[9], + 1, + `${name} is a classification tool attempt`, + ); + } + toolName = "current_time"; + report.scenarios.push( + "all five classification tool variants included in classification counts", + ); + const original = env.CHAT_AE; + env.CHAT_AE = { + writeDataPoint() { + throw new Error("telemetry down"); + }, + } as AnalyticsEngineDataset; + mode = "success"; + assert.match(await (await send(message)).text(), /The clock answered/); + await drain(); + env.CHAT_AE = original; + report.scenarios.push("telemetry failure never breaks chat"); +} finally { + globalThis.fetch = originalFetch; + provider.stop(true); + db.close(); + await mkdir("captures", { recursive: true }); + await writeFile( + "captures/chat-analytics.json", + JSON.stringify(report, null, 2), + ); +} +console.log( + `${report.scenarios.length} chat analytics scenarios passed; captures/chat-analytics.json`, +); diff --git a/e2e/chat.mjs b/e2e/chat.mjs new file mode 100644 index 0000000..76f70bc --- /dev/null +++ b/e2e/chat.mjs @@ -0,0 +1,140 @@ +import { Miniflare, convertV4MiniflareOptions, Response as WorkerResponse } from 'miniflare'; +import { mkdir, writeFile, readdir } from 'node:fs/promises'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const report = { runtime: 'built Worker and real rate-limiter Durable Object', upstream: 'deterministic provider fixtures; no live inference', results: [] }; +let modelCalls = 0, classifications = 0; +const modules = (await readdir('dist/server', { recursive: true })).filter(p => /\.(js|wasm)$/.test(p)).sort((a, b) => a === 'index.js' ? -1 : b === 'index.js' ? 1 : a.localeCompare(b)).map(p => ({ type: p.endsWith('.wasm') ? 'CompiledWasm' : 'ESModule', path: `dist/server/${p}` })); +const sse = delta => new WorkerResponse(`data: ${JSON.stringify({ choices: [{ delta }] })}\n\ndata: [DONE]\n\n`, { headers: { 'content-type': 'text/event-stream' } }); +const mf = new Miniflare(convertV4MiniflareOptions({ workers: [{ name: 'chat', modules, modulesRoot: 'dist/server', compatibilityDate: '2026-08-01', compatibilityFlags: ['nodejs_compat'], + durableObjects: { FREE_BUDGET: { className: 'FreeBudget', useSQLite: true }, LIMITER: { className: 'RateLimiter', useSQLite: true } }, kvNamespaces: ['STATS'], + bindings: { SPENDING_ENABLED: 'true', PRIVACY_SALT: 'fixture', SPUR_API_KEY: 'fixture', TYPESAFE_API_KEY: 'fixture', OPENROUTER_API_KEY: 'fixture', ENTERPRISE_API_KEY: 'fixture-enterprise' }, + outboundService: async request => { + if (request.url.startsWith('https://api.spur.us/')) return WorkerResponse.json({}); + const body = await request.json(); + if (request.url.includes('openrouter.ai')) { + modelCalls++; + const last = body.messages.at(-1); + if (last.content === 'fail upstream') return new WorkerResponse('unavailable', { status: 503 }); + if (last.content === 'slow reply') await new Promise(resolve => setTimeout(resolve, 1000)); + if (last.role === 'user') return sse({ tool_calls: [{ index: 0, id: 'call_1', function: { name: 'classify_texts', arguments: JSON.stringify({ inputs: ['Please refund my invoice'], labels: ['billing', 'support'] }) } }] }); + assert.equal(last.role, 'tool'); + assert.match(last.content, /billing\t0.90/); + return sse({ content: 'The message is billing (90% confidence).' }); + } + assert.ok(request.url.includes('typesafe.ai'), `Unexpected upstream: ${request.url}`); + classifications++; + return WorkerResponse.json({ model: 'jev-1.13.0', usage: { input_tokens: 100, output_tokens: 0 }, answers: Object.fromEntries(Object.entries(body.questions).map(([id, question]) => { + const keys = Object.keys(question.criteria); + return [id, { choice: keys[0], confidence: 0.9, probabilities: { [keys[0]]: 0.9, [keys[1]]: 0.1 } }]; + })) }); + }, +}] })); +const post = (messages, ip = '203.0.113.10', path = '/v1/chat') => mf.dispatchFetch(`https://classifier.dev${path}`, { + method: 'POST', headers: { 'content-type': 'application/json', 'cf-connecting-ip': ip }, body: JSON.stringify({ messages }), +}); +const message = content => [{ role: 'user', content }]; +const events = async response => (await response.text()).split('\n\n').filter(line => line.startsWith('data: ')).map(line => JSON.parse(line.slice(6))); +try { + await mf.ready; + for (const path of ['/', '/chat', '/docs', '/benchmark']) { + const response = await mf.dispatchFetch(`https://classifier.dev${path}`, { headers: { accept: 'text/html' } }); + assert.equal(response.status, 200, path); + const html = await response.text(); + assert.match(html, /data-chat-open/); + assert.match(html, /id="chat" aria-label="Chat"/); + if (path === '/chat') assert.match(html, /id="chat" aria-label="Chat">/); + for (const script of html.matchAll(/', 'safe & sound'] : [`category ${i}`, 'billing', 'support'], firstSeen: '2026-09-23T00:00:00.000Z', + })); + await labelStore.put(`clsn:${fingerprint}`, '1'); + } + await labelStore.put('cls:ls_legacy', '2026-09-01T00:00:00.000Z'); + await labelStore.put('cls:ls_malformed', '{broken'); + await labelStore.put('unrelated-secret', 'must not appear'); + await labelStore.put('cls:obsolete-raw-key', '2026-09-01T00:00:00.000Z'); + let next = adminUrl; + const seen = new Set(); + const pageSizes = []; + while (next) { + const response = await adminGet(next); + assert.equal(response.status, 200); + assert.match(response.headers.get('cache-control'), /private/); + const html = await response.text(); + assert.match(html, /All label sets/); + assert.doesNotMatch(html, /