From 4122f996c41d01fad3072b129a63a9631eb78acd Mon Sep 17 00:00:00 2001 From: Michael Ryaboy Date: Wed, 23 Sep 2026 19:52:28 -0700 Subject: [PATCH] Add URL classification with metered Context.dev scraping --- .github/workflows/check.yml | 7 + .github/workflows/deploy.yml | 7 + e2e/url-classification.ts | 203 ++++++++++++++++++++++++++++ e2e/url.live.mjs | 41 ++++++ package.json | 3 +- src/docs.ts | 47 +++++++ src/http/classification.ts | 2 +- src/http/spending-classification.ts | 60 ++++++-- src/index.ts | 4 +- src/mcp.ts | 34 +++-- src/openapi.ts | 15 ++ src/pages.ts | 11 +- src/pricingui.ts | 3 + src/scrape.ts | 81 +++++++++++ src/spending/policy.ts | 5 +- 15 files changed, 499 insertions(+), 24 deletions(-) create mode 100644 e2e/url-classification.ts create mode 100644 e2e/url.live.mjs create mode 100644 src/scrape.ts diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 66563cd..1e0123f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -52,6 +52,13 @@ jobs: run: npm run test:e2e:usage - name: Verify long-context screening and judgment run: npm run test:e2e:long-context + - name: Verify URL scraping and billing + run: npm run test:e2e:url + - name: Save URL classification evidence + uses: actions/upload-artifact@v4 + with: + name: url-classification-e2e + path: captures/url-classification.json - name: Verify long-context workspace billing run: npm run test:e2e:long-context-billing - name: Verify ten-million-token job and settlement diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e9c9362..fb4f12c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -98,6 +98,13 @@ jobs: run: npm run test:e2e:usage - name: Verify long-context screening and judgment run: npm run test:e2e:long-context + - name: Verify URL scraping and billing + run: npm run test:e2e:url + - name: Save URL classification evidence + uses: actions/upload-artifact@v4 + with: + name: url-classification-e2e + path: captures/url-classification.json - name: Verify long-context workspace billing run: npm run test:e2e:long-context-billing - name: Verify complimentary Pro lifecycle diff --git a/e2e/url-classification.ts b/e2e/url-classification.ts new file mode 100644 index 0000000..5fc9dbb --- /dev/null +++ b/e2e/url-classification.ts @@ -0,0 +1,203 @@ +import assert from "node:assert/strict"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { PGlite } from "@electric-sql/pglite"; +import { countTokens } from "gpt-tokenizer/encoding/cl100k_base"; +import { accountApi } from "../src/http/account-api"; +import { AppError, parseCreditInteger, postgresDatabase } from "../src/server/db"; +import { appEnvironment } from "../src/server/environment"; +import { performAction } from "../src/server/agents"; +import { provisionTestAccount } from "../tests/support/account"; +import { RateLimiter } from "../src/limiter"; +import type { Env } from "../src/index"; + +// Run with bun e2e/url-classification.ts. Real migrations, ledger and Worker +// inference; only provider responses and Cloudflare storage bindings are fixtures. +const artifact = "captures/url-classification.json"; +type AnalyticsPoint = { indexes: string[]; blobs: string[]; doubles: number[] }; +const longEvents: AnalyticsPoint[] = []; +const report: { runtime: string; results: unknown[]; dedicatedAnalytics: AnalyticsPoint[] } = { + runtime: `Bun account HTTP handler → Worker inference → local HTTP provider; in-memory PGlite with all migrations; DATABASE_URL ignored`, + results: [], + dedicatedAnalytics: longEvents, +}; +const pg = new PGlite({ parsers: { 20: parseCreditInteger, 1700: parseCreditInteger } }); +const originalFetch = globalThis.fetch; +let providerCalls = 0; +const reportedTokens = 123; +const longArticle = "This annual contract renews automatically. ".repeat(1500); +let mode = "normal"; +let scrapeCalls = 0; +const provider = Bun.serve({ hostname: "127.0.0.1", port: 0, async fetch(request) { + providerCalls++; + const body = await request.json() as { model: string; questions: Record }> }; + const screening = Object.values(body.questions).some(q => Object.hasOwn(q.criteria, "irrelevant")); + if (!screening && mode === "fail-final") + return Response.json({ detail: { error_type: "invalid_request" } }, { status: 400 }); + return Response.json({ model: body.model, usage: { input_tokens: reportedTokens, output_tokens: 0 }, + answers: Object.fromEntries(Object.entries(body.questions).map(([id, q]) => { + const labels = Object.keys(q.criteria); + const choice = screening ? mode === "none" ? "irrelevant" : "relevant" : labels[0]; + return [id, { choice, confidence: 0.99, + probabilities: Object.fromEntries(labels.map(label => [label, label === choice ? 0.99 : 0.01 / (labels.length - 1)])) }]; + })), + }); +} }); +globalThis.fetch = ((input, init) => { + if (String(input).startsWith("https://api.context.dev/")) { + scrapeCalls++; + const url = new URL(String(input)); + assert.equal(url.searchParams.get("pdf[ocr]"), "false"); + assert.equal(new Headers(init?.headers).get("authorization"), "Bearer fixture-context"); + if (mode === "network") return Promise.reject(new Error("Network response lost")); + if (mode === "oversized") return Promise.resolve(new Response("x".repeat(8_000_001))); + if (mode === "blocked") return Promise.resolve(Response.json({ error_code: "WEBSITE_BLOCKED", key_metadata: { credits_consumed: 0 } }, { status: 403 })); + if (mode === "missing") return Promise.resolve(Response.json({ error_code: "NOT_FOUND", key_metadata: { credits_consumed: 1 } }, { status: 404 })); + return Promise.resolve(Response.json({ success: true, markdown: mode === "empty" ? "" : mode === "long" ? longArticle : "# Annual contract\nThis contract renews annually.", html: "
This contract renews annually.
", metadata: { title: "Annual contract" }, key_metadata: { credits_consumed: 1 } })); + } + assert.equal(String(input), "https://api.typesafe.ai/v1/systemone", "unexpected external request"); + return originalFetch(provider.url, init); +}) as typeof fetch; + +const objects = new Map(); +const limiter = { + idFromName: (name: string) => name, + get(name: string) { + if (!objects.has(name)) { + const data = new Map(); + let queue = Promise.resolve(); + objects.set(name, new RateLimiter({ storage: { + get: async (key: string) => structuredClone(data.get(key)), + put: async (values: Record) => { + for (const [key, value] of Object.entries(values)) data.set(key, structuredClone(value)); + }, + }, blockConcurrencyWhile(run: () => Promise) { + const result = queue.then(run); + queue = result.then(() => {}, () => {}); + return result; + } } as unknown as DurableObjectState)); + } + return { fetch: (input: string | Request) => objects.get(name)!.fetch(new Request(input)) }; + }, +} as unknown as DurableObjectNamespace; +const db = postgresDatabase(queries => pg.transaction(async tx => { + const results = []; + for (const query of queries) { + const result = await tx.query>(query.sql, query.params); + results.push({ results: result.rows, meta: { changes: result.affectedRows ?? 0 } }); + } + return results; +})); +const pending: Promise[] = []; +const ctx = { waitUntil(promise: Promise) { pending.push(promise); } } as ExecutionContext; +async function flush() { while (pending.length) await Promise.all(pending.splice(0)); } +const kv = new Map(); +const events: { doubles: number[] }[] = []; +const env = appEnvironment({ APP_DB: db, APP_ACCOUNTS_ENABLED: "true", SPENDING_ENABLED: "true", + PRIVACY_SALT: "local-e2e-privacy", API_KEY_ENCRYPTION_KEY: "test-only-key-encryption-secret-32-characters", + CONTEXT_API_KEY: "fixture-context", TYPESAFE_API_KEY: "fixture", AI_GATEWAY_DISABLED: "true", LIMITER: limiter, + STATS: { get: async (key: string) => kv.get(key) ?? null, put: async (key: string, value: string) => { kv.set(key, value); } }, + ACCOUNT_AE: { writeDataPoint(point: { doubles: number[] }) { events.push(point); } }, + LONG_CONTEXT_AE: { writeDataPoint(point: AnalyticsPoint) { longEvents.push(point); } }, +} as unknown as Env & { APP_DB: typeof db }); +let secret = ""; +const body = { url: "https://example.com/article", labels: ["renewing", "not-renewing"] }; +async function send(payload: Record = body, idem?: string) { + const request = new Request("https://classifier.dev/v1/classify", { method: "POST", headers: { + authorization: `Bearer ${secret}`, "content-type": "application/json", "cf-connecting-ip": "203.0.113.1", + ...(idem ? { "idempotency-key": idem } : {}), + }, body: JSON.stringify(payload) }); + let response; + try { response = await accountApi(request, env, ctx); } + catch (error) { if (!(error instanceof AppError)) throw error; response = Response.json({ error: error.message }, { status: error.status }); } + assert.ok(response); + await flush(); + return { response, data: await response.json() as any }; +} +try { + const directory = new URL("../migrations/postgres/", import.meta.url); + for (const name of (await readdir(directory)).filter(name => name.endsWith(".sql")).sort()) await pg.exec(await readFile(new URL(name, directory), "utf8")); + await provisionTestAccount(new Request("http://localhost/auth/demo"), env); + secret = String((await performAction("local-demo", { type: "enroll", client: "Codex" }, env)).secret); + await db.prepare("UPDATE app_accounts SET paid_balance=balance WHERE id='local-demo'").run(); + for (const spending of ["true", "false"]) { + env.SPENDING_ENABLED = spending; + const { response, data } = await send({ ...body, include: ["markdown", "html"] }); + assert.equal(response.status, 200, JSON.stringify(data)); + assert.equal(data.results[0].label, "renewing"); + assert.ok(data.article.markdown.includes("contract")); + assert.ok(data.article.html.includes("
")); + assert.equal(data.pricing.scrape_usd, 0.0022); + assert.equal(data.pricing.total_usd, (2200000 + reportedTokens * 42) / 1e9); + const ledger = await db.prepare("SELECT status,actual_nano::text AS nano FROM app_usage WHERE id=?").bind(response.headers.get("x-request-id")).first(); + assert.equal(ledger?.status, "completed"); + assert.equal(ledger?.nano, String(2200000 + reportedTokens * 42)); + report.results.push({ name: `URL and both formats, spending=${spending}`, data, ledger }); + } + env.SPENDING_ENABLED = "true"; + const compact = await send(body, "one-url"); + assert.equal(compact.response.status, 200); + assert.equal(compact.data.article.markdown, undefined); + assert.equal(compact.data.article.html, undefined); + let calls = scrapeCalls; + assert.equal((await send(body, "one-url")).response.status, 409); + assert.equal(scrapeCalls, calls); + report.results.push({ name: "compact response and duplicate does not scrape", status: "passed" }); + for (const payload of [{ ...body, input: "ambiguous" }, { ...body, url: "http://127.0.0.1" }, { ...body, url: "file:///etc/passwd" }, { ...body, include: ["pdf"] }, { ...body, labels: ["one"] }]) { + calls = scrapeCalls; + assert.equal((await send(payload)).response.status, 400); + assert.equal(scrapeCalls, calls); + } + report.results.push({ name: "invalid requests rejected before scrape", status: "passed" }); + for (const [failure, status, charged] of [["blocked", 422, 0], ["missing", 422, 0.0022], ["empty", 422, 0.0022], ["fail-final", 503, 0.0022], ["network", 502, 0.0022], ["oversized", 413, 0.0022]] as const) { + mode = failure; + const { response, data } = await send(); + assert.equal(response.status, status, JSON.stringify(data)); + assert.equal(data.pricing.total_usd, charged); + const ledger = await db.prepare("SELECT status,actual_nano::text AS nano FROM app_usage WHERE id=?").bind(response.headers.get("x-request-id")).first(); + assert.equal(ledger?.status, charged ? "completed" : "refunded"); + if (charged) assert.equal(ledger?.nano, "2200000"); + report.results.push({ name: failure, status, pricing: data.pricing }); + } + mode = "long"; + const lengthy = await send(); + assert.equal(lengthy.response.status, 200, JSON.stringify(lengthy.data)); + assert.equal(lengthy.data.pricing.total_usd, (2200000 + countTokens(longArticle) * 84) / 1e9); + assert.ok(lengthy.data.usage.long_context.context_tokens > 0); + report.results.push({ name: "long article classifies without truncation", pricing: lengthy.data.pricing }); + mode = "normal"; + const dimensions = await send({ url: body.url, dimensions: { renewal: body.labels } }); + assert.equal(dimensions.response.status, 200, JSON.stringify(dimensions.data)); + report.results.push({ name: "URL dimensions", data: dimensions.data }); + const mcp = await accountApi(new Request("https://classifier.dev/mcp", { method: "POST", headers: { + authorization: `Bearer ${secret}`, "content-type": "application/json", accept: "application/json, text/event-stream", + }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "classify_texts", arguments: { ...body, include: ["markdown"] } } }) }), env, ctx); + await flush(); + assert.ok(mcp); + const rpc = await mcp.json() as any; + assert.ok(!rpc.error && !rpc.result.isError, JSON.stringify(rpc)); + assert.ok(rpc.result.structuredContent.article.markdown.includes("contract")); + report.results.push({ name: "MCP URL content and billing", result: rpc.result.structuredContent }); + env.PAID_REQUEST_USD = "0.002"; calls = scrapeCalls; + assert.equal((await send()).response.status, 402); + assert.equal(scrapeCalls, calls); + delete env.PAID_REQUEST_USD; + report.results.push({ name: "request spending cap includes scrape", status: "passed" }); + calls = scrapeCalls; + const duplicate = await Promise.all([send(body, "concurrent-url"), send(body, "concurrent-url")]); + assert.deepEqual(duplicate.map(r => r.response.status).sort(), [200, 409]); + assert.equal(scrapeCalls, calls + 1); + report.results.push({ name: "concurrent duplicate admits one provider request", status: "passed" }); + for (const sql of ["UPDATE app_accounts SET balance=1,paid_balance=1 WHERE id='local-demo'", "UPDATE app_accounts SET balance=500000,paid_balance=0,billing_plan='free' WHERE id='local-demo'"]) { + await db.prepare(sql).run(); calls = scrapeCalls; + assert.equal((await send()).response.status, 402); + assert.equal(scrapeCalls, calls); + } + report.results.push({ name: "insufficient balance and unfunded workspace incur no provider cost", status: "passed" }); +} finally { + await mkdir("captures", { recursive: true }); + await writeFile(artifact, JSON.stringify(report, null, 2)); + globalThis.fetch = originalFetch; + provider.stop(true); + await pg.close(); +} +console.log(`${report.results.length} URL E2E scenarios passed; ${artifact}`); diff --git a/e2e/url.live.mjs b/e2e/url.live.mjs new file mode 100644 index 0000000..698855b --- /dev/null +++ b/e2e/url.live.mjs @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { mkdir, writeFile } from 'node:fs/promises'; +const base = process.env.CLASSIFIER_BASE_URL ?? 'https://classifier.dev'; +const key = process.env.CLASSIFIER_API_KEY; +assert.ok(key?.startsWith('classifier_agent_'), 'Set CLASSIFIER_API_KEY to a funded workspace key.'); +const headers = { authorization: `Bearer ${key}`, 'content-type': 'application/json' }; +const report = { base, at: new Date().toISOString(), results: [] }; +try { + for (const [url, labels, include] of [ + ['https://example.com', ['documentation', 'news'], ['markdown', 'html']], + ['https://www.paulgraham.com/avg.html', ['programming', 'cooking'], []], + ]) { + const idempotency = crypto.randomUUID(); + const body = JSON.stringify({ url, labels, include }); + const response = await fetch(`${base}/v1/classify`, { method: 'POST', headers: { ...headers, 'idempotency-key': idempotency }, body }); + const result = await response.json(); + report.results.push({ url, status: response.status, requestId: response.headers.get('x-request-id'), label: result.results?.[0]?.label, + pricing: result.pricing, usage: result.usage, article: { url: result.article?.url, markdownLength: result.article?.markdown?.length, htmlLength: result.article?.html?.length } }); + assert.equal(response.status, 200, JSON.stringify(result)); + assert.equal(result.results[0].label, labels[0]); + assert.equal(result.pricing.scrape_usd, 0.0022); + assert.ok(result.pricing.total_usd >= 0.0022); + assert.equal(result.article.url, url.endsWith('.com') ? `${url}/` : url); + if (include.length) { + assert.ok(result.article.markdown.includes('Example Domain')); + assert.ok(result.article.html.includes('<')); + } else { + assert.equal(result.article.markdown, undefined); + assert.equal(result.article.html, undefined); + } + const duplicate = await fetch(`${base}/v1/classify`, { method: 'POST', headers: { ...headers, 'idempotency-key': idempotency }, body }); + assert.equal(duplicate.status, 409, await duplicate.text()); + } + const anonymous = await fetch(`${base}/v1/classify`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ url: 'https://example.com', labels: ['a', 'b'] }) }); + assert.equal(anonymous.status, 401, await anonymous.text()); + report.results.push({ name: 'anonymous scrape denied', status: 401 }); +} finally { + await mkdir('captures', { recursive: true }); + await writeFile(`captures/url-live-${new URL(base).hostname}.json`, JSON.stringify(report, null, 2)); +} +console.log(JSON.stringify(report, null, 2)); diff --git a/package.json b/package.json index f5bd835..31253e2 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "test:e2e:long-context": "node e2e/long-context.mjs", "test:e2e:long-context-billing": "bun e2e/long-context-billing.ts", "test:e2e:long-context-job": "bun e2e/long-context-job.ts --full", - "test:e2e:whole-document": "node e2e/whole-document.mjs --full" + "test:e2e:whole-document": "node e2e/whole-document.mjs --full", + "test:e2e:url": "bun e2e/url-classification.ts" }, "keywords": [], "author": "Michael Ryaboy", diff --git a/src/docs.ts b/src/docs.ts index 6220936..146ff3c 100644 --- a/src/docs.ts +++ b/src/docs.ts @@ -1,3 +1,4 @@ +import { SCRAPE_PRICE } from "./scrape"; import { vsJevText } from "./vsjev"; import { roadmapDoc } from "./newsletter"; import { INPUT_PRICE_PER_MILLION, LONG_CONTEXT_PRICING } from "./lib/classification-pricing"; @@ -20,6 +21,49 @@ export const SPENDING_LIMITS = `SPENDING LIMITS whole-document UUID keys return the same job. Free keys are scoped to the IP and UTC day; workspace keys are scoped to the account.`; +export const URL_CLASSIFICATION = `SCRAPE AND CLASSIFY A URL + + Send one public HTTP(S) URL instead of input, inputs or items. Any public + website is supported; inaccessible, blocked or login-only pages may fail. + Use a funded workspace key from https://classifier.dev/app/keys. + + curl https://classifier.dev/v1/classify \\ + -H "Authorization: Bearer $CLASSIFIER_API_KEY" \\ + -H "Content-Type: application/json" \\ + -H "Idempotency-Key: article-001" \\ + -d '{ + "url": "https://example.com", + "labels": ["documentation", "news"], + "include": ["markdown", "html"] + }' + + Context.dev extracts the article once; Jev classifies the Markdown. + Results, confidence, scores and usage keep the normal classification shape. + article contains url and title. include opts into article.markdown and/or + article.html; omit it for compact output. HTML is untrusted page content: + sanitize it before rendering. labels, dimensions, instructions and multi + work as with text. Model is Jev; long articles require tier fast. + + Scraping costs $${SCRAPE_PRICE} per provider-billed request ($${(SCRAPE_PRICE * 1000).toFixed(2)}/1,000), plus + normal classification. Both formats use the same scrape, at no extra cost. + pricing separates scrape_usd, classification_usd and total_usd. Credits are + reserved before scraping; a typical URL request temporarily holds $0.0232 + and releases the unused amount after settlement. The workspace must have + paid credits or an active paid plan and enough balance for the whole hold. + Scraping counts toward the default $10 request limit. + + A successful scrape remains billable if classification fails. Provider-billed + errors (including processed 404s) and uncertain network failures retain the + scrape charge; confirmed unbilled failures release it. Read pricing even on + errors. No automatic scrape retries. Idempotency-Key rejects repeats with + 409 before provider spend; use a new key only for intentional new work. + + Limits: one URL per request, 8 MB extracted response, 250,000 article tokens, + and existing classification decision limits. No OCR or browser actions. + Article text is never silently truncated. Request source and article content + are not stored in the usage ledger. MCP accepts url and include on + classify_texts, classify_dimensions and classify_multi_label.`; + export const DOCS = `classifier.dev Zero-shot text classification over plain HTTP. You send text and a list of @@ -32,6 +76,9 @@ Agents: the OpenAPI 3.1 description is at https://classifier.dev/openapi.json and a short index at https://classifier.dev/llms.txt +${URL_CLASSIFICATION} + + TYPESAFE SDK COMPATIBILITY classifier.dev implements TypeSafe's System One wire contract at the same diff --git a/src/http/classification.ts b/src/http/classification.ts index e2e82df..1e2f6e3 100644 --- a/src/http/classification.ts +++ b/src/http/classification.ts @@ -46,7 +46,7 @@ export async function accountClassification(request: Request, env: AppEnv & Part } else { try { body = JSON.parse(text); } catch { throw new AppError(400, "Send valid JSON."); } if (!body || typeof body !== "object" || Array.isArray(body)) throw new AppError(400, "Send a JSON object."); - if (isLongContextRequest(body)) return spendingClassification( + if (Object.hasOwn(body, "url") || isLongContextRequest(body)) return spendingClassification( new Request(request.url, { method: "POST", headers: request.headers, body: text }), env, source, ctx); const rawInputs = body.inputs ?? body.items ?? body.input; const inputs = typeof rawInputs === "string" ? [rawInputs] : rawInputs; diff --git a/src/http/spending-classification.ts b/src/http/spending-classification.ts index a1791f3..fe82540 100644 --- a/src/http/spending-classification.ts +++ b/src/http/spending-classification.ts @@ -1,3 +1,4 @@ +import { scrapeOptions, scrapeArticle, SCRAPE_NANODOLLARS, SCRAPE_PRICE, type Article } from "../scrape"; import { classificationPricing, withResponsePricing } from "../classification-usage"; import worker, { type Env } from "../index"; import { newMeter } from "../cost"; @@ -19,7 +20,10 @@ export async function spendingClassification(request: Request, env: AppEnv & Par let body: Record; try { body = JSON.parse(text); } catch { throw new AppError(400, "Send valid JSON."); } if (!body || typeof body !== "object" || Array.isArray(body)) throw new AppError(400, "Send a JSON object."); - const longContext = new URL(request.url).pathname !== "/v1/systemone" && isLongContextRequest(body); + const scrape = scrapeOptions(body); + if (scrape && new URL(request.url).pathname === "/v1/systemone") throw new SpendingError(400, "invalid_request", "Use /v1/classify for URL classification."); + if (scrape && !env.CONTEXT_API_KEY) throw new SpendingError(503, "scrape_unavailable", "URL scraping is temporarily unavailable."); + let longContext = new URL(request.url).pathname !== "/v1/systemone" && isLongContextRequest(body); if (longContext) { if (["inputs", "items", "input"].filter(key => Object.hasOwn(body, key)).length > 1) throw new SpendingError(400, "invalid_request", "Use only one of inputs, items or input for long context."); @@ -62,8 +66,10 @@ export async function spendingClassification(request: Request, env: AppEnv & Par const trial = body.model === "laya" || body.model === "kev" || body.model === "chunklaya" || dgemma; const quotedCharge = contextTokens !== undefined ? longContextCharge(contextTokens) : classificationCharge(trial ? 0 : 65536 * decisions, body.tier === "smart" ? decisions : 0); - const quote = Number((quotedCharge.nanodollars + 9999n) / 10000n); - if (quote > maxCredits) throw new SpendingError(402, "request_spending_limit", "This request exceeds the workspace request allowance. Split the batch."); + const classificationBound = scrape && longContextCharge(LONG_CONTEXT_MAX_TOKENS).nanodollars > quotedCharge.nanodollars + ? longContextCharge(LONG_CONTEXT_MAX_TOKENS).nanodollars : quotedCharge.nanodollars; + const quote = Number((classificationBound + BigInt(scrape ? SCRAPE_NANODOLLARS : 0) + 9999n) / 10000n); + if ((scrape && classificationBound + BigInt(SCRAPE_NANODOLLARS) > BigInt(limits.paidRequest)) || quote > maxCredits) throw new SpendingError(402, "request_spending_limit", "This request exceeds the workspace request allowance. Split the batch."); const idempotencyHash = idem ? await fingerprint(env, `account-idempotency:${idem}`) : null; const result = await env.APP_DB.prepare(`WITH owner AS ( SELECT a.id,a.balance,a.paid_balance,(a.paid_balance>0 OR (a.billing_plan IN ('pro','max','scale') AND a.reset_at::timestamptz>now())) AS funded,k.id AS agent_id FROM app_accounts a JOIN app_agents k ON k.account_id=a.id @@ -72,14 +78,14 @@ export async function spendingClassification(request: Request, env: AppEnv & Par INSERT INTO app_usage(id,account_id,agent_id,items,credits,status,created_at,paid_credits,usage_type,metering_mode,idempotency_key,classifications) SELECT ?,id,agent_id,?,?::bigint,'pending',?, GREATEST(0,?::bigint-(balance-paid_balance)),?,'tokens',?,? - FROM owner WHERE balance>0 AND (funded OR balance>=?) AND (NOT ?::boolean OR funded) ON CONFLICT DO NOTHING RETURNING * + FROM owner WHERE balance>0 AND ((funded AND NOT ?::boolean) OR balance>=?) AND (NOT ?::boolean OR funded) ON CONFLICT DO NOTHING RETURNING * ), debited AS ( UPDATE app_accounts a SET balance=a.balance-h.credits,paid_balance=a.paid_balance-h.paid_credits FROM held h WHERE a.id=h.account_id RETURNING a.id ), agent AS ( UPDATE app_agents a SET used=a.used+h.credits FROM held h,debited d WHERE a.id=h.agent_id AND d.id=h.account_id RETURNING a.id ) SELECT h.account_id,h.agent_id,h.credits,o.funded FROM held h JOIN owner o ON o.id=h.account_id JOIN agent k ON k.id=h.agent_id`) - .bind(keyHash, id, items, quote, now(), quote, `${source} · Classification`, idempotencyHash, decisions, quote, longContext) + .bind(keyHash, id, items, quote, now(), quote, `${source} · ${scrape ? "URL classification" : "Classification"}`, idempotencyHash, decisions, !!scrape, quote, longContext || !!scrape) .first<{ account_id: string; agent_id: string; credits: number; funded: boolean }>(); if (!result) { const reason = await env.APP_DB.prepare(`SELECT k.status,a.billing_hold, @@ -90,25 +96,42 @@ export async function spendingClassification(request: Request, env: AppEnv & Par if (!reason) throw new SpendingError(401, "invalid_api_key", "This workspace API key is not valid."); if (!["pending", "connected"].includes(reason.status)) throw new SpendingError(403, "inactive_api_key", "This workspace API key is paused or revoked."); if (reason.request_id) throw new SpendingError(409, "duplicate_request", "This workspace already admitted the idempotency key.", { requestId: reason.request_id }); + if (scrape && !reason.funded) throw new SpendingError(402, "scrape_payment_required", "URL scraping requires paid credits or an active paid subscription."); if (longContext && !reason.funded) throw new SpendingError(402, "long_context_payment_required", "Long context requires a funded workspace: add paid credits or use an active paid subscription. Signup credit alone does not qualify."); throw new SpendingError(402, "insufficient_balance", reason.billing_hold ? "Workspace billing is awaiting review; funds remain held." : "The workspace cannot fund a new request. Add funds to clear any debt or cover the free-credit reservation, or wait for in-flight reservations to settle.", { reservationUsd: quote / 100000 }); } const meter = newMeter(); - const permit = result.funded ? new Permit(limits.paidRequest, Date.now() + 90000) : undefined; + const permit = result.funded ? new Permit(limits.paidRequest - (scrape ? SCRAPE_NANODOLLARS : 0), Date.now() + 90000) : undefined; if (permit) { meter.permit = permit; meter.beforeCall = async () => {}; } const started = Date.now(); let response: Response; + let article: Article | undefined; + let scrapeCharged = false; try { + if (scrape) { + article = await scrapeArticle(scrape, env.CONTEXT_API_KEY!, request.signal, charged => { scrapeCharged = charged; }); + const { url: _url, include: _include, ...classification } = body; + body = { ...classification, input: article.markdown }; + longContext = isLongContextRequest(body); + if (longContext) { + contextTokens = longContextInputTokens([article.markdown]); + if (contextTokens > LONG_CONTEXT_MAX_TOKENS) throw new SpendingError(413, "scrape_too_large", "The article exceeds 250,000 tokens; use the whole-document API for larger documents."); + } + request = new Request(request.url, { method: "POST", headers: request.headers, body: JSON.stringify(body), signal: request.signal }); + } response = await worker.fetch(request, env as Env, ctx, { meter, account: { id: result.account_id, multiplier: result.funded ? 10 : 1 }, funded: result.funded }); if (permit?.error && !(response.ok && permit.error.code === "request_spending_limit")) response = errorResponse(permit.error); } catch (error) { response = error instanceof SpendingError ? errorResponse(error) : Response.json({ error: "Classification failed." }, { status: 502 }); } const payload = response.ok ? await response.clone().json().catch(() => null) as { usage?: { escalated?: unknown } } | null : null; - const escalations = longContext || new URL(request.url).pathname === "/v1/systemone" ? 0 : payload?.usage?.escalated; + const escalations = !response.ok || longContext || new URL(request.url).pathname === "/v1/systemone" ? 0 : payload?.usage?.escalated; const inputTokens = contextTokens ?? (trial ? 0 : classificationInputTokens(meter.tokens)); - const charge = response.ok && inputTokens !== null && typeof escalations === "number" && Number.isSafeInteger(escalations) && escalations >= 0 && escalations <= decisions + const inferenceCharge = response.ok && inputTokens !== null && typeof escalations === "number" && Number.isSafeInteger(escalations) && escalations >= 0 && escalations <= decisions ? contextTokens !== undefined ? longContextCharge(contextTokens) : classificationCharge(inputTokens, escalations) : null; + const charge = scrape + ? (response.ok && inferenceCharge === null ? null : { version: "2026-09-24-url-v1", nanodollars: (inferenceCharge?.nanodollars ?? 0n) + BigInt(scrapeCharged ? SCRAPE_NANODOLLARS : 0) }) + : inferenceCharge; const settle = async () => { const actual = meter.permit; actual?.close(); @@ -118,7 +141,7 @@ export async function spendingClassification(request: Request, env: AppEnv & Par ? tokens.reduce((sum, t) => sum + t.inputTokens!, 0) : null; for (let attempt = 0; attempt < 3; attempt++) { try { - if (!response.ok || (!longContext && (!actual || actual.used === 0))) await refundTokenReservation(env.APP_DB, id); + if (!scrapeCharged && (!response.ok || (!longContext && (!actual || actual.used === 0)))) await refundTokenReservation(env.APP_DB, id); else await settleTokenReservation(env.APP_DB, id, charge, { inputTokens: longContext ? upstreamInputTokens : inputTokens, outputTokens: tokens.length && tokens.every(t => t.outputTokens !== null) ? tokens.reduce((sum, t) => sum + t.outputTokens!, 0) : null, @@ -128,8 +151,8 @@ export async function spendingClassification(request: Request, env: AppEnv & Par tier: body.tier === "smart" ? "smart" : "fast", status: response.ok ? "success" : "error", items, inputTokens: longContext ? upstreamInputTokens : inputTokens, longContext: meter.longContext, outputTokens: tokens.every(t => t.outputTokens !== null) ? tokens.reduce((sum, t) => sum + t.outputTokens!, 0) : null, - cachedInputTokens: null, model: tokens.map(t => t.model).join(","), providerCostUsd: actual?.unknown ? null : (actual?.used ?? 0) / 1e9, - retailCostUsd: !response.ok ? 0 : charge ? Number(charge.nanodollars) / 1e9 : null, latencyMs: Date.now() - started, escalations: typeof escalations === "number" ? escalations : 0 }); + cachedInputTokens: null, model: tokens.map(t => t.model).join(","), providerCostUsd: actual?.unknown || scrapeCharged ? null : (actual?.used ?? 0) / 1e9, + retailCostUsd: charge ? Number(charge.nanodollars) / 1e9 : response.ok ? null : 0, latencyMs: Date.now() - started, escalations: typeof escalations === "number" ? escalations : 0 }); return; } catch { /* Durable pending funds remain unavailable until reconciliation. */ } } @@ -143,6 +166,21 @@ export async function spendingClassification(request: Request, env: AppEnv & Par } headers.set("x-request-id", id); headers.set("x-billing-status", "pending"); headers.set("cache-control", "no-store"); const resultResponse = new Response(response.body, { status: response.status, headers }); + if (scrape) { + const payload = await resultResponse.json() as Record; + const pricing = classificationPricing(meter, typeof escalations === "number" ? escalations : 0, contextTokens); + headers.delete("content-length"); + return Response.json({ ...payload, + ...(article ? { article: { url: article.url, title: article.title, + ...(scrape.include.includes("markdown") ? { markdown: article.markdown } : {}), + ...(scrape.include.includes("html") ? { html: article.html } : {}) } } : {}), + pricing: { ...pricing, rate_version: "2026-09-24-url-v1", scrape_requests: scrapeCharged ? 1 : 0, + usd_per_scrape: SCRAPE_PRICE, scrape_usd: scrapeCharged ? SCRAPE_PRICE : 0, + classification_usd: response.ok ? inferenceCharge ? Number(inferenceCharge.nanodollars) / 1e9 : null : 0, + estimated_usd: charge ? Number(charge.nanodollars) / 1e9 : null, + total_usd: charge ? Number(charge.nanodollars) / 1e9 : null, billing_status: "pending" }, + }, { status: response.status, headers }); + } return response.ok && new URL(request.url).pathname !== "/v1/systemone" ? withResponsePricing(resultResponse, { ...classificationPricing(meter, typeof escalations === "number" ? escalations : 0, contextTokens), diff --git a/src/index.ts b/src/index.ts index c9406a4..e29c3e1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,7 +71,7 @@ export interface Env extends LayaEnv, SpendingEnv { JEV_AE?: AnalyticsEngineDataset; LONG_CONTEXT_AE?: AnalyticsEngineDataset; LONG_CONTEXT_JOBS?: DurableObjectNamespace; - /** context.dev, for the chat's web search and page reads only. Never served. */ + /** Context.dev for page scraping and chat web tools. Never served. */ CONTEXT_API_KEY?: string; ENTERPRISE_API_KEY?: string; /** Dedicated operator credential for bulk agent work; independent of enterprise callers. */ @@ -306,6 +306,7 @@ const agentView = (origin: string) => ({ authentication: { required: false, optional_bearer: "Workspace keys use the workspace balance; Pro workspaces get 10x limits. Partner keys have separately arranged limits.", docs: `${origin}/auth.md` }, api: { classify: { method: "POST", url: `${origin}/v1/classify`, alias: `${origin}/`, body: { inputs: ["..."], labels: ["a", "b"], tier: "fast|smart", multi: false } }, + classify_url: { method: "POST", url: `${origin}/v1/classify`, body: { url: "https://example.com", labels: ["documentation", "news"], include: ["markdown", "html"] }, scrape_usd: 0.0022, authentication: "Funded workspace API key" }, classify_dimensions: { method: "POST", url: `${origin}/v1/classify`, body: { items: ["..."], dimensions: { team: ["billing", "platform"], kind: ["bug", "request"] } } }, classify_one: { method: "GET", url: `${origin}/{labels}/{text}`, query_form: `${origin}/?labels={a,b}&text={text}` }, typesafe_system_one: { method: "POST", url: `${origin}/v1/systemone`, compatibility: "TypeSafe System One wire contract; use the official SDK with a placeholder for anonymous use or a classifier_agent_ workspace key for workspace quota and billing" }, @@ -2099,6 +2100,7 @@ const worker = { return fail('Body must be a JSON object such as {"input":"...","labels":["a","b"]}. See https://classifier.dev', 400, "bad_json"); } const b = body as Record; + if (Object.hasOwn(b, "url")) return fail("URL classification requires a funded workspace key. Use POST /v1/classify with Authorization: Bearer classifier_agent_...", 401, "invalid_api_key"); if (b.model !== undefined && b.model !== "jev" && b.model !== "laya" && b.model !== "kev" && b.model !== "chunklaya") return fail('model must be "jev", "laya", "kev" or "chunklaya"', 400, "bad_model"); // `processing` without a model still means Laya: it is the lane selector diff --git a/src/mcp.ts b/src/mcp.ts index 78cfddf..4b43c16 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -122,6 +122,15 @@ function classifyOrError(status: number, body: Record): ToolRes type Row = { label?: string; labels?: string[]; confidence?: number | null; scores?: Record | null }; export function productServer(classify: ClassifyFn): McpServer { + const urlProperties = { + url: { type: "string", pattern: "^https?://", maxLength: 8192, description: "Scrape one public URL instead of inputs/items. Requires funded workspace access. Context.dev costs $0.0022 per billed attempt plus classification; long articles need Fast. Errors disclose retained charges. No automatic scrape retries." }, + include: { type: "array", items: { type: "string", enum: ["markdown", "html"] }, description: "With url: return article.markdown and/or article.html at no extra scrape cost. Omit for compact output." }, + }; + async function urlResult(args: Record, ctx: ToolContext, multi = false): Promise { + if (!Object.hasOwn(args, "url")) return null; + const result = await classify({ ...args, ...(multi ? { multi: true } : {}) }, ctx.req); + return { text: JSON.stringify(result.body), structured: result.body, ...(result.status !== 200 ? { isError: true } : {}) }; + } const tools: Tool[] = [ { name: "classify_texts", @@ -135,10 +144,11 @@ export function productServer(classify: ClassifyFn): McpServer { "so act on the sure ones and look at the rest yourself, or pass tier \"smart\" to have the unsure ones re-asked of a reasoning model.", inputSchema: { type: "object", - properties: { inputs: INPUTS_SCHEMA, labels: LABELS_SCHEMA, instructions: INSTRUCTIONS_SCHEMA, tier: TIER_SCHEMA, + properties: { ...urlProperties, inputs: INPUTS_SCHEMA, labels: LABELS_SCHEMA, instructions: INSTRUCTIONS_SCHEMA, tier: TIER_SCHEMA, model: { type: "string", enum: ["jev", "laya", "kev", "chunklaya"], description: "Jev is default. Default/explicit jev inputs over 32,000 characters use paid Fast-only long context: up to 250,000 original cl100k_base context tokens total, 20 documents, 32 decisions and a 1 MB body. Requires paid workspace balance or active paid subscription, not signup credit. $0.084/M original context tokens counted once across inputs, independent of dimensions. Final Jev uses selected evidence; eligible chunks may be omitted, disclosed in usage.long_context. No evidence returns 422 long_context_no_evidence without charge. Explicit 'chunklaya' retains legacy opt-in (4,000,000 characters/input, 20 inputs, subject to body limit). 'laya' (512-token context) and 'kev' (8K context) remain experimental alternatives." }, processing: { type: "string", enum: ["fast", "bulk"], description: "Optional. Implies Laya if model is omitted; has no effect with explicit Jev. With Laya, omit to select fast for one decision or bulk for batches automatically. Explicit fast accepts one decision. Shared capacity limits can return 429." } }, - required: ["inputs", "labels"], + required: ["labels"], + oneOf: [{ required: ["inputs"], not: { required: ["url"] } }, { required: ["url"], not: { required: ["inputs"] } }], additionalProperties: false, }, outputSchema: { @@ -164,8 +174,10 @@ export function productServer(classify: ClassifyFn): McpServer { }, required: ["results"], }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true }, async run(a, ctx) { + const scraped = await urlResult(a, ctx); + if (scraped) return scraped; const body = { inputs: strings(a.inputs, "inputs", 1, 1000), labels: strings(a.labels, "labels", 2, 100), @@ -187,12 +199,14 @@ export function productServer(classify: ClassifyFn): McpServer { title: "Classify several dimensions per text", description: "Classify each text by several named dimensions, such as team, urgency and kind, in one request. Returns a label, confidence, scores and model for each field. At most 1,000 item × dimension decisions; every field counts toward the quota. Use per-dimension instructions to define ambiguous categories.", inputSchema: { - type: "object", required: ["items", "dimensions"], additionalProperties: false, - properties: { items: INPUTS_SCHEMA, dimensions: DIMENSIONS_SCHEMA, instructions: { ...INSTRUCTIONS_SCHEMA, maxLength: 4000 }, tier: TIER_SCHEMA, + type: "object", required: ["dimensions"], oneOf: [{ required: ["items"], not: { required: ["url"] } }, { required: ["url"], not: { required: ["items"] } }], additionalProperties: false, + properties: { ...urlProperties, items: INPUTS_SCHEMA, dimensions: DIMENSIONS_SCHEMA, instructions: { ...INSTRUCTIONS_SCHEMA, maxLength: 4000 }, tier: TIER_SCHEMA, model: { type: "string", enum: ["jev", "laya", "kev", "chunklaya"] }, processing: { type: "string", enum: ["fast", "bulk"], description: "Optional. Implies Laya if model is omitted; has no effect with explicit Jev. Omit for automatic fast/bulk selection based on item × dimension decisions." } }, }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true }, async run(a, ctx) { + const scraped = await urlResult(a, ctx); + if (scraped) return scraped; const r = await classify({ items: strings(a.items, "items", 1, 1000), dimensions: a.dimensions ?? null, instructions: str(a.instructions, "instructions", { optional: true, max: 4000 }), tier: tier(a.tier), ...(a.model !== undefined ? { model: a.model } : {}), ...(a.processing !== undefined ? { processing: a.processing } : {}) }, ctx.req); @@ -209,6 +223,7 @@ export function productServer(classify: ClassifyFn): McpServer { inputSchema: { type: "object", properties: { + ...urlProperties, inputs: INPUTS_SCHEMA, labels: LABELS_SCHEMA, instructions: INSTRUCTIONS_SCHEMA, @@ -216,7 +231,8 @@ export function productServer(classify: ClassifyFn): McpServer { model: { type: "string", enum: ["jev", "laya", "kev", "chunklaya"] }, processing: { type: "string", enum: ["fast", "bulk"], description: "Optional. Implies Laya if model is omitted; has no effect with explicit Jev. Omit to select fast for up to four labels on one text, or bulk for larger work automatically." }, }, - required: ["inputs", "labels"], + required: ["labels"], + oneOf: [{ required: ["inputs"], not: { required: ["url"] } }, { required: ["url"], not: { required: ["inputs"] } }], additionalProperties: false, }, outputSchema: { @@ -237,8 +253,10 @@ export function productServer(classify: ClassifyFn): McpServer { }, required: ["results"], }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true }, async run(a, ctx) { + const scraped = await urlResult(a, ctx, true); + if (scraped) return scraped; const max = number(a.max_labels, "max_labels", 1, 100); if (max !== undefined && !Number.isInteger(max)) throw new InvalidParams("max_labels must be a whole number"); const body: Record = { diff --git a/src/openapi.ts b/src/openapi.ts index 1521964..a431f4d 100644 --- a/src/openapi.ts +++ b/src/openapi.ts @@ -13,6 +13,7 @@ import { MAX_DESIRED_LATENCY_MS, MIN_DESIRED_LATENCY_MS, ROADMAP, ROADMAP_KEYS } */ export const ERROR_CODES = [ ...SPENDING_ERROR_CODES, + "scrape_unavailable", "scrape_payment_required", "scrape_failed", "scrape_empty", "scrape_too_large", "bad_model", "bad_processing", "laya_input", "laya_rate_limit", "laya_unavailable", "chunklaya_input", "chunklaya_busy", "chunklaya_unavailable", "dgemma_input", "dgemma_busy", "dgemma_unavailable", "images_unsupported", @@ -1058,12 +1059,17 @@ export const OPENAPI = { { required: ["labels"], not: { required: ["dimensions"] } }, { required: ["dimensions"], not: { anyOf: [{ required: ["labels"] }, { required: ["multi"] }, { required: ["max_labels"] }] } }, ], + dependentRequired: { include: ["url"] }, + allOf: [{ if: { required: ["url"] }, then: { not: { anyOf: [{ required: ["input"] }, { required: ["inputs"] }, { required: ["items"] }] } } }], examples: [ + { url: "https://example.com", labels: ["documentation", "news"], include: ["markdown", "html"] }, { items: ["Checkout charges me twice"], dimensions: { team: ["billing", "identity", "platform"], kind: ["bug", "request", "question"] } }, { inputs: ["the checkout button does nothing", "love the new dark mode"], labels: ["bug", "praise", "feature"] }, { inputs: ["postgres index tuning for ML feature stores"], labels: ["databases", "ml", "frontend"], multi: true, max_labels: 2 }, ], properties: { + url: { type: "string", format: "uri", maxLength: 8192, description: "One public HTTP(S) URL instead of input/inputs/items. Requires a funded workspace key and sufficient balance. Context.dev scrapes once for $0.0022 per provider-billed attempt plus classification. Only Jev; long articles use Fast tier. No OCR. Up to 8 MB response and 250,000 tokens, never silently truncated. Successful scrapes and uncertain dispatched attempts remain charged even if classification fails; inspect pricing on errors." }, + include: { type: "array", items: { type: "string", enum: ["markdown", "html"] }, description: "With url only: opt into article.markdown and/or article.html. Both formats share one scrape; no extra fee. Omit for compact classification results. Treat HTML as untrusted." }, model: { type: "string", enum: ["jev", "laya", "kev", "chunklaya"], description: "Defaults to Jev unless processing implies Laya. Default/explicit jev inputs over 32,000 characters use paid Fast-only Jev long context: 600-token Chonkie chunks, parallel evidence screening and final Jev over whole eligible chunks in source order, bounded by 20,000 cl100k_base tokens and a conservative provider estimate. Eligible evidence may be omitted; usage.long_context discloses selection. Requires paid workspace balance or active paid subscription, not signup credit. Synchronous limits: 250,000 original cl100k_base tokens, 20 documents, 32 decisions, 1 MB body. One whole document on POST / or /v1/classify supports 10M tokens and 100 MB as a background job. Price: $0.084/M original context tokens summed once across inputs, independent of dimensions and actual inference usage. Explicit chunklaya retains the legacy opt-in (4,000,000 characters/input, 20 inputs, subject to 1 MB body; chunklaya/multilingual results; no Smart). Laya and Kev are experimental Beam models with 512-token and 8,192-token contexts respectively, 2–16 short labels, text ≤2,000 characters and instructions ≤400 characters. Results use jev/laya or jev/kev; Jev calibration claims do not apply." }, processing: { type: "string", enum: ["fast", "bulk"], description: "Implies Laya when model is omitted. Accepted but has no effect with explicit model jev, which handles batching automatically. When omitted for Laya, automatically selects fast for one decision with up to 4 yes/no questions, otherwise bulk. Explicit Laya lanes are honored. Fast allows 60 questions/min and 2,000/day per caller. Bulk chunks batches up to 1,000 questions per call, 1,000/min and 20,000/day. These caps also apply to paid/operator keys. Same model weights in both lanes. Overload returns 429; a cold bulk worker returns 503 with Retry-After. Smart review is independent." }, dimensions: DIMENSIONS_SCHEMA, @@ -1132,6 +1138,10 @@ export const OPENAPI = { type: "object", description: "USD customer pricing, separate from upstream provider spend. total_usd is zero for unbilled requests, null when unknown, and before credit rounding. Pending settlement is not a receipt. Ordinary pricing.input_tokens includes billable primary calls; Jev long context uses original cl100k_base context tokens summed once across inputs at $0.084/M, independent of dimensions and actual screening/final usage. Plain-text responses and TypeSafe-compatible responses retain their existing shapes.", properties: { + scrape_requests: { type: "integer", minimum: 0, maximum: 1 }, + usd_per_scrape: { type: "number", const: 0.0022 }, + scrape_usd: { type: "number", description: "Retained scrape charge, including on errors after a billed or uncertain provider attempt." }, + classification_usd: { type: ["number", "null"] }, currency: { const: "USD" }, rate_version: { type: "string" }, basis: { type: "string", const: "original_context_tokens", description: "Present for Jev long context." }, tokenizer: { type: "string", const: "cl100k_base", description: "Present for Jev long-context billing." }, @@ -1151,6 +1161,7 @@ export const OPENAPI = { properties: { usage: { $ref: "#/components/schemas/TokenUsage" }, pricing: { $ref: "#/components/schemas/ClassificationPricing" }, + article: { type: "object", properties: { url: { type: "string" }, title: { type: "string" }, markdown: { type: "string" }, html: { type: "string" } } }, label: { type: "string" }, confidence: { type: ["number", "null"], @@ -1195,6 +1206,7 @@ export const OPENAPI = { type: "object", properties: { pricing: { $ref: "#/components/schemas/ClassificationPricing" }, + article: { type: "object", properties: { url: { type: "string" }, title: { type: "string" }, markdown: { type: "string" }, html: { type: "string" } } }, tier: { type: "string" }, model: { type: "string", @@ -1343,6 +1355,7 @@ export const OPENAPI = { type: "object", required: ["error", "code"], properties: { + pricing: { $ref: "#/components/schemas/ClassificationPricing", description: "On admitted URL requests, reports any retained scrape charge even when classification fails." }, error: { type: "string", description: "Human-readable message saying what to change." }, code: { type: "string", @@ -1390,6 +1403,8 @@ export const OPENAPI = { export const LLMS_TXT = `# classifier.dev +URL classification: POST /v1/classify with url, labels (or dimensions), and optional include: ["markdown","html"]. Funded workspace key required. Context.dev scraping: $2.20/1,000 requests plus normal classification. MCP: classify_texts, classify_dimensions, classify_multi_label accept url and include. Read pricing even on errors. + > Zero-shot text classification over plain HTTP, with no API key and no account. > Send text and a list of labels, get back the label that fits and a calibrated > confidence. Up to 1,000 texts per request, back in about a second. diff --git a/src/pages.ts b/src/pages.ts index 1509cdb..dba320b 100644 --- a/src/pages.ts +++ b/src/pages.ts @@ -1,3 +1,4 @@ +import { SCRAPE_PRICE } from "./scrape"; /** * The pages that are not the API reference: who runs this, how to reach them, * what is logged, what it costs, where developers and agents start, and how to @@ -8,7 +9,7 @@ * canonical document and nothing can drift between the three. */ -import { SPENDING_LIMITS } from "./docs"; +import { SPENDING_LIMITS, URL_CLASSIFICATION } from "./docs"; import { SITE, SITE_UPDATED } from "./wellknown"; import { codeLang } from "./ui"; import { BILLING_PLANS, formatCreditsUsd } from "./lib/billing"; @@ -171,6 +172,8 @@ export const DEVELOPERS = `classifier.dev developers From reading this to a first classification, with no account, key or sign-up in between. The API is free within per-IP limits; production is the sandbox. +${URL_CLASSIFICATION} + QUICKSTART @@ -469,6 +472,12 @@ Try classifier.dev without an API key, account or card. Create a workspace when you want usage credits, named API keys, shared billing and usage by connection. Every plan includes fast and smart classification, REST, MCP and the CLI. +URL scraping costs $${(SCRAPE_PRICE * 1000).toFixed(2)}/1,000 provider-billed requests plus normal +classification. Markdown and HTML are included on request at no extra cost. +Requires funded access and enough balance for the up-front reservation. +Successful scrapes remain billable if classification fails; errors disclose +any retained scrape charge. See https://classifier.dev/developers. + FREE diff --git a/src/pricingui.ts b/src/pricingui.ts index d037247..f7e208b 100644 --- a/src/pricingui.ts +++ b/src/pricingui.ts @@ -1,3 +1,4 @@ +import { SCRAPE_PRICE } from "./scrape"; import { FOOT, HOME_CSS, META, NAV } from "./home"; import { BILLING_PLANS, formatCreditsUsd } from "./lib/billing"; import { INPUT_PRICE_PER_MILLION, ESCALATION_PRICE_PER_THOUSAND, LONG_CONTEXT_PRICING } from "./lib/classification-pricing"; @@ -62,7 +63,9 @@ export function pricingHtml(signedIn = false) { Contact sales

Usage prices

+

Scrape and classify a public URL with a funded workspace key. Request Markdown and HTML at no extra cost. Provider-billed scrapes remain charged if classification fails; errors disclose the charge. See the URL API.

+ diff --git a/src/scrape.ts b/src/scrape.ts new file mode 100644 index 0000000..2435cde --- /dev/null +++ b/src/scrape.ts @@ -0,0 +1,81 @@ +import { readDimensions } from "./dimensions"; +import { SpendingError } from "./spending/policy"; + +export const SCRAPE_NANODOLLARS = 2_200_000; +export const SCRAPE_PRICE = SCRAPE_NANODOLLARS / 1e9; +export interface ScrapeOptions { url: string; include: string[] } +export interface Article { url: string; markdown: string; html?: string; title?: string } +export function scrapeOptions(body: Record): ScrapeOptions | undefined { + if (!Object.hasOwn(body, "url")) { + if (Object.hasOwn(body, "include")) throw new SpendingError(400, "invalid_request", "include requires url."); + return; + } + const invalid = (message: string): never => { throw new SpendingError(400, "invalid_request", message); }; + if (["input", "inputs", "items"].some(key => Object.hasOwn(body, key))) invalid("Use url or text inputs, not both."); + if (typeof body.url !== "string" || body.url.length > 8192) invalid("url must be a complete public HTTP or HTTPS URL, at most 8192 characters."); + let url: URL; + try { url = new URL(body.url as string); } catch { return invalid("url must be a complete public HTTP or HTTPS URL."); } + const host = url.hostname.toLowerCase().replace(/\.$/, ""); + if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || + !host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || host.endsWith(".internal") || + host.includes(":") || /^(0|10|127|169\.254|192\.168|172\.(1[6-9]|2\d|3[01]))\./.test(host)) + invalid("url must address a public HTTP or HTTPS website without embedded credentials."); + const include = body.include ?? []; + if (!Array.isArray(include) || include.some(format => !["markdown", "html"].includes(format))) invalid('include must be an array containing "markdown" and/or "html".'); + if (body.model !== undefined && body.model !== "jev") invalid("URL classification uses model jev."); + if (body.processing !== undefined) invalid("processing is not supported for URL classification."); + if (body.tier !== undefined && !["fast", "smart"].includes(String(body.tier))) invalid('tier must be "fast" or "smart".'); + if (body.instructions !== undefined && (typeof body.instructions !== "string" || body.instructions.length > 4000)) invalid("instructions must fit 4000 characters."); + if (Object.hasOwn(body, "dimensions")) { + if (["labels", "multi", "max_labels"].some(key => Object.hasOwn(body, key))) invalid("dimensions cannot be combined with labels, multi or max_labels."); + try { readDimensions(body.dimensions); } catch { invalid("Provide valid classification dimensions."); } + } else { + const labels = body.labels; + if (!Array.isArray(labels) || labels.length < 2 || labels.length > 100 || labels.some(label => typeof label !== "string" || !label.trim() || label.length > 200) || new Set(labels).size !== labels.length) + invalid("Provide 2–100 distinct nonempty labels, at most 200 characters each."); + } + return { url: url.href, include: include as string[] }; +} + +export async function scrapeArticle(options: ScrapeOptions, key: string, signal: AbortSignal, + billed: (charged: boolean) => void): Promise
{ + const endpoint = new URL("https://api.context.dev/v1/web/scrape/markdown"); + endpoint.searchParams.set("url", options.url); + endpoint.searchParams.set("useMainContentOnly", "true"); + endpoint.searchParams.set("includeHTML", String(options.include.includes("html"))); + endpoint.searchParams.set("pdf[ocr]", "false"); + endpoint.searchParams.set("timeoutOpts[milliseconds]", "45000"); + endpoint.searchParams.set("timeoutOpts[behavior]", "fail"); + // A lost response can still incur provider cost. Keep the bounded charge unless + // the provider explicitly confirms it did not bill this attempt; never retry. + if (signal.aborted) throw new SpendingError(400, "invalid_request", "The request was cancelled before scraping."); + billed(true); + try { + const response = await fetch(endpoint, { headers: { authorization: `Bearer ${key}` }, redirect: "manual", + signal: AbortSignal.any([signal, AbortSignal.timeout(50000)]) }); + const reader = response.body?.getReader(); + if (!reader) throw new Error("Missing body"); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > 8_000_000) { await reader.cancel(); throw new SpendingError(413, "scrape_too_large", "The scraped response exceeds 8 MB."); } + chunks.push(value); + } + const bytes = new Uint8Array(size); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; } + const data = JSON.parse(new TextDecoder().decode(bytes)); + if (data?.key_metadata?.credits_consumed === 0) billed(false); + if (!response.ok || data?.success !== true) throw new SpendingError(response.status === 429 ? 503 : 422, "scrape_failed", "Context.dev could not retrieve this URL. The page may be unavailable, blocked, or require a login."); + if (typeof data.markdown !== "string" || !data.markdown.trim()) throw new SpendingError(422, "scrape_empty", "The URL returned no readable article text."); + if (options.include.includes("html") && typeof data.html !== "string") throw new SpendingError(502, "scrape_failed", "Context.dev did not return the requested HTML."); + return { url: options.url, markdown: data.markdown, + ...(options.include.includes("html") ? { html: data.html } : {}), + ...(typeof data.metadata?.title === "string" ? { title: data.metadata.title } : {}) }; + } catch (error) { + if (error instanceof SpendingError) throw error; + throw new SpendingError(502, "scrape_failed", "The scrape did not complete. Check pricing before retrying; a dispatched request may have incurred a scrape charge."); + } +} diff --git a/src/spending/policy.ts b/src/spending/policy.ts index 4896a8c..1a72608 100644 --- a/src/spending/policy.ts +++ b/src/spending/policy.ts @@ -100,7 +100,10 @@ export const SPENDING_ERROR_CODES = ["operator_daily_budget", "free_daily_budget export function errorResponse(error: SpendingError): Response { const retryable = (error.status === 429 || error.status === 503) && !["reputation_budget", "spending_configuration", "unpriced_model"].includes(error.code); const retryAfter = retryable ? Number(error.details.retryAfter ?? 60) : undefined; - const action = error.code === "duplicate_request" ? "Do not repeat this operation with a new key: it may already have run. Use the original response or request ID. Cached responses are not stored." + const action = error.code === "scrape_unavailable" ? "Scraping is temporarily unavailable. Retry later or submit article text directly." + : error.code === "scrape_payment_required" ? "Add paid credits or choose a paid subscription, then use a workspace API key." + : error.code.startsWith("scrape_") ? "Check the public URL and the pricing in this response. Use article text directly when the page cannot be scraped. A new request may incur another scrape charge." + : error.code === "duplicate_request" ? "Do not repeat this operation with a new key: it may already have run. Use the original response or request ID. Cached responses are not stored." : error.code === "payload_too_large" || error.code === "request_spending_limit" ? "Split the batch, shorten input text, labels or instructions, or use a funded workspace API key. Repeating the same request will not increase its allowance." : error.code === "insufficient_balance" ? "Wait for in-flight reservations to settle, add funds at https://classifier.dev/app/billing, or contact support with the request ID if billing is under review." : error.code === "caller_identity" ? "Send the request directly to https://classifier.dev, or use a funded workspace key. Caller-supplied forwarding headers cannot establish a free allowance."
UsagePrice
URL scraping$${(SCRAPE_PRICE * 1000).toFixed(2)} / 1,000 requests + classification
Input tokens$${INPUT_PRICE_PER_MILLION.toFixed(3)} / million
Smart escalation+$${ESCALATION_PRICE_PER_THOUSAND.toFixed(2)} / 1,000
Jev long context (original context tokens)$${(LONG_CONTEXT_PRICING.inputNanodollars / 1000).toFixed(3)} / million