diff --git a/harness/src/web/config.ts b/harness/src/web/config.ts new file mode 100644 index 000000000..e10d21274 --- /dev/null +++ b/harness/src/web/config.ts @@ -0,0 +1,57 @@ +/** + * Config for the `web::fetch` worker. Pulled from the `web:` section + * of the harness config.yaml; missing keys fall back to safe defaults. + * + * The numeric caps here are the HARD ceilings — a caller can ask for + * less (smaller timeout, smaller max_bytes) per-call, but never more. + * That keeps a runaway agent from spending the whole budget on one + * fetch. + */ + +import { getNumber, getSection, getString } from '../runtime/config.js'; + +export type WebConfig = { + /** Hard ceiling on per-request timeout. */ + max_timeout_ms: number; + /** Hard ceiling on response body bytes accepted before truncation. */ + max_response_bytes: number; + /** Max redirect hops before giving up. */ + max_redirects: number; + /** UA we identify ourselves as. */ + user_agent: string; + /** + * Allow requests to loopback addresses (`127.0.0.0/8`, `::1`, + * `localhost`). Defaults to `true` because the iii harness itself + * binds workers like `iii-http` to `localhost:3111` and the agent's + * own smoke-test workflow targets them. Flip to `false` for + * non-dev deployments where loopback hosts sensitive services + * (admin UIs, debug ports, cloud agents). Other private ranges + * (RFC1918, link-local incl. AWS metadata, ULA, multicast) stay + * blocked unconditionally. + */ + allow_loopback: boolean; +}; + +const DEFAULTS: WebConfig = { + max_timeout_ms: 30_000, + max_response_bytes: 5 * 1024 * 1024, + max_redirects: 5, + user_agent: 'iii-harness/0.1 (+web::fetch)', + allow_loopback: true, +}; + +function getBoolean(cfg: Record, key: string, fallback: boolean): boolean { + const v = cfg[key]; + return typeof v === 'boolean' ? v : fallback; +} + +export function loadWebConfig(cfg: Record): WebConfig { + const section = getSection(cfg, 'web'); + return { + max_timeout_ms: getNumber(section, 'max_timeout_ms', DEFAULTS.max_timeout_ms), + max_response_bytes: getNumber(section, 'max_response_bytes', DEFAULTS.max_response_bytes), + max_redirects: getNumber(section, 'max_redirects', DEFAULTS.max_redirects), + user_agent: getString(section, 'user_agent', DEFAULTS.user_agent), + allow_loopback: getBoolean(section, 'allow_loopback', DEFAULTS.allow_loopback), + }; +} diff --git a/harness/src/web/fetch.ts b/harness/src/web/fetch.ts new file mode 100644 index 000000000..a64974761 --- /dev/null +++ b/harness/src/web/fetch.ts @@ -0,0 +1,386 @@ +/** + * The actual HTTP call for `web::fetch`. + * + * SSRF-safe IP pinning: the host is resolved + validated ONCE in + * `ssrf.ts::checkTarget`, then the socket is pinned to that exact IP via + * the `lookup` request option — node never re-resolves, defeating DNS + * rebinding (TOCTOU between the check and the connect). Crucially the + * request URL stays on the ORIGINAL hostname, so SNI, TLS certificate + * validation, and the `Host` header all bind to the real host. (An + * earlier version rewrote the URL to the IP literal, which made HTTPS + * fail certificate validation against the IP — see fetch.integration.test.) + * + * Built on `node:http` / `node:https` (not global `fetch`) because only the + * node request API exposes the per-request `lookup` + `servername` hooks the + * pinning needs; undici is not a dependency here. Adds: + * - manual redirect handling so each hop re-runs the SSRF check, + * - streamed body read with a hard byte cap (truncate-and-mark, not throw), + * - cross-origin / scheme-downgrade Authorization + Cookie strip on redirect. + */ + +import { Buffer } from 'node:buffer'; +import * as nodeHttp from 'node:http'; +import type { IncomingHttpHeaders, IncomingMessage } from 'node:http'; +import * as nodeHttps from 'node:https'; +import type { Readable } from 'node:stream'; +import { logger } from '../runtime/otel.js'; +import type { WebConfig } from './config.js'; +import type { FetchPayload, FetchResult, ResponseFormat } from './schemas.js'; +import { type ParsedTarget, type SsrfPolicy, checkTarget, parseTarget } from './ssrf.js'; + +const HEADER_DENY_ON_REDIRECT = new Set(['authorization', 'cookie', 'proxy-authorization']); + +type RawResponse = { + status: number; + statusText: string; + headers: Record; + bytes: Buffer; + truncated: boolean; +}; + +type RequestOutcome = + | { kind: 'response'; resp: RawResponse } + | { kind: 'timeout' } + | { kind: 'transport_error'; message: string }; + +/** + * Read a node Readable (IncomingMessage) into a Buffer, stopping at `cap` + * bytes and marking `truncated`. Mirrors the web-stream capped reader it + * replaced; exported for unit tests. + */ +export function readIncomingCapped( + stream: Readable, + cap: number, +): Promise<{ bytes: Buffer; truncated: boolean }> { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + + const finish = (truncated: boolean) => { + if (settled) return; + settled = true; + resolve({ bytes: Buffer.concat(chunks), truncated }); + }; + + stream.on('data', (raw: Buffer | string) => { + if (settled) return; + const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(raw); + if (total + chunk.length > cap) { + const remaining = cap - total; + if (remaining > 0) chunks.push(chunk.subarray(0, remaining)); + // Resolve BEFORE destroy so a post-destroy 'error'/'close' can't + // flip the outcome — the cap is the contract. + finish(true); + stream.destroy(); + return; + } + chunks.push(chunk); + total += chunk.length; + }); + stream.on('end', () => finish(false)); + stream.on('close', () => finish(false)); + stream.on('error', (err) => { + if (settled) return; + settled = true; + reject(err); + }); + }); +} + +function flattenIncomingHeaders(h: IncomingHttpHeaders): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(h)) { + if (v === undefined) continue; + // node already lower-cases keys; set-cookie arrives as an array. + out[k.toLowerCase()] = Array.isArray(v) ? v.join(', ') : v; + } + return out; +} + +/** + * `lookup` shim that forces every connection attempt onto the pre-validated + * IP, ignoring the hostname node would otherwise resolve. Matches the + * `net.LookupFunction` contract, including the `{ all: true }` variant. + */ +function pinnedLookup(address: string, family: 4 | 6): nodeHttps.RequestOptions['lookup'] { + return ((hostname: string, options: unknown, callback: unknown) => { + void hostname; + const opts = (options ?? {}) as { all?: boolean }; + if (opts.all) { + (callback as (e: Error | null, addrs: Array<{ address: string; family: number }>) => void)( + null, + [{ address, family }], + ); + } else { + (callback as (e: Error | null, addr: string, fam: number) => void)(null, address, family); + } + }) as nodeHttps.RequestOptions['lookup']; +} + +/** Drop any caller-supplied Host header so node derives it from the URL. */ +function withoutHostHeader(headers: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase() === 'host') continue; + out[k] = v; + } + return out; +} + +function performRequest( + parsed: ParsedTarget, + address: string, + family: 4 | 6, + method: string, + headers: Record, + body: string | undefined, + timeoutMs: number, + maxBytes: number, +): Promise { + const isHttps = parsed.url.protocol === 'https:'; + const mod = isHttps ? nodeHttps : nodeHttp; + + return new Promise((resolve) => { + let settled = false; + const done = (o: RequestOutcome) => { + if (settled) return; + settled = true; + resolve(o); + }; + + const options: nodeHttps.RequestOptions = { + method, + headers: withoutHostHeader(headers), + lookup: pinnedLookup(address, family), + }; + // SNI + certificate identity bind to the real hostname (not the IP). + if (isHttps) options.servername = parsed.hostname; + + const req = mod.request(parsed.url, options, (res: IncomingMessage) => { + readIncomingCapped(res, maxBytes).then( + ({ bytes, truncated }) => + done({ + kind: 'response', + resp: { + status: res.statusCode ?? 0, + statusText: res.statusMessage ?? '', + headers: flattenIncomingHeaders(res.headers), + bytes, + truncated, + }, + }), + (err) => + done({ + kind: 'transport_error', + message: err instanceof Error ? err.message : String(err), + }), + ); + }); + + req.setTimeout(timeoutMs, () => { + req.destroy( + Object.assign(new Error(`request timed out after ${timeoutMs}ms`), { __timeout: true }), + ); + }); + req.on('error', (err: Error & { __timeout?: boolean }) => { + if (err.__timeout) done({ kind: 'timeout' }); + else done({ kind: 'transport_error', message: err.message }); + }); + + if (body !== undefined && method !== 'GET' && method !== 'HEAD') req.write(body); + req.end(); + }); +} + +function encodeBody(bytes: Buffer, format: ResponseFormat): string { + if (format === 'base64') return bytes.toString('base64'); + // 'json' falls through here too — we always return the raw text in + // `body` and additionally surface the parsed value in `json` (see + // tryParseJson below). This keeps `body` typed as `string` and gives + // the agent both the structured value and the original text. + return bytes.toString('utf8'); +} + +function tryParseJson( + text: string, +): { value: unknown; error?: undefined } | { value?: undefined; error: string } { + try { + return { value: JSON.parse(text) }; + } catch (err) { + return { error: err instanceof Error ? err.message : String(err) }; + } +} + +/** + * If `payload.json` is set, force the request to a JSON shape: + * - body = JSON.stringify(payload.json) + * - content-type = application/json (only if the caller didn't set one) + * Returns the effective body string and headers. `payload.json` wins + * over `payload.body` when both are set — the description warns the + * caller about this. + */ +function applyJsonPayload( + payload: FetchPayload, + headers: Record, +): { body: string | undefined; headers: Record } { + if (payload.json === undefined) { + return { body: payload.body, headers }; + } + const lowered: Record = {}; + for (const [k, v] of Object.entries(headers)) lowered[k.toLowerCase()] = v; + if (!lowered['content-type']) { + headers = { ...headers, 'content-type': 'application/json' }; + } + return { body: JSON.stringify(payload.json), headers }; +} + +export function stripCrossOriginAuth( + headers: Record, + from: URL, + to: URL, +): Record { + // Strip auth/cookie when the redirect leaves the origin OR downgrades the + // scheme (https → http). A same-host downgrade still leaks credentials in + // cleartext, so host equality alone is not enough. + const sameHost = from.host === to.host; + const schemeDowngrade = from.protocol === 'https:' && to.protocol === 'http:'; + if (sameHost && !schemeDowngrade) return headers; + const out: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (HEADER_DENY_ON_REDIRECT.has(k.toLowerCase())) continue; + out[k] = v; + } + return out; +} + +export async function executeFetch(payload: FetchPayload, cfg: WebConfig): Promise { + const t0 = Date.now(); + const method = payload.method ?? 'GET'; + const followRedirects = payload.follow_redirects ?? true; + const responseFormat: ResponseFormat = payload.response_format ?? 'text'; + const timeoutMs = Math.min(payload.timeout_ms ?? cfg.max_timeout_ms, cfg.max_timeout_ms); + const maxBytes = Math.min(payload.max_bytes ?? cfg.max_response_bytes, cfg.max_response_bytes); + + let currentUrl = payload.url; + const baseHeaders: Record = { + 'user-agent': cfg.user_agent, + ...(payload.headers ?? {}), + }; + const jsonApplied = applyJsonPayload(payload, baseHeaders); + let currentHeaders = jsonApplied.headers; + const effectiveBody = jsonApplied.body; + const redirectChain: string[] = []; + const policy: SsrfPolicy = { allow_loopback: cfg.allow_loopback }; + + const telemetryHost = (() => { + try { + return new URL(payload.url).host; + } catch { + return 'unparseable'; + } + })(); + + const logResult = (result: FetchResult): FetchResult => { + const ms = Date.now() - t0; + if (result.ok) { + logger.info('web::fetch ok', { + host: telemetryHost, + method, + status: result.status, + bytes_truncated: result.bytes_truncated, + redirects: result.redirect_chain?.length ?? 0, + response_format: result.response_format, + ms, + }); + } else { + logger.warn('web::fetch error', { host: telemetryHost, method, error: result.error, ms }); + } + return result; + }; + + for (let hop = 0; hop <= cfg.max_redirects; hop++) { + const parsed = parseTarget(currentUrl); + if ('error' in parsed) { + return logResult({ ok: false, error: 'invalid_url', message: parsed.error }); + } + const check = await checkTarget(parsed, policy); + if (!check.ok) { + return logResult(check); + } + + const outcome = await performRequest( + parsed, + check.address, + check.family, + method, + currentHeaders, + effectiveBody, + timeoutMs, + maxBytes, + ); + if (outcome.kind === 'timeout') { + return logResult({ + ok: false, + error: 'timeout', + message: `request timed out after ${timeoutMs}ms (raise timeout_ms or shrink response with smaller max_bytes)`, + }); + } + if (outcome.kind === 'transport_error') { + return logResult({ ok: false, error: 'transport_error', message: outcome.message }); + } + const resp = outcome.resp; + + if (followRedirects && resp.status >= 300 && resp.status < 400) { + const location = resp.headers.location; + if (location) { + let nextUrl: URL; + try { + nextUrl = new URL(location, parsed.url); + } catch { + return logResult({ + ok: false, + error: 'transport_error', + message: `redirect target is not a valid URL: ${location}`, + }); + } + if (hop === cfg.max_redirects) { + return logResult({ + ok: false, + error: 'too_many_redirects', + message: `exceeded max_redirects (${cfg.max_redirects})`, + }); + } + redirectChain.push(currentUrl); + currentHeaders = stripCrossOriginAuth(currentHeaders, parsed.url, nextUrl); + currentUrl = nextUrl.toString(); + continue; + } + // 3xx without Location — fall through and return the response. + } + + const body = encodeBody(resp.bytes, responseFormat); + const result: FetchResult = { + ok: true, + status: resp.status, + status_text: resp.statusText, + headers: resp.headers, + body, + response_format: responseFormat, + bytes_truncated: resp.truncated, + }; + if (redirectChain.length > 0) result.redirect_chain = redirectChain; + if (responseFormat === 'json' && !resp.truncated) { + const json = tryParseJson(body); + if ('error' in json) result.parse_error = json.error; + else result.json = json.value; + } + return logResult(result); + } + + return logResult({ + ok: false, + error: 'too_many_redirects', + message: `exceeded max_redirects (${cfg.max_redirects})`, + }); +} diff --git a/harness/src/web/handlers/fetch.ts b/harness/src/web/handlers/fetch.ts new file mode 100644 index 000000000..925cba3ee --- /dev/null +++ b/harness/src/web/handlers/fetch.ts @@ -0,0 +1,30 @@ +/** + * `web::fetch` handler. Validates the payload, runs the SSRF-guarded + * HTTP call, and returns the structured envelope. Never throws — the + * envelope is the contract. + */ + +import type { ISdk } from '../../runtime/iii.js'; +import type { WebConfig } from '../config.js'; +import { executeFetch } from '../fetch.js'; +import { FetchPayloadSchema, type FetchResult, fetchFunctionOptions } from '../schemas.js'; + +export function register(iii: ISdk, cfg: WebConfig): void { + iii.registerFunction( + 'web::fetch', + async (payload: unknown): Promise => { + const parsed = FetchPayloadSchema.safeParse(payload); + if (!parsed.success) { + return { + ok: false, + error: 'invalid_payload', + message: parsed.error.issues + .map((i) => `${i.path.join('.') || ''}: ${i.message}`) + .join('; '), + }; + } + return executeFetch(parsed.data, cfg); + }, + fetchFunctionOptions, + ); +} diff --git a/harness/src/web/iii.worker.yaml b/harness/src/web/iii.worker.yaml new file mode 100644 index 000000000..7efeb0f56 --- /dev/null +++ b/harness/src/web/iii.worker.yaml @@ -0,0 +1,14 @@ +iii: v1 +name: web +language: node +deploy: binary +manifest: package.json +bin: iii-web +description: Outbound HTTP client on the iii bus (web::fetch). + +runtime: + kind: node + +scripts: + install: pnpm install + start: node ./dist/web/main.js --config ./config.yaml diff --git a/harness/src/web/main.ts b/harness/src/web/main.ts new file mode 100644 index 000000000..ead5afe1d --- /dev/null +++ b/harness/src/web/main.ts @@ -0,0 +1,9 @@ +#!/usr/bin/env node +import { bootstrapWorker } from '../runtime/worker.js'; +import { register } from './register.js'; + +await bootstrapWorker({ + name: 'web', + description: 'Outbound HTTP client on the iii bus (web::fetch).', + register: (iii, ctx) => register(iii, ctx), +}); diff --git a/harness/src/web/register.ts b/harness/src/web/register.ts new file mode 100644 index 000000000..e49005bdf --- /dev/null +++ b/harness/src/web/register.ts @@ -0,0 +1,15 @@ +/** + * Worker entry: load config, register every `web::*` handler. Mirrors + * `auth-credentials/register.ts`. + */ + +import { loadConfig } from '../runtime/config.js'; +import type { ISdk } from '../runtime/iii.js'; +import { loadWebConfig } from './config.js'; +import { register as registerFetch } from './handlers/fetch.js'; + +export async function register(iii: ISdk, ctx: { configPath: string }): Promise { + const cfg = await loadConfig(ctx.configPath); + const web = loadWebConfig(cfg); + registerFetch(iii, web); +} diff --git a/harness/src/web/schemas.ts b/harness/src/web/schemas.ts new file mode 100644 index 000000000..08c58d120 --- /dev/null +++ b/harness/src/web/schemas.ts @@ -0,0 +1,127 @@ +/** + * Wire schema + tool description for `web::fetch`. Zod validates + * ingress; `zodToJsonSchema` exports the same shape so the model sees + * a proper tool-call schema in `request_format`. + * + * The description steers the model AWAY from `shell::exec` curl — + * without that line the LLM keeps reaching for curl from training- + * data habit. Same lever `shell::exec`'s own description uses to keep + * the model from passing argv as an array. + */ + +import type { RegisterFunctionOptions } from 'iii-sdk'; +import { z } from 'zod'; +import { zodToJsonSchema } from 'zod-to-json-schema'; + +// Case-insensitive: 'get' / 'Get' / 'GET' all accepted, normalised to upper. +export const HttpMethodSchema = z.preprocess( + (v) => (typeof v === 'string' ? v.toUpperCase() : v), + z.enum(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']), +); +export type HttpMethod = z.infer; + +export const ResponseFormatSchema = z.enum(['text', 'base64', 'json']); +export type ResponseFormat = z.infer; + +export const FetchPayloadSchema = z.object({ + url: z.string().min(1).describe('Absolute http(s):// URL to fetch.'), + method: HttpMethodSchema.optional().describe( + 'HTTP method. Defaults to GET. Case-insensitive ("get" works).', + ), + headers: z + .record(z.string(), z.string()) + .optional() + .describe('Request headers. Lower-cased keys recommended.'), + body: z + .string() + .optional() + .describe('Raw request body as a string. Prefer `json` for JSON payloads.'), + json: z + .unknown() + .optional() + .describe( + 'Structured JSON payload. When set, the handler stringifies it and forces `content-type: application/json` — do NOT also set `body`. If both are present, `json` wins.', + ), + timeout_ms: z + .number() + .int() + .positive() + .optional() + .describe("Per-request timeout in ms. Capped by the worker's max_timeout_ms."), + max_bytes: z + .number() + .int() + .positive() + .optional() + .describe( + 'Cap on response body bytes. Larger responses are truncated and bytes_truncated:true is returned.', + ), + follow_redirects: z + .boolean() + .optional() + .describe( + 'Follow 3xx redirects. Defaults to true. Each hop is re-checked against the SSRF blocklist.', + ), + response_format: ResponseFormatSchema.optional().describe( + 'How to return the response body: "text" (default), "base64" for binary, or "json" to auto-parse application/json responses into the `json` field.', + ), +}); +export type FetchPayload = z.infer; + +export const FetchErrorCodeSchema = z.enum([ + 'invalid_payload', + 'invalid_url', + 'blocked_host', + 'timeout', + // Reserved: oversize responses are TRUNCATED (bytes_truncated:true), not + // errored — the worker does not currently emit `too_large`. Kept in the + // wire contract for forward-compat and parser tolerance. + 'too_large', + 'too_many_redirects', + 'transport_error', +]); +export type FetchErrorCode = z.infer; + +export type FetchResult = + | { + ok: true; + status: number; + status_text: string; + headers: Record; + body: string; + /** Set when response_format is "json" and the body parsed cleanly. */ + json?: unknown; + /** Set when response_format is "json" but parsing failed. */ + parse_error?: string; + response_format: ResponseFormat; + bytes_truncated: boolean; + /** The chain of intermediate URLs walked before the final response. Omitted when no redirects happened. */ + redirect_chain?: string[]; + } + | { + ok: false; + error: FetchErrorCode; + message: string; + status?: number; + }; + +const TOOL_DESCRIPTION = [ + 'Fetch a URL over HTTP(S) and return the response as a structured envelope.', + 'Use this INSTEAD of `shell::exec` with curl for any HTTP request — it', + 'returns {ok, status, headers, body} as JSON, enforces size/timeout caps,', + 'and blocks private / cloud-metadata / link-local addresses server-side', + '(SSRF guard; loopback is allowed by default for harness dev workflows).', + 'For JSON: pass `json: {...}` (auto-stringifies + sets content-type) and', + '`response_format: "json"` (auto-parses response into the `json` field).', + 'Method is case-insensitive. On failure returns `{ok:false, error, message}`', + 'where `error` is one of: invalid_payload, invalid_url, blocked_host, timeout,', + 'too_many_redirects, transport_error. Branch on `error`, not on text.', +].join(' '); + +export const fetchFunctionOptions: RegisterFunctionOptions = { + description: TOOL_DESCRIPTION, + request_format: zodToJsonSchema(FetchPayloadSchema, { name: 'FetchPayload' }) as Record< + string, + unknown + >, +}; diff --git a/harness/src/web/skills/index.md b/harness/src/web/skills/index.md new file mode 100644 index 000000000..f15fe9884 --- /dev/null +++ b/harness/src/web/skills/index.md @@ -0,0 +1,150 @@ +--- +type: index +title: web +description: Outbound HTTP(S) client on the iii bus — the single web::fetch trigger. Use instead of shell::exec curl. Authoring guide for an agent calling it: the minimal call, the full request/response envelope, the ok:true-vs-ok:false rule (HTTP 4xx/5xx are ok:true), an error→cause→fix table, the json-vs-body and response_format rules, and the SSRF guard (blocked ranges, pin-to-IP, per-hop redirect re-check, cross-origin auth stripping). Self-contained; meant for system-prompt injection — do not re-fetch. +functions: + - web::fetch +--- + +> **Callable id:** `web::fetch` — pass to `agent_trigger { function: "web::fetch" }` (NOT the `iii://...` skill path; that's docs, not a function id). Use this **instead of** `shell::exec` with `curl`/`wget` for any HTTP request: you get a parsed `{ ok, status, headers, body }` envelope, enforced size/timeout caps, and server-side SSRF protection a shell `curl` doesn't have. + +# Decide fast + +| You want to… | Call | +|---|---| +| GET a page/API | `{ "url": "https://…" }` | +| Parse a JSON API response | `{ "url": "https://…", "response_format": "json" }` | +| POST/PUT JSON | `{ "url": "…", "method": "post", "json": { … } }` | +| Send a raw body / form | `{ "url": "…", "method": "post", "body": "…", "headers": { "content-type": "…" } }` | +| Download binary | `{ "url": "…", "response_format": "base64" }` | +| Run a shell `curl` | ❌ stop — use `web::fetch` | + +# Minimal call + +```jsonc +// agent_trigger { function: "web::fetch", payload: { "url": "https://api.github.com/zen" } } +// → { "ok": true, "status": 200, "body": "Design for failure.", ... } +``` + +That's the whole T0 path: a URL is the only required field. Everything else has a default. + +# Read the result correctly (the #1 trap) + +**`ok` tells you if the request COMPLETED, not whether the server was happy.** + +- A 404 or 500 is a **successful fetch** → `ok: true`, `status: 404`. Branch on `status` for HTTP outcomes. +- `ok: false` means the request never produced a response (bad URL, blocked host, timeout, transport failure). Branch on `error` for those. + +```jsonc +if (!r.ok) → fetch failed; look at r.error (see table below) +else if (r.status>=400) → server returned an HTTP error; look at r.status / r.body +else → success; use r.body or r.json +``` + +Do **not** treat `ok: true` as "2xx". Always check `status` too. + +# Request fields + +| Field | Default | Notes | +|---|---|---| +| `url` *(required)* | — | absolute `http(s)://` | +| `method` | `GET` | case-insensitive (`"post"` works); GET HEAD POST PUT PATCH DELETE OPTIONS | +| `headers` | `{}` | object; keys as you write them | +| `json` | — | structured payload; auto-stringified + sets `content-type: application/json`. **Wins over `body`.** | +| `body` | — | raw string body; use for non-JSON. Ignored on GET/HEAD. | +| `response_format` | `"text"` | `"text"` \| `"base64"` (binary) \| `"json"` (also parses into `json`) | +| `timeout_ms` | worker max (30000) | clamped DOWN to the ceiling; can't raise it | +| `max_bytes` | worker max (5 MiB) | over-cap body is truncated, not errored | +| `follow_redirects` | `true` | each hop re-checked against the SSRF blocklist | + +# Response + +**Success** (`ok: true`) — returned for *any* completed response, 2xx through 5xx: + +```jsonc +{ + "ok": true, + "status": 200, + "status_text": "OK", + "headers": { "content-type": "application/json", ... }, // keys LOWER-CASED; set-cookie joined with ", " + "body": "", + "json": { ... }, // only when response_format="json" AND parse succeeded AND not truncated + "parse_error": "…", // only when response_format="json" AND JSON.parse failed (body still set) + "response_format": "json", + "bytes_truncated": false, // true when body hit max_bytes (NOT an error) + "redirect_chain": ["https://…/a"] // omitted when no redirects +} +``` + +**Failure** (`ok: false`) — the fetch never produced a response: + +```jsonc +{ "ok": false, "error": "blocked_host", "message": "…" } // no status field +``` + +## error → cause → fix + +| `error` | Cause | Fix | +|---|---|---| +| `invalid_payload` | Payload failed schema (bad `method`, wrong types). `message` lists the bad fields. | Correct the named fields. | +| `invalid_url` | `url` isn't a parseable absolute `http(s)://` URL. | Pass a full absolute URL incl. scheme. | +| `blocked_host` | Target resolves to a private / link-local / cloud-metadata IP (SSRF guard). | Don't target internal/metadata hosts. For loopback in dev, the operator sets `web.allow_loopback`. | +| `timeout` | Slower than `timeout_ms` (or the 30 s ceiling). | Raise `timeout_ms` (up to ceiling) or shrink work via `max_bytes`. | +| `too_many_redirects` | More than `max_redirects` (5) hops. | Use the final URL directly, or set `follow_redirects: false` and read the `location` header. | +| `transport_error` | Connection refused/reset, TLS failure, DNS failure, or a redirect `Location` that won't parse. | Check host/port/cert; retry if transient. | + +There is **no `too_large` error** — oversize responses come back `ok: true` with `bytes_truncated: true`. Branch on the flag, not on an error. + +# Rules that save a turn + +- **`json` vs `body`: set exactly one.** `json` wins if both are present and forces `content-type: application/json`. Use `body` + your own `content-type` for form/text/XML. +- **GET/HEAD ignore the body.** Putting `json`/`body` on a GET sends nothing — switch `method`. +- **`response_format: "json"` can succeed with `parse_error`.** Non-JSON (or truncated) bodies return `ok: true`, no `json`, `parse_error` set; the raw text is still in `body`. Check for `json` before reading it. +- **Caps only go down.** `timeout_ms`/`max_bytes` above the worker ceiling are clamped; you can't request more than the operator allows. +- **Response header keys are lower-cased.** Index `headers["content-type"]`, never `"Content-Type"`. +- **Auth headers don't survive a cross-origin redirect** (see below) — for an authed endpoint, hit the final URL directly. + +# SSRF guard (server-side, not optional) + +DNS is resolved **once**, every resolved address is checked, then the request is dialed at the **validated IP** (pinned `lookup` + TLS `servername`) — so the IP that passed the check is the IP connected to (no DNS-rebind window). + +Blocked **unconditionally** → `error: "blocked_host"`: + +- Private RFC1918 (`10/8`, `172.16/12`, `192.168/16`) +- Link-local incl. **cloud metadata** `169.254.169.254` +- IPv6 ULA (`fc00::/7`), link-local (`fe80::/10`), multicast +- **`::ffff:`-mapped IPv4** in both dotted (`::ffff:169.254.169.254`) and hex (`::ffff:a9fe:a9fe`) forms + +Loopback (`127.0.0.0/8`, `::1`, `localhost`) is **allowed by default** (dev convenience); operators flip `web.allow_loopback: false` for prod, after which loopback returns `blocked_host`. + +On each 3xx the `Location` is re-resolved and **re-validated** before following, and `Authorization` / `Cookie` / `Proxy-Authorization` are **stripped** when the redirect changes host or downgrades `https → http`. + +# Examples + +```jsonc +// Parse a JSON API +{ "url": "https://api.example.com/status", "response_format": "json" } +// → { ok:true, status:200, json:{ healthy:true } } + +// POST JSON (content-type set for you) +{ "url": "https://api.example.com/things", "method": "post", + "json": { "name": "x" }, "response_format": "json" } + +// Authenticated GET (auth stays only because there's no cross-host redirect) +{ "url": "https://api.example.com/me", + "headers": { "authorization": "Bearer TOKEN" }, "response_format": "json" } + +// Bounded download of a maybe-large file +{ "url": "https://example.com/big.log", "max_bytes": 65536 } +// → { ok:true, body:"", bytes_truncated:true } +``` + +# Authoritative schema + +This page covers behavior the schema can't. For exact field types, call +`engine::functions::info { function_id: "web::fetch" }` — the live schema wins if it ever disagrees with this page. + +# Related + +- [`shell/index`](iii://shell) — local filesystem + process ops; `web::fetch` is the network counterpart (use it, not `shell::exec curl`). +- [`sandbox/index`](iii://sandbox) — code *inside* a sandbox reaches the host engine via the boot-time `III_ENGINE_URL` rewrite, not `web::fetch`. diff --git a/harness/src/web/ssrf.ts b/harness/src/web/ssrf.ts new file mode 100644 index 000000000..91d3bbbbc --- /dev/null +++ b/harness/src/web/ssrf.ts @@ -0,0 +1,270 @@ +/** + * SSRF defense for `web::fetch`. Parses the target URL, resolves the + * host to its concrete addresses, and rejects any address that lands + * in a private, loopback, link-local, multicast, or reserved range. + * + * The resolve is done ONCE here; the actual `fetch()` is then called + * with the resolved IP literal so undici cannot re-resolve and land + * on a different (private) address — defending against DNS rebinding. + * TLS / vhost routing is preserved by overriding the `Host` header to + * the original hostname. + */ + +import { isIPv4, isIPv6 } from 'node:net'; +import { lookup as dnsLookup } from 'node:dns/promises'; + +export type SsrfPolicy = { + /** When true, 127.0.0.0/8 and ::1 are NOT blocked. Other private + * ranges (RFC1918, link-local, ULA, multicast, AWS metadata) remain + * blocked regardless. */ + allow_loopback: boolean; +}; + +export const DEFAULT_POLICY: SsrfPolicy = { allow_loopback: true }; + +export type SsrfCheck = + | { ok: true; address: string; family: 4 | 6; hostname: string; port: number } + | { ok: false; error: 'invalid_url' | 'blocked_host'; message: string }; + +export type ParsedTarget = { + url: URL; + hostname: string; + port: number; +}; + +const DEFAULT_PORTS: Record = { 'http:': 80, 'https:': 443 }; + +/** Parse + validate the URL scheme and host shape. No DNS yet. */ +export function parseTarget(raw: string): ParsedTarget | { error: string } { + let url: URL; + try { + url = new URL(raw); + } catch { + return { error: 'url is not a valid absolute URL' }; + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return { error: `scheme not allowed: ${url.protocol} (only http: and https: are permitted)` }; + } + if (url.hostname.length === 0) { + return { error: 'url has no hostname' }; + } + const portRaw = url.port.length > 0 ? Number(url.port) : DEFAULT_PORTS[url.protocol]; + if (typeof portRaw !== 'number' || !Number.isInteger(portRaw) || portRaw < 1 || portRaw > 65535) { + return { error: `invalid port: ${url.port}` }; + } + return { url, hostname: url.hostname, port: portRaw }; +} + +/** + * IPv4 CIDR range as a packed 32-bit integer base + mask bit count. + * Order matters for human inspection only; matching is O(N) over the + * full list and N is small. + */ +type Cidr4 = { base: number; bits: number; label: string }; + +const BLOCKED_V4: readonly Cidr4[] = [ + { base: ipv4ToInt('0.0.0.0'), bits: 8, label: 'this-network' }, + { base: ipv4ToInt('10.0.0.0'), bits: 8, label: 'private rfc1918' }, + { base: ipv4ToInt('100.64.0.0'), bits: 10, label: 'cgnat rfc6598' }, + { base: ipv4ToInt('127.0.0.0'), bits: 8, label: 'loopback' }, + { base: ipv4ToInt('169.254.0.0'), bits: 16, label: 'link-local (incl. AWS metadata)' }, + { base: ipv4ToInt('172.16.0.0'), bits: 12, label: 'private rfc1918' }, + { base: ipv4ToInt('192.0.0.0'), bits: 24, label: 'ietf protocol assignments' }, + { base: ipv4ToInt('192.0.2.0'), bits: 24, label: 'documentation' }, + { base: ipv4ToInt('192.168.0.0'), bits: 16, label: 'private rfc1918' }, + { base: ipv4ToInt('198.18.0.0'), bits: 15, label: 'benchmarking' }, + { base: ipv4ToInt('198.51.100.0'), bits: 24, label: 'documentation' }, + { base: ipv4ToInt('203.0.113.0'), bits: 24, label: 'documentation' }, + { base: ipv4ToInt('224.0.0.0'), bits: 4, label: 'multicast' }, + { base: ipv4ToInt('240.0.0.0'), bits: 4, label: 'reserved' }, +]; + +function ipv4ToInt(addr: string): number { + const parts = addr.split('.').map(Number); + if (parts.length !== 4 || parts.some((p) => !Number.isInteger(p) || p < 0 || p > 255)) { + throw new Error(`bad ipv4 literal in blocklist: ${addr}`); + } + const a = parts[0] ?? 0; + const b = parts[1] ?? 0; + const c = parts[2] ?? 0; + const d = parts[3] ?? 0; + return ((a << 24) | (b << 16) | (c << 8) | d) >>> 0; +} + +function isBlockedIpv4( + addr: string, + policy: SsrfPolicy = DEFAULT_POLICY, +): { blocked: boolean; label?: string } { + let n: number; + try { + n = ipv4ToInt(addr); + } catch { + return { blocked: true, label: 'unparseable ipv4' }; + } + for (const { base, bits, label } of BLOCKED_V4) { + if (policy.allow_loopback && label === 'loopback') continue; + const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0; + if ((n & mask) === (base & mask)) return { blocked: true, label }; + } + return { blocked: false }; +} + +/** + * IPv6 block via prefix check on the canonical form returned by + * `dns.lookup` — handles loopback `::1`, unspecified `::`, link-local + * `fe80::/10`, ULA `fc00::/7`, multicast `ff00::/8`, and IPv4-mapped + * `::ffff:X.X.X.X` (delegates the inner v4 to the v4 check). + */ +function isBlockedIpv6( + addr: string, + policy: SsrfPolicy = DEFAULT_POLICY, +): { blocked: boolean; label?: string } { + const lower = addr.toLowerCase(); + if (lower === '::1') { + if (policy.allow_loopback) return { blocked: false }; + return { blocked: true, label: 'loopback' }; + } + if (lower === '::') return { blocked: true, label: 'unspecified' }; + if ( + lower.startsWith('fe8') || + lower.startsWith('fe9') || + lower.startsWith('fea') || + lower.startsWith('feb') + ) { + return { blocked: true, label: 'link-local fe80::/10' }; + } + if (/^f[cd][0-9a-f]{0,2}:/.test(lower)) { + return { blocked: true, label: 'unique-local fc00::/7' }; + } + if (lower.startsWith('ff')) { + return { blocked: true, label: 'multicast ff00::/8' }; + } + // IPv4-mapped (::ffff:0:0/96) embeds a v4 address the OS routes to v4. + // dns.lookup emits the dotted form (::ffff:1.2.3.4), but a literal URL + // can carry the SAME address in hex (::ffff:a9fe:a9fe) — decode BOTH so + // the v4 blocklist still applies. Decoding only the dotted form left a + // hole: `http://[::ffff:a9fe:a9fe]/` reached 169.254.169.254 (metadata). + const mappedV4 = decodeV4MappedIpv6(lower); + if (mappedV4) { + return isBlockedIpv4(mappedV4, policy); + } + return { blocked: false }; +} + +/** + * Extract the embedded IPv4 of an IPv4-mapped IPv6 address (`::ffff:/96`), + * in either textual form, or `null` when the address is not v4-mapped. + * Restricted to the `::ffff:` prefix on purpose — that is the range OSes + * actually route to v4; decoding bare `::g:g` would false-block legitimate + * low global-unicast addresses. + */ +function decodeV4MappedIpv6(lower: string): string | null { + const dotted = lower.match(/^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/); + if (dotted?.[1]) return dotted[1]; + const hex = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/); + if (hex?.[1] && hex?.[2]) { + const hi = Number.parseInt(hex[1], 16); + const lo = Number.parseInt(hex[2], 16); + return `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`; + } + return null; +} + +/** + * Resolve the URL's host, validate every resolved address against the + * blocklist, and return the address to dial. If any returned address + * is blocked the whole call is refused — defends against a DNS record + * that returns both a public and a private answer. + * + * If the hostname is already a literal IP, skip DNS and check it + * directly. This is the common case for `http://1.2.3.4/`. + */ +export async function checkTarget( + target: ParsedTarget, + policy: SsrfPolicy = DEFAULT_POLICY, +): Promise { + const { hostname, port, url } = target; + + const loopbackHint = !policy.allow_loopback + ? ' (set web.allow_loopback=true in config.yaml if loopback is intentional in this environment)' + : ''; + if (isIPv4(hostname)) { + const v = isBlockedIpv4(hostname, policy); + if (v.blocked) { + const hint = v.label === 'loopback' ? loopbackHint : ''; + return { + ok: false, + error: 'blocked_host', + message: `address ${hostname} is in ${v.label}${hint}`, + }; + } + return { ok: true, address: hostname, family: 4, hostname, port }; + } + if (isIPv6(hostname.replace(/^\[|\]$/g, ''))) { + const stripped = hostname.replace(/^\[|\]$/g, ''); + const v = isBlockedIpv6(stripped, policy); + if (v.blocked) { + const hint = v.label === 'loopback' ? loopbackHint : ''; + return { + ok: false, + error: 'blocked_host', + message: `address ${stripped} is in ${v.label}${hint}`, + }; + } + return { ok: true, address: stripped, family: 6, hostname, port }; + } + + let resolved: { address: string; family: number }[]; + try { + resolved = await dnsLookup(hostname, { all: true }); + } catch (err) { + return { + ok: false, + error: 'blocked_host', + message: `dns lookup failed for ${hostname}: ${String(err)}`, + }; + } + if (resolved.length === 0) { + return { + ok: false, + error: 'blocked_host', + message: `dns returned no addresses for ${hostname}`, + }; + } + + for (const r of resolved) { + const check = + r.family === 4 ? isBlockedIpv4(r.address, policy) : isBlockedIpv6(r.address, policy); + if (check.blocked) { + const hint = + check.label === 'loopback' && !policy.allow_loopback + ? ' (set web.allow_loopback=true in config.yaml if loopback is intentional in this environment)' + : ''; + return { + ok: false, + error: 'blocked_host', + message: `${hostname} resolves to ${r.address} (${check.label}); refusing to dial${hint}`, + }; + } + } + + const first = resolved[0]; + if (!first) { + return { + ok: false, + error: 'blocked_host', + message: `dns returned no addresses for ${hostname}`, + }; + } + void url; + return { + ok: true, + address: first.address, + family: first.family === 4 ? 4 : 6, + hostname, + port, + }; +} + +/** Exposed for tests. */ +export const __testing = { isBlockedIpv4, isBlockedIpv6, ipv4ToInt }; diff --git a/harness/tests/web/fetch.integration.test.ts b/harness/tests/web/fetch.integration.test.ts new file mode 100644 index 000000000..f225e5928 --- /dev/null +++ b/harness/tests/web/fetch.integration.test.ts @@ -0,0 +1,152 @@ +/** + * End-to-end transport tests for executeFetch against a real loopback + * `http.createServer`. Unlike fetch.test.ts (which only exercises the guard + * surface + helpers), these drive the node:http request path the worker now + * uses — proving IP pinning, per-hop SSRF re-validation, the byte cap, the + * timeout, and POST body handling actually work over a socket. + * + * Plaintext loopback only: the HTTPS servername/cert-identity path can't be + * faithfully unit-tested without generating a trusted cert (own mini-project), + * so it stays correct-by-construction (options.servername = hostname) plus the + * live smoke step. allow_loopback defaults true, so 127.0.0.1 targets pass the + * SSRF guard and dial the test server. + */ + +import type { AddressInfo } from 'node:net'; +import { type Server, createServer } from 'node:http'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { loadWebConfig } from '../../src/web/config.js'; +import { executeFetch } from '../../src/web/fetch.js'; + +let server: Server; +let base: string; +let lastReceivedHost: string | undefined; + +beforeAll(async () => { + server = createServer((req, res) => { + lastReceivedHost = req.headers.host; + const url = req.url ?? '/'; + if (url === '/ok') { + res.writeHead(200, { 'content-type': 'text/plain', 'x-custom': 'yes' }); + res.end('hello'); + return; + } + if (url === '/big') { + res.writeHead(200); + res.end(Buffer.alloc(10_000, 0x41)); + return; + } + if (url === '/redirect-to-blocked') { + res.writeHead(302, { location: 'http://169.254.169.254/latest/meta-data/' }); + res.end(); + return; + } + if (url === '/redirect-relative') { + res.writeHead(302, { location: '/ok' }); + res.end(); + return; + } + if (url === '/slow') { + setTimeout(() => { + res.writeHead(200); + res.end('late'); + }, 300); + return; + } + if (url === '/echo' && req.method === 'POST') { + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(Buffer.from(c))); + req.on('end', () => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end( + JSON.stringify({ + received_content_type: req.headers['content-type'] ?? null, + body: Buffer.concat(chunks).toString('utf8'), + }), + ); + }); + return; + } + res.writeHead(404); + res.end('nope'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as AddressInfo).port; + base = `http://127.0.0.1:${port}`; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); +}); + +const cfg = () => loadWebConfig({}); + +describe('executeFetch transport (loopback http server)', () => { + it('GETs a body, status, and headers over a real socket', async () => { + const r = await executeFetch({ url: `${base}/ok` }, cfg()); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.status).toBe(200); + expect(r.body).toBe('hello'); + expect(r.headers['x-custom']).toBe('yes'); + expect(r.bytes_truncated).toBe(false); + } + // Host header is derived from the URL authority (pinning doesn't clobber it). + expect(lastReceivedHost).toBe(base.replace('http://', '')); + }); + + it('truncates a response that exceeds max_bytes', async () => { + const r = await executeFetch({ url: `${base}/big`, max_bytes: 100 }, cfg()); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.bytes_truncated).toBe(true); + expect(r.body.length).toBe(100); + } + }); + + it('re-runs the SSRF check on each redirect hop (302 → metadata is blocked)', async () => { + const r = await executeFetch({ url: `${base}/redirect-to-blocked` }, cfg()); + expect(r.ok).toBe(false); + if (!r.ok) { + expect(r.error).toBe('blocked_host'); + expect(r.message).toMatch(/169\.254\.169\.254/); + } + }); + + it('follows a same-host relative redirect and records the chain', async () => { + const r = await executeFetch({ url: `${base}/redirect-relative` }, cfg()); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.body).toBe('hello'); + expect(r.redirect_chain).toEqual([`${base}/redirect-relative`]); + } + }); + + it('returns error:timeout when the server is slower than timeout_ms', async () => { + const r = await executeFetch({ url: `${base}/slow`, timeout_ms: 50 }, cfg()); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('timeout'); + }); + + it('sends a JSON body with content-type and parses the response', async () => { + const r = await executeFetch( + { url: `${base}/echo`, method: 'POST', json: { a: 1 }, response_format: 'json' }, + cfg(), + ); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.json).toMatchObject({ + received_content_type: 'application/json', + body: '{"a":1}', + }); + } + }); + + it('returns transport_error when the connection is refused', async () => { + // Port 1 on loopback is closed; SSRF guard passes (loopback allowed), + // the socket connect fails → transport_error (not a thrown exception). + const r = await executeFetch({ url: 'http://127.0.0.1:1/' }, cfg()); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('transport_error'); + }); +}); diff --git a/harness/tests/web/fetch.test.ts b/harness/tests/web/fetch.test.ts new file mode 100644 index 000000000..6c7b29901 --- /dev/null +++ b/harness/tests/web/fetch.test.ts @@ -0,0 +1,216 @@ +import { Buffer } from 'node:buffer'; +import { Readable } from 'node:stream'; +import { describe, expect, it } from 'vitest'; +import { loadWebConfig } from '../../src/web/config.js'; +import { executeFetch, readIncomingCapped, stripCrossOriginAuth } from '../../src/web/fetch.js'; +import { FetchPayloadSchema } from '../../src/web/schemas.js'; + +// The SSRF guard is exhaustively tested in ssrf.test.ts. These tests +// focus on: +// 1. The error surface of executeFetch (invalid url, blocked host). +// 2. The body-cap helper that bounds response size. +// 3. Config loading defaults / overrides. +// Live HTTP integration (timeouts, redirect chains hitting public +// hosts) is left to the manual smoke step in the plan's verification +// section — going out over the real network from unit tests is flaky +// and the security-critical paths are already covered. + +describe('executeFetch payload + guard surface', () => { + it('returns invalid_url for a non-http scheme', async () => { + const cfg = loadWebConfig({}); + const r = await executeFetch({ url: 'file:///etc/passwd' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('invalid_url'); + }); + + it('returns invalid_url for garbage input', async () => { + const cfg = loadWebConfig({}); + const r = await executeFetch({ url: 'not a url' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('invalid_url'); + }); + + it('returns blocked_host for AWS metadata IP literal', async () => { + const cfg = loadWebConfig({}); + const r = await executeFetch({ url: 'http://169.254.169.254/latest/' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('blocked_host'); + }); + + it('allows loopback when allow_loopback is true (default for harness UX)', async () => { + const cfg = loadWebConfig({}); + // 127.0.0.1:1 is a closed port — we expect the SSRF guard to PASS + // and the request to fail at the transport layer (connection refused). + const r = await executeFetch({ url: 'http://127.0.0.1:1/' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('transport_error'); + }); + + it('blocks loopback when allow_loopback is explicitly disabled', async () => { + const cfg = loadWebConfig({ web: { allow_loopback: false } }); + const r = await executeFetch({ url: 'http://127.0.0.1:8080/secret' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('blocked_host'); + }); + + it('returns blocked_host for a private RFC1918 address', async () => { + const cfg = loadWebConfig({}); + const r = await executeFetch({ url: 'http://192.168.1.1/' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('blocked_host'); + }); + + it('strict-mode blocked_host on loopback gives a config hint', async () => { + const cfg = loadWebConfig({ web: { allow_loopback: false } }); + const r = await executeFetch({ url: 'http://127.0.0.1:8080/' }, cfg); + expect(r.ok).toBe(false); + if (!r.ok) { + expect(r.error).toBe('blocked_host'); + expect(r.message).toMatch(/web\.allow_loopback=true/); + } + }); +}); + +describe('schema preprocessing — case-insensitive method + json field', () => { + it('normalises lowercase methods to upper', () => { + const r = FetchPayloadSchema.safeParse({ url: 'http://x/', method: 'post' }); + expect(r.success).toBe(true); + if (r.success) expect(r.data.method).toBe('POST'); + }); + + it('normalises mixed-case methods', () => { + const r = FetchPayloadSchema.safeParse({ url: 'http://x/', method: 'Get' }); + expect(r.success).toBe(true); + if (r.success) expect(r.data.method).toBe('GET'); + }); + + it('rejects nonsense methods', () => { + const r = FetchPayloadSchema.safeParse({ url: 'http://x/', method: 'BREW' }); + expect(r.success).toBe(false); + }); + + it('accepts an unknown json payload of any shape', () => { + const r1 = FetchPayloadSchema.safeParse({ url: 'http://x/', json: { a: 1, b: [2, 3] } }); + const r2 = FetchPayloadSchema.safeParse({ url: 'http://x/', json: 'a string is valid json' }); + const r3 = FetchPayloadSchema.safeParse({ url: 'http://x/', json: null }); + expect(r1.success && r2.success && r3.success).toBe(true); + }); + + it('accepts response_format: "json"', () => { + const r = FetchPayloadSchema.safeParse({ url: 'http://x/', response_format: 'json' }); + expect(r.success).toBe(true); + }); +}); + +describe('stripCrossOriginAuth', () => { + const AUTH = { authorization: 'Bearer secret', cookie: 'sid=abc', accept: 'text/html' }; + + it('keeps auth on a same-origin redirect', () => { + const out = stripCrossOriginAuth( + AUTH, + new URL('https://api.example.com/a'), + new URL('https://api.example.com/b'), + ); + expect(out.authorization).toBe('Bearer secret'); + expect(out.cookie).toBe('sid=abc'); + }); + + it('strips auth on a cross-host redirect', () => { + const out = stripCrossOriginAuth( + AUTH, + new URL('https://api.example.com/a'), + new URL('https://evil.example.net/b'), + ); + expect(out.authorization).toBeUndefined(); + expect(out.cookie).toBeUndefined(); + expect(out.accept).toBe('text/html'); // non-sensitive header survives + }); + + it('strips auth on a same-host https→http downgrade (no cleartext leak)', () => { + const out = stripCrossOriginAuth( + AUTH, + new URL('https://api.example.com/a'), + new URL('http://api.example.com/b'), + ); + expect(out.authorization).toBeUndefined(); + expect(out.cookie).toBeUndefined(); + }); + + it('keeps auth on a same-host http→https upgrade', () => { + const out = stripCrossOriginAuth( + AUTH, + new URL('http://api.example.com/a'), + new URL('https://api.example.com/b'), + ); + expect(out.authorization).toBe('Bearer secret'); + }); +}); + +describe('readIncomingCapped', () => { + // Multi-chunk stream so the cap is exercised across chunk boundaries, the + // way a real IncomingMessage delivers a body. + function makeStream(bytes: Buffer, chunkSize = 4096): Readable { + const chunks: Buffer[] = []; + for (let i = 0; i < bytes.length; i += chunkSize) { + chunks.push(bytes.subarray(i, i + chunkSize)); + } + return Readable.from(chunks.length > 0 ? chunks : []); + } + + it('reads full body when under cap', async () => { + const r = await readIncomingCapped(makeStream(Buffer.from('hello world')), 1024); + expect(r.truncated).toBe(false); + expect(r.bytes.toString('utf8')).toBe('hello world'); + }); + + it('truncates body at exactly the cap (across chunks)', async () => { + const r = await readIncomingCapped(makeStream(Buffer.alloc(10_000, 0x41)), 100); + expect(r.truncated).toBe(true); + expect(r.bytes.length).toBe(100); + }); + + it('handles empty body', async () => { + const r = await readIncomingCapped(makeStream(Buffer.alloc(0)), 1024); + expect(r.truncated).toBe(false); + expect(r.bytes.length).toBe(0); + }); + + it('honours a tiny cap', async () => { + const r = await readIncomingCapped(makeStream(Buffer.from('aaaaaaaaaa')), 1); + expect(r.truncated).toBe(true); + expect(r.bytes.length).toBe(1); + }); +}); + +describe('loadWebConfig', () => { + it('produces sane defaults from empty config', () => { + const cfg = loadWebConfig({}); + expect(cfg.max_timeout_ms).toBe(30_000); + expect(cfg.max_response_bytes).toBe(5 * 1024 * 1024); + expect(cfg.max_redirects).toBe(5); + expect(cfg.user_agent).toMatch(/iii-harness/); + expect(cfg.allow_loopback).toBe(true); + }); + + it('honours overrides under web: section', () => { + const cfg = loadWebConfig({ + web: { + max_timeout_ms: 5000, + max_response_bytes: 1024, + max_redirects: 1, + user_agent: 'custom', + allow_loopback: false, + }, + }); + expect(cfg.max_timeout_ms).toBe(5000); + expect(cfg.max_response_bytes).toBe(1024); + expect(cfg.max_redirects).toBe(1); + expect(cfg.user_agent).toBe('custom'); + expect(cfg.allow_loopback).toBe(false); + }); + + it('ignores non-numeric override values', () => { + const cfg = loadWebConfig({ web: { max_timeout_ms: 'fast' } }); + expect(cfg.max_timeout_ms).toBe(30_000); + }); +}); diff --git a/harness/tests/web/handler.test.ts b/harness/tests/web/handler.test.ts new file mode 100644 index 000000000..7ce5a4187 --- /dev/null +++ b/harness/tests/web/handler.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { ISdk } from '../../src/runtime/iii.js'; +import type { WebConfig } from '../../src/web/config.js'; +import { register } from '../../src/web/handlers/fetch.js'; + +function fakeIii(): { iii: ISdk; registered: Map Promise> } { + const registered = new Map Promise>(); + const iii = { + registerFunction: (id: string, handler: (payload: unknown) => Promise) => { + registered.set(id, handler); + }, + } as unknown as ISdk; + return { iii, registered }; +} + +const cfg: WebConfig = { + max_timeout_ms: 30_000, + max_response_bytes: 5 * 1024 * 1024, + max_redirects: 5, + user_agent: 'test', + allow_loopback: true, +}; + +describe('web::fetch handler', () => { + it('registers under the right function id and exposes a request_format', () => { + const { iii, registered } = fakeIii(); + const spy = vi.spyOn(iii, 'registerFunction'); + register(iii, cfg); + expect(registered.has('web::fetch')).toBe(true); + expect(spy).toHaveBeenCalledWith( + 'web::fetch', + expect.any(Function), + expect.objectContaining({ + description: expect.stringMatching(/INSTEAD of `shell::exec` with curl/), + request_format: expect.any(Object), + }), + ); + }); + + it('returns invalid_payload envelope (not throw) for missing url', async () => { + const { iii, registered } = fakeIii(); + register(iii, cfg); + const handler = registered.get('web::fetch'); + if (!handler) throw new Error('handler missing'); + const r = (await handler({})) as { ok: boolean; error?: string }; + expect(r.ok).toBe(false); + expect(r.error).toBe('invalid_payload'); + }); + + it('returns invalid_payload for a wrong-type body', async () => { + const { iii, registered } = fakeIii(); + register(iii, cfg); + const handler = registered.get('web::fetch'); + if (!handler) throw new Error('handler missing'); + const r = (await handler({ url: 'https://example.com/', body: 12345 })) as { + ok: boolean; + error?: string; + }; + expect(r.ok).toBe(false); + expect(r.error).toBe('invalid_payload'); + }); + + it('returns invalid_url envelope for a bad scheme (validation passes, guard catches it)', async () => { + const { iii, registered } = fakeIii(); + register(iii, cfg); + const handler = registered.get('web::fetch'); + if (!handler) throw new Error('handler missing'); + const r = (await handler({ url: 'file:///etc/passwd' })) as { ok: boolean; error?: string }; + expect(r.ok).toBe(false); + expect(r.error).toBe('invalid_url'); + }); +}); diff --git a/harness/tests/web/ssrf.test.ts b/harness/tests/web/ssrf.test.ts new file mode 100644 index 000000000..0fb60baeb --- /dev/null +++ b/harness/tests/web/ssrf.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from 'vitest'; +import { __testing, checkTarget, parseTarget } from '../../src/web/ssrf.js'; + +const { isBlockedIpv4, isBlockedIpv6 } = __testing; + +const STRICT = { allow_loopback: false }; +const LAX = { allow_loopback: true }; + +describe('parseTarget', () => { + it('accepts http and https', () => { + expect(parseTarget('http://example.com/')).toMatchObject({ hostname: 'example.com', port: 80 }); + expect(parseTarget('https://example.com/')).toMatchObject({ port: 443 }); + }); + + it('honours explicit ports', () => { + const r = parseTarget('http://example.com:8080/x'); + expect(r).toMatchObject({ port: 8080 }); + }); + + it('rejects non-http schemes', () => { + expect(parseTarget('file:///etc/passwd')).toEqual({ + error: expect.stringMatching(/scheme not allowed/), + }); + expect(parseTarget('ftp://example.com/')).toEqual({ + error: expect.stringMatching(/scheme not allowed/), + }); + expect(parseTarget('javascript:alert(1)')).toEqual({ + error: expect.stringMatching(/scheme not allowed/), + }); + expect(parseTarget('data:text/plain,hi')).toEqual({ + error: expect.stringMatching(/scheme not allowed/), + }); + }); + + it('rejects garbage urls', () => { + expect(parseTarget('not a url')).toHaveProperty('error'); + expect(parseTarget('http://')).toHaveProperty('error'); + }); +}); + +describe('isBlockedIpv4', () => { + it('blocks loopback variants under strict policy', () => { + expect(isBlockedIpv4('127.0.0.1', STRICT).blocked).toBe(true); + expect(isBlockedIpv4('127.255.255.254', STRICT).blocked).toBe(true); + }); + + it('allows loopback under default (lax) policy — harness UX', () => { + expect(isBlockedIpv4('127.0.0.1', LAX).blocked).toBe(false); + expect(isBlockedIpv4('127.0.0.1').blocked).toBe(false); // default arg + }); + + it('blocks AWS / cloud metadata', () => { + const v = isBlockedIpv4('169.254.169.254'); + expect(v.blocked).toBe(true); + expect(v.label).toMatch(/link-local/); + }); + + it('blocks all RFC1918 ranges', () => { + expect(isBlockedIpv4('10.0.0.1').blocked).toBe(true); + expect(isBlockedIpv4('10.255.255.255').blocked).toBe(true); + expect(isBlockedIpv4('172.16.0.1').blocked).toBe(true); + expect(isBlockedIpv4('172.31.255.255').blocked).toBe(true); + expect(isBlockedIpv4('192.168.0.1').blocked).toBe(true); + expect(isBlockedIpv4('192.168.255.255').blocked).toBe(true); + }); + + it('does NOT block public addresses', () => { + expect(isBlockedIpv4('8.8.8.8').blocked).toBe(false); + expect(isBlockedIpv4('1.1.1.1').blocked).toBe(false); + expect(isBlockedIpv4('140.82.114.4').blocked).toBe(false); // github.com sample + }); + + it('blocks 0.0.0.0', () => { + expect(isBlockedIpv4('0.0.0.0').blocked).toBe(true); + }); + + it('blocks 172.31 (top of rfc1918 172/12) but allows 172.32 (outside)', () => { + expect(isBlockedIpv4('172.31.0.1').blocked).toBe(true); + expect(isBlockedIpv4('172.32.0.1').blocked).toBe(false); + }); +}); + +describe('isBlockedIpv6', () => { + it('blocks loopback ::1 under strict policy', () => { + expect(isBlockedIpv6('::1', STRICT).blocked).toBe(true); + }); + + it('allows loopback ::1 under default (lax) policy', () => { + expect(isBlockedIpv6('::1', LAX).blocked).toBe(false); + expect(isBlockedIpv6('::1').blocked).toBe(false); + }); + + it('blocks unspecified ::', () => { + expect(isBlockedIpv6('::').blocked).toBe(true); + }); + + it('blocks link-local fe80::/10', () => { + expect(isBlockedIpv6('fe80::1').blocked).toBe(true); + }); + + it('blocks unique-local fc00::/7', () => { + expect(isBlockedIpv6('fc00::1').blocked).toBe(true); + expect(isBlockedIpv6('fd00::1').blocked).toBe(true); + }); + + it('blocks multicast', () => { + expect(isBlockedIpv6('ff02::1').blocked).toBe(true); + }); + + it('blocks IPv4-mapped private ranges (loopback only under strict policy)', () => { + expect(isBlockedIpv6('::ffff:127.0.0.1', STRICT).blocked).toBe(true); + expect(isBlockedIpv6('::ffff:127.0.0.1', LAX).blocked).toBe(false); + // Metadata IP is blocked regardless of allow_loopback. + expect(isBlockedIpv6('::ffff:169.254.169.254', STRICT).blocked).toBe(true); + expect(isBlockedIpv6('::ffff:169.254.169.254', LAX).blocked).toBe(true); + }); + + it('blocks IPv4-mapped private ranges in HEX form (regression: metadata bypass)', () => { + // ::ffff:a9fe:a9fe IS 169.254.169.254 — the dotted-only decode let this + // hex form slip past the guard and reach cloud metadata. + expect(isBlockedIpv6('::ffff:a9fe:a9fe').blocked).toBe(true); + expect(isBlockedIpv6('::ffff:A9FE:A9FE').blocked).toBe(true); // case-insensitive + // 10.0.0.1 = ::ffff:0a00:0001 (rfc1918, blocked under any policy) + expect(isBlockedIpv6('::ffff:0a00:0001', LAX).blocked).toBe(true); + // 127.0.0.1 = ::ffff:7f00:0001 (loopback — lax allows, strict blocks) + expect(isBlockedIpv6('::ffff:7f00:0001', STRICT).blocked).toBe(true); + expect(isBlockedIpv6('::ffff:7f00:0001', LAX).blocked).toBe(false); + // A public address in hex stays allowed (8.8.8.8 = ::ffff:0808:0808). + expect(isBlockedIpv6('::ffff:0808:0808').blocked).toBe(false); + }); + + it('does NOT block public IPv6', () => { + expect(isBlockedIpv6('2606:4700::1111').blocked).toBe(false); // cloudflare + expect(isBlockedIpv6('2001:4860:4860::8888').blocked).toBe(false); // google + }); +}); + +describe('checkTarget (literal IPs, no DNS)', () => { + it('passes a public ipv4 literal', async () => { + const p = parseTarget('http://8.8.8.8/'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p); + expect(r.ok).toBe(true); + }); + + it('blocks AWS metadata IP literal', async () => { + const p = parseTarget('http://169.254.169.254/latest/meta-data/'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p); + expect(r.ok).toBe(false); + if (!r.ok) { + expect(r.error).toBe('blocked_host'); + expect(r.message).toMatch(/169\.254\.169\.254/); + } + }); + + it('allows loopback ipv4 literal by default (harness UX)', async () => { + const p = parseTarget('http://127.0.0.1:8080/admin'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p); + expect(r.ok).toBe(true); + }); + + it('blocks loopback ipv4 literal under strict policy', async () => { + const p = parseTarget('http://127.0.0.1:8080/admin'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p, STRICT); + expect(r.ok).toBe(false); + }); + + it('allows bracketed ipv6 loopback by default', async () => { + const p = parseTarget('http://[::1]:8080/'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p); + expect(r.ok).toBe(true); + }); + + it('blocks bracketed ipv6 loopback under strict policy', async () => { + const p = parseTarget('http://[::1]:8080/'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p, STRICT); + expect(r.ok).toBe(false); + }); + + it('allows DNS that resolves to localhost by default (the bug we fixed)', async () => { + const p = parseTarget('http://localhost/'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p); + expect(r.ok).toBe(true); + }); + + it('blocks DNS that resolves to localhost under strict policy', async () => { + const p = parseTarget('http://localhost/'); + if ('error' in p) throw new Error(p.error); + const r = await checkTarget(p, STRICT); + expect(r.ok).toBe(false); + }); +});