From 1010af0b0c370b70521ca675f11b19f959bfa77f Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:31:02 -0300 Subject: [PATCH 01/22] feat(web): scaffold standalone Rust crate with --manifest --- web/Cargo.toml | 43 ++++++++++ web/build.rs | 6 ++ web/config.yaml.example | 9 +++ web/iii.worker.yaml | 10 +++ web/skills/index.md | 173 ++++++++++++++++++++++++++++++++++++++++ web/src/lib.rs | 8 ++ web/src/main.rs | 38 +++++++++ web/src/manifest.rs | 40 ++++++++++ 8 files changed, 327 insertions(+) create mode 100644 web/Cargo.toml create mode 100644 web/build.rs create mode 100644 web/config.yaml.example create mode 100644 web/iii.worker.yaml create mode 100644 web/skills/index.md create mode 100644 web/src/lib.rs create mode 100644 web/src/main.rs create mode 100644 web/src/manifest.rs diff --git a/web/Cargo.toml b/web/Cargo.toml new file mode 100644 index 0000000000..4321295ecc --- /dev/null +++ b/web/Cargo.toml @@ -0,0 +1,43 @@ +[workspace] + +[package] +name = "web" +version = "1.0.0" +edition = "2021" +publish = false + +[[bin]] +name = "iii-web" +path = "src/main.rs" + +[lib] +name = "web" +path = "src/lib.rs" + +[dependencies] +iii-sdk = "=0.19.2" +arc-swap = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time", "net"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +anyhow = "1" +thiserror = "2" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +clap = { version = "4", features = ["derive"] } +async-trait = "0.1" +schemars = "0.8" +futures = "0.3" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "stream", "http2"] } +htmd = "0.5" +tl = { package = "astral-tl", version = "0.7" } +ipnet = "2" +base64 = "0.22" + +[dev-dependencies] +serde_json = "1" +tempfile = "3" +wiremock = "0.6" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time", "net"] } +futures = "0.3" diff --git a/web/build.rs b/web/build.rs new file mode 100644 index 0000000000..81caa36d69 --- /dev/null +++ b/web/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); +} diff --git a/web/config.yaml.example b/web/config.yaml.example new file mode 100644 index 0000000000..dc6123fdb3 --- /dev/null +++ b/web/config.yaml.example @@ -0,0 +1,9 @@ +web: + default_timeout_ms: 30000 + max_timeout_ms: 120000 + default_response_bytes: 262144 + max_response_bytes: 5242880 + max_transform_bytes: 1048576 + max_redirects: 5 + user_agent: 'iii-harness/0.1 (+web::fetch)' + allow_loopback: true diff --git a/web/iii.worker.yaml b/web/iii.worker.yaml new file mode 100644 index 0000000000..9ec1b40a59 --- /dev/null +++ b/web/iii.worker.yaml @@ -0,0 +1,10 @@ +iii: v1 +name: web +language: rust +deploy: binary +manifest: Cargo.toml +bin: iii-web +description: Outbound HTTP client on the iii bus (web::fetch). + +dependencies: + configuration: "^0.19.0" diff --git a/web/skills/index.md b/web/skills/index.md new file mode 100644 index 0000000000..4d78362418 --- /dev/null +++ b/web/skills/index.md @@ -0,0 +1,173 @@ +--- +type: index +title: web +description: Outbound HTTP(S) client on the iii bus — the single web::fetch trigger. Use instead of shell::exec curl. Authoring guide for an agent calling it: the minimal call, the full request/response envelope, the ok:true-vs-ok:false rule (HTTP 4xx/5xx are ok:true), an error→cause→fix table, the json-vs-body and response_format rules, page-reading mode (format:"markdown" → HTML→Markdown, browser UA, viewable images), and the SSRF guard (blocked ranges, pin-to-IP, per-hop redirect re-check, cross-origin auth stripping). Self-contained — read once via directory::skills::get; do not re-fetch. +functions: + - web::fetch +--- + +> **Callable id:** `web::fetch` — pass to `agent_trigger { function: "web::fetch" }` (NOT the `iii://...` skill path; that's docs, not a function id). Use this **instead of** `shell::exec` with `curl`/`wget` for any HTTP request: you get a parsed `{ ok, status, headers, body }` envelope, enforced size/timeout caps, and server-side SSRF protection a shell `curl` doesn't have. + +# Decide fast + +| You want to… | Call | +|---|---| +| **Read a web page** (docs, articles) | `{ "url": "https://…", "format": "markdown" }` | +| GET a page/API | `{ "url": "https://…" }` | +| Parse a JSON API response | `{ "url": "https://…", "response_format": "json" }` | +| POST/PUT JSON | `{ "url": "…", "method": "post", "json": { … } }` | +| Send a raw body / form | `{ "url": "…", "method": "post", "body": "…", "headers": { "content-type": "…" } }` | +| Download binary | `{ "url": "…", "response_format": "base64" }` | +| Run a shell `curl` | ❌ stop — use `web::fetch` | + +# Minimal call + +```jsonc +// agent_trigger { function: "web::fetch", payload: { "url": "https://api.github.com/zen" } } +// → { "ok": true, "status": 200, "body": "Design for failure.", ... } +``` + +That's the whole T0 path: a URL is the only required field. Everything else has a default. + +# Read the result correctly (the #1 trap) + +**`ok` tells you if the request COMPLETED, not whether the server was happy.** + +- A 404 or 500 is a **successful fetch** → `ok: true`, `status: 404`. Branch on `status` for HTTP outcomes. +- `ok: false` means the request never produced a response (bad URL, blocked host, timeout, transport failure). Branch on `error` for those. + +```jsonc +if (!r.ok) → fetch failed; look at r.error (see table below) +else if (r.status>=400) → server returned an HTTP error; look at r.status / r.body +else → success; use r.body or r.json +``` + +Do **not** treat `ok: true` as "2xx". Always check `status` too. + +One exception: in page-reading mode (`format` set), an `image/*` response returns the image itself plus a one-line text summary instead of this envelope — there is no top-level `ok`/`status` to branch on (see "Images in page-reading mode" below). + +# Request fields + +| Field | Default | Notes | +|---|---|---| +| `url` *(required)* | — | absolute `http(s)://` | +| `method` | `GET` | case-insensitive (`"post"` works); GET HEAD POST PUT PATCH DELETE OPTIONS | +| `headers` | `{}` | object; keys as you write them | +| `json` | — | structured payload; auto-stringified + sets `content-type: application/json`. **Wins over `body`.** | +| `body` | — | raw string body; use for non-JSON. Ignored on GET/HEAD. | +| `response_format` | `"text"` | `"text"` \| `"base64"` (binary) \| `"json"` (also parses into `json`) | +| `format` | — | page-reading mode: `"markdown"` (HTML→Markdown) \| `"text"` (HTML→plain text) \| `"html"` (raw). Sends a browser UA + matching `Accept`; retries once with the honest UA on a Cloudflare challenge; images come back viewable. Forces text transport (`response_format` ignored). | +| `timeout_ms` | 30000 | clamped DOWN to the worker ceiling (120000 by default); can't raise past it | +| `max_bytes` | 5 MiB (256 KiB in `format` mode) | raw fetches default to the 5 MiB ceiling; page-reading mode (`format` set) uses a context-safe 256 KiB. Pass an explicit value to override (up to the 5 MiB ceiling). Over-cap body is truncated, not errored | +| `follow_redirects` | `true` | each hop re-checked against the SSRF blocklist | + +# Response + +**Success** (`ok: true`) — returned for *any* completed response, 2xx through 5xx: + +```jsonc +{ + "ok": true, + "status": 200, + "status_text": "OK", + "headers": { "content-type": "application/json", ... }, // keys LOWER-CASED; set-cookie joined with ", " + "body": "", + "json": { ... }, // only when response_format="json" AND parse succeeded AND not truncated + "parse_error": "…", // only when response_format="json" AND JSON.parse failed (body still set) + "response_format": "json", + "bytes_truncated": false, // true when body hit max_bytes (NOT an error) + "redirect_chain": ["https://…/a"], // omitted when no redirects + "content_type": "text/html", // only in page-reading mode (format set) + "transformed": "markdown" // only when an HTML transform actually ran +} +``` + +**Failure** (`ok: false`) — the fetch never produced a response: + +```jsonc +{ "ok": false, "error": "blocked_host", "message": "…" } // no status field +``` + +## error → cause → fix + +| `error` | Cause | Fix | +|---|---|---| +| `invalid_payload` | Payload failed schema (bad `method`, wrong types). `message` lists the bad fields. | Correct the named fields. | +| `invalid_url` | `url` isn't a parseable absolute `http(s)://` URL. | Pass a full absolute URL incl. scheme. | +| `blocked_host` | Target resolves to a private / link-local / cloud-metadata IP (SSRF guard). | Don't target internal/metadata hosts. For loopback in dev, the operator sets `web.allow_loopback`. | +| `timeout` | Slower than `timeout_ms` (30 s default; 120 s ceiling). | Raise `timeout_ms` (up to ceiling) or shrink work via `max_bytes`. | +| `too_many_redirects` | More than `max_redirects` (5) hops. | Use the final URL directly, or set `follow_redirects: false` and read the `location` header. | +| `transport_error` | Connection refused/reset, TLS failure, DNS failure, or a redirect `Location` that won't parse. | Check host/port/cert; retry if transient. | + +There is **no `too_large` error** — oversize responses come back `ok: true` with `bytes_truncated: true`. Branch on the flag, not on an error. + +# Page reading vs API fetch + +Two orthogonal knobs — pick ONE: + +- **`response_format`** = transport encoding for APIs/binaries (`text`/`base64`/`json`). The body is returned untouched. +- **`format`** = page-reading mode (`markdown`/`text`/`html`). The request goes out with a browser User-Agent + format-matched `Accept` header, and `text/html` responses are transformed (`markdown` is the right default for reading pages — far fewer tokens than raw HTML). Non-HTML bodies pass through unchanged. If Cloudflare answers `403` with a challenge, the worker retries once with its honest UA (beats the UA-fingerprint rule only, not full JS challenges). For very large pages, lower `max_bytes` — conversion runs on the capped body. + +Don't combine them: when `format` is set, `response_format` is ignored (treated as `"text"`). + +**Images in page-reading mode:** a viewable `image/*` response returns the actual image (plus a one-line text summary like `Image fetched (image/png, 8123 bytes)`) instead of the JSON envelope — providers that support tool-result images (Anthropic) show it to the model; others see the text line. "Viewable" means jpeg/png/gif/webp, 2xx status, complete (not truncated), and non-empty — anything else (svg, error pages served as images, truncated bytes) comes back as the normal envelope with `response_format: "base64"` so a hostile image can't fail the provider request. Without `format`, use `response_format: "base64"` as before. + +**Transform bounds:** the HTML→markdown/text conversion runs only on bodies ≤ `web.max_transform_bytes` (1 MiB default) — larger pages come back raw with `transformed` unset (lower `max_bytes` to read huge pages). If the body was truncated at `max_bytes`, the transformed text ends with a visible `[Content truncated at max_bytes — …]` line. + +# Rules that save a turn + +- **`json` vs `body`: set exactly one.** `json` wins if both are present and forces `content-type: application/json`. Use `body` + your own `content-type` for form/text/XML. +- **GET/HEAD ignore the body.** Putting `json`/`body` on a GET sends nothing — switch `method`. +- **`response_format: "json"` can succeed with `parse_error`.** Non-JSON (or truncated) bodies return `ok: true`, no `json`, `parse_error` set; the raw text is still in `body`. Check for `json` before reading it. +- **Caps only go down.** `timeout_ms`/`max_bytes` above the worker ceiling are clamped; you can't request more than the operator allows. +- **Response header keys are lower-cased.** Index `headers["content-type"]`, never `"Content-Type"`. +- **Auth headers don't survive a cross-origin redirect** (see below) — for an authed endpoint, hit the final URL directly. + +# SSRF guard (server-side, not optional) + +DNS is resolved **once**, every resolved address is checked, then the request is dialed at the **validated IP** (pinned `lookup` + TLS `servername`) — so the IP that passed the check is the IP connected to (no DNS-rebind window). + +Blocked **unconditionally** → `error: "blocked_host"`: + +- Private RFC1918 (`10/8`, `172.16/12`, `192.168/16`) +- Link-local incl. **cloud metadata** `169.254.169.254` +- IPv6 ULA (`fc00::/7`), link-local (`fe80::/10`), multicast +- **`::ffff:`-mapped IPv4** in both dotted (`::ffff:169.254.169.254`) and hex (`::ffff:a9fe:a9fe`) forms + +Loopback (`127.0.0.0/8`, `::1`, `localhost`) is **allowed by default** (dev convenience); operators flip `web.allow_loopback: false` for prod, after which loopback returns `blocked_host`. + +On each 3xx the `Location` is re-resolved and **re-validated** before following, and `Authorization` / `Cookie` / `Proxy-Authorization` are **stripped** when the redirect changes host or downgrades `https → http`. + +# Examples + +```jsonc +// Read a documentation page as Markdown +{ "url": "https://docs.example.com/guide", "format": "markdown" } +// → { ok:true, status:200, body:"# Guide\n\n…", transformed:"markdown", content_type:"text/html" } + +// Parse a JSON API +{ "url": "https://api.example.com/status", "response_format": "json" } +// → { ok:true, status:200, json:{ healthy:true } } + +// POST JSON (content-type set for you) +{ "url": "https://api.example.com/things", "method": "post", + "json": { "name": "x" }, "response_format": "json" } + +// Authenticated GET (auth stays only because there's no cross-host redirect) +{ "url": "https://api.example.com/me", + "headers": { "authorization": "Bearer TOKEN" }, "response_format": "json" } + +// Bounded download of a maybe-large file +{ "url": "https://example.com/big.log", "max_bytes": 65536 } +// → { ok:true, body:"", bytes_truncated:true } +``` + +# Authoritative schema + +This page covers behavior the schema can't. For exact field types, call +`engine::functions::info { function_id: "web::fetch" }` — the live schema wins if it ever disagrees with this page. + +# Related + +- [`shell/index`](iii://shell) — local filesystem + process ops; `web::fetch` is the network counterpart (use it, not `shell::exec curl`). +- [`sandbox/index`](iii://sandbox) — code *inside* a sandbox reaches the host engine via the boot-time `III_ENGINE_URL` rewrite, not `web::fetch`. diff --git a/web/src/lib.rs b/web/src/lib.rs new file mode 100644 index 0000000000..87a653e96f --- /dev/null +++ b/web/src/lib.rs @@ -0,0 +1,8 @@ +//! `web` — outbound HTTP client on the iii bus (`web::fetch`). +//! +//! SSRF-guarded fetch with redirect handling, size/timeout caps, JSON +//! helpers, and a page-reading mode (HTML→Markdown/text, viewable images). +//! Standalone Rust port of the former `harness/src/web` TypeScript worker. + +pub mod config; +pub mod manifest; diff --git a/web/src/main.rs b/web/src/main.rs new file mode 100644 index 0000000000..f69a994596 --- /dev/null +++ b/web/src/main.rs @@ -0,0 +1,38 @@ +//! `iii-web` binary entry. Full boot sequence is completed in Task 11; +//! this version supports `--manifest` so the crate is exercisable early. + +use clap::Parser; + +use web::manifest; + +#[derive(Parser, Debug)] +#[command(name = "iii-web", about = "Outbound HTTP client on the iii bus (web::fetch).")] +struct Cli { + #[arg(long)] + config: Option, + #[arg(long, default_value = "ws://127.0.0.1:49134")] + url: String, + #[arg(long)] + manifest: bool, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let cli = Cli::parse(); + if cli.manifest { + let m = manifest::build_manifest(); + println!("{}", serde_json::to_string_pretty(&m).unwrap()); + return Ok(()); + } + + // Full boot wiring is implemented in Task 11. + let _ = (cli.config, cli.url); + Ok(()) +} diff --git a/web/src/manifest.rs b/web/src/manifest.rs new file mode 100644 index 0000000000..58c845d56e --- /dev/null +++ b/web/src/manifest.rs @@ -0,0 +1,40 @@ +//! `--manifest` subcommand output. Same contract as every binary worker. + +use serde::Serialize; + +use crate::config::WebConfig; + +#[derive(Serialize)] +pub struct ModuleManifest { + pub name: String, + pub version: String, + pub description: String, + pub default_config: serde_json::Value, + pub supported_targets: Vec, +} + +pub fn build_manifest() -> ModuleManifest { + ModuleManifest { + name: env!("CARGO_PKG_NAME").to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + description: "Outbound HTTP client on the iii bus (web::fetch).".to_string(), + default_config: WebConfig::default().to_json(), + supported_targets: vec![env!("TARGET").to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_has_required_fields() { + let m = build_manifest(); + let v = serde_json::to_value(&m).unwrap(); + assert_eq!(v["name"], env!("CARGO_PKG_NAME")); + assert_eq!(v["version"], env!("CARGO_PKG_VERSION")); + assert!(v["description"].as_str().is_some_and(|s| !s.is_empty())); + assert_eq!(v["default_config"]["user_agent"], "iii-harness/0.1 (+web::fetch)"); + assert!(!v["supported_targets"].as_array().unwrap().is_empty()); + } +} From 3dde93dcc002e67a7c71adf3b008ef6444608035 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:31:12 -0300 Subject: [PATCH 02/22] feat(web): add WebConfig with defaults, schema, json roundtrip --- web/src/config.rs | 112 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 web/src/config.rs diff --git a/web/src/config.rs b/web/src/config.rs new file mode 100644 index 0000000000..a7d0303ee1 --- /dev/null +++ b/web/src/config.rs @@ -0,0 +1,112 @@ +//! Config for `web::fetch`. Numeric caps are HARD ceilings — a caller may +//! ask for less per call, never more. All fields are read per request, so +//! every field hot-reloads (no topology/restart partition). + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub type SharedConfig = Arc>; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct WebConfig { + /// Per-request timeout used when the caller omits `timeout_ms`. + pub default_timeout_ms: u64, + /// Hard ceiling on per-request timeout. + pub max_timeout_ms: u64, + /// Body cap used in page-reading mode when the caller omits `max_bytes`. + pub default_response_bytes: u64, + /// Hard ceiling on response body bytes accepted before truncation. + pub max_response_bytes: u64, + /// Max HTML body size the page-reading transforms will process. + pub max_transform_bytes: u64, + /// Max redirect hops before giving up. + pub max_redirects: u32, + /// User-Agent we identify ourselves as outside page-reading mode. + pub user_agent: String, + /// Allow requests to loopback (`127.0.0.0/8`, `::1`). Other private + /// ranges stay blocked regardless. + pub allow_loopback: bool, +} + +impl Default for WebConfig { + fn default() -> Self { + Self { + default_timeout_ms: 30_000, + max_timeout_ms: 120_000, + default_response_bytes: 256 * 1024, + max_response_bytes: 5 * 1024 * 1024, + max_transform_bytes: 1024 * 1024, + max_redirects: 5, + user_agent: "iii-harness/0.1 (+web::fetch)".to_string(), + allow_loopback: true, + } + } +} + +impl WebConfig { + pub fn json_schema() -> serde_json::Value { + serde_json::to_value(schemars::schema_for!(WebConfig)).expect("WebConfig schema serializes") + } + + /// Parse from a JSON object; missing keys fall back to defaults + /// (`#[serde(default)]`). The configuration worker may store the value + /// under a `web` wrapper or flat — accept both. + pub fn from_json(v: &serde_json::Value) -> Result { + let inner = v.get("web").unwrap_or(v); + serde_json::from_value(inner.clone()).map_err(|e| format!("invalid web config: {e}")) + } + + pub fn to_json(&self) -> serde_json::Value { + serde_json::to_value(self).expect("WebConfig serializes") + } + + pub fn into_shared(self) -> SharedConfig { + Arc::new(ArcSwap::from_pointee(self)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_match_spec() { + let c = WebConfig::default(); + assert_eq!(c.default_timeout_ms, 30_000); + assert_eq!(c.max_timeout_ms, 120_000); + assert_eq!(c.default_response_bytes, 256 * 1024); + assert_eq!(c.max_response_bytes, 5 * 1024 * 1024); + assert_eq!(c.max_transform_bytes, 1024 * 1024); + assert_eq!(c.max_redirects, 5); + assert_eq!(c.user_agent, "iii-harness/0.1 (+web::fetch)"); + assert!(c.allow_loopback); + } + + #[test] + fn json_roundtrip() { + let c = WebConfig { max_redirects: 2, allow_loopback: false, ..WebConfig::default() }; + let back = WebConfig::from_json(&c.to_json()).unwrap(); + assert_eq!(back.max_redirects, 2); + assert!(!back.allow_loopback); + } + + #[test] + fn from_json_fills_missing_with_defaults() { + let v = serde_json::json!({ "max_redirects": 9 }); + let c = WebConfig::from_json(&v).unwrap(); + assert_eq!(c.max_redirects, 9); + assert_eq!(c.max_timeout_ms, 120_000); // default preserved + } + + #[test] + fn schema_lists_fields() { + let s = WebConfig::json_schema(); + let props = &s["properties"]; + assert!(props.get("user_agent").is_some()); + assert!(props.get("allow_loopback").is_some()); + } +} From 6e80ca10ff437393b68782e02da4f5ed7938ab28 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:35:17 -0300 Subject: [PATCH 03/22] feat(web): SSRF blocklist (ipnet) + parse_target --- web/src/lib.rs | 1 + web/src/ssrf.rs | 170 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 web/src/ssrf.rs diff --git a/web/src/lib.rs b/web/src/lib.rs index 87a653e96f..7c1ca16529 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -6,3 +6,4 @@ pub mod config; pub mod manifest; +pub mod ssrf; diff --git a/web/src/ssrf.rs b/web/src/ssrf.rs new file mode 100644 index 0000000000..34b1a31bbc --- /dev/null +++ b/web/src/ssrf.rs @@ -0,0 +1,170 @@ +//! SSRF defense for `web::fetch`. Parse + validate the URL, resolve the +//! host, and reject any address in a private / loopback / link-local / +//! multicast / reserved range. The resolve+validate happens once; the +//! socket is pinned to the validated IP (see fetch.rs) to defeat DNS +//! rebinding (TOCTOU between check and connect). + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use ipnet::{Ipv4Net, Ipv6Net}; + +#[derive(Debug, Clone, Copy)] +pub struct SsrfPolicy { + pub allow_loopback: bool, +} + +#[derive(Debug, Clone)] +pub struct ParsedTarget { + pub url: reqwest::Url, + pub hostname: String, + pub port: u16, +} + +/// Parse + validate scheme and host shape. No DNS. +pub fn parse_target(raw: &str) -> Result { + let url = reqwest::Url::parse(raw).map_err(|_| "url is not a valid absolute URL".to_string())?; + let scheme = url.scheme(); + if scheme != "http" && scheme != "https" { + return Err(format!( + "scheme not allowed: {scheme}: (only http: and https: are permitted)" + )); + } + let hostname = url + .host_str() + .filter(|h| !h.is_empty()) + .ok_or_else(|| "url has no hostname".to_string())? + .to_string(); + let port = url + .port_or_known_default() + .ok_or_else(|| "invalid port".to_string())?; + Ok(ParsedTarget { url, hostname, port }) +} + +fn v4_blocklist() -> Vec<(Ipv4Net, &'static str)> { + [ + ("0.0.0.0/8", "this-network"), + ("10.0.0.0/8", "private rfc1918"), + ("100.64.0.0/10", "cgnat rfc6598"), + ("127.0.0.0/8", "loopback"), + ("169.254.0.0/16", "link-local (incl. AWS metadata)"), + ("172.16.0.0/12", "private rfc1918"), + ("192.0.0.0/24", "ietf protocol assignments"), + ("192.0.2.0/24", "documentation"), + ("192.168.0.0/16", "private rfc1918"), + ("198.18.0.0/15", "benchmarking"), + ("198.51.100.0/24", "documentation"), + ("203.0.113.0/24", "documentation"), + ("224.0.0.0/4", "multicast"), + ("240.0.0.0/4", "reserved"), + ] + .into_iter() + .map(|(cidr, label)| (cidr.parse::().expect("valid v4 cidr"), label)) + .collect() +} + +fn check_ipv4(addr: Ipv4Addr, policy: &SsrfPolicy) -> Option<&'static str> { + for (net, label) in v4_blocklist() { + if label == "loopback" && policy.allow_loopback { + continue; + } + if net.contains(&addr) { + return Some(label); + } + } + None +} + +fn check_ipv6(addr: Ipv6Addr, policy: &SsrfPolicy) -> Option<&'static str> { + // IPv4-mapped (::ffff:a.b.c.d) routes to v4 — delegate to the v4 check. + if let Some(v4) = addr.to_ipv4_mapped() { + return check_ipv4(v4, policy); + } + if addr == Ipv6Addr::LOCALHOST { + return if policy.allow_loopback { None } else { Some("loopback") }; + } + if addr == Ipv6Addr::UNSPECIFIED { + return Some("unspecified"); + } + let blocks: [(Ipv6Net, &'static str); 3] = [ + ("fe80::/10".parse().unwrap(), "link-local fe80::/10"), + ("fc00::/7".parse().unwrap(), "unique-local fc00::/7"), + ("ff00::/8".parse().unwrap(), "multicast ff00::/8"), + ]; + for (net, label) in blocks { + if net.contains(&addr) { + return Some(label); + } + } + None +} + +/// Returns `Some(label)` if the address is blocked, `None` if allowed. +pub fn check_ip(addr: IpAddr, policy: &SsrfPolicy) -> Option<&'static str> { + match addr { + IpAddr::V4(v4) => check_ipv4(v4, policy), + IpAddr::V6(v6) => check_ipv6(v6, policy), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::IpAddr; + + fn blocked(ip: &str, allow_loopback: bool) -> Option<&'static str> { + check_ip(ip.parse::().unwrap(), &SsrfPolicy { allow_loopback }) + } + + #[test] + fn blocks_private_v4_ranges() { + for ip in ["10.0.0.1", "172.16.5.5", "192.168.1.1", "169.254.169.254", "100.64.0.1", "0.0.0.0", "224.0.0.1", "240.0.0.1"] { + assert!(blocked(ip, true).is_some(), "{ip} should be blocked"); + } + } + + #[test] + fn allows_public_v4() { + assert!(blocked("1.1.1.1", true).is_none()); + assert!(blocked("93.184.216.34", true).is_none()); + } + + #[test] + fn loopback_policy_v4_and_v6() { + assert!(blocked("127.0.0.1", true).is_none()); + assert!(blocked("127.0.0.1", false).is_some()); + assert!(blocked("::1", true).is_none()); + assert!(blocked("::1", false).is_some()); + // loopback=true still blocks other private ranges + assert!(blocked("169.254.169.254", true).is_some()); + assert!(blocked("10.0.0.1", true).is_some()); + } + + #[test] + fn blocks_v6_ranges() { + for ip in ["::", "fe80::1", "febf::1", "fc00::1", "fd12::1", "ff02::1"] { + assert!(blocked(ip, true).is_some(), "{ip} should be blocked"); + } + assert!(blocked("2606:4700:4700::1111", true).is_none()); + } + + #[test] + fn blocks_v4_mapped_v6_metadata() { + // both textual and (parsed) hex forms collapse to 169.254.169.254 + assert!(blocked("::ffff:169.254.169.254", true).is_some()); + assert!(blocked("::ffff:a9fe:a9fe", true).is_some()); + } + + #[test] + fn parse_target_rejects_non_http() { + assert!(parse_target("ftp://example.com/").is_err()); + assert!(parse_target("file:///etc/passwd").is_err()); + assert!(parse_target("not a url").is_err()); + } + + #[test] + fn parse_target_defaults_ports() { + assert_eq!(parse_target("http://example.com/").unwrap().port, 80); + assert_eq!(parse_target("https://example.com/").unwrap().port, 443); + assert_eq!(parse_target("http://example.com:8080/").unwrap().port, 8080); + } +} From 786cff3ced2cc7734891ab33ba3a06e8fef38161 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:38:17 -0300 Subject: [PATCH 04/22] refactor(web): hoist SSRF blocklists into OnceLock statics --- web/src/ssrf.rs | 65 +++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/web/src/ssrf.rs b/web/src/ssrf.rs index 34b1a31bbc..ec67ace8fd 100644 --- a/web/src/ssrf.rs +++ b/web/src/ssrf.rs @@ -5,6 +5,7 @@ //! rebinding (TOCTOU between check and connect). use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::sync::OnceLock; use ipnet::{Ipv4Net, Ipv6Net}; @@ -40,31 +41,46 @@ pub fn parse_target(raw: &str) -> Result { Ok(ParsedTarget { url, hostname, port }) } -fn v4_blocklist() -> Vec<(Ipv4Net, &'static str)> { - [ - ("0.0.0.0/8", "this-network"), - ("10.0.0.0/8", "private rfc1918"), - ("100.64.0.0/10", "cgnat rfc6598"), - ("127.0.0.0/8", "loopback"), - ("169.254.0.0/16", "link-local (incl. AWS metadata)"), - ("172.16.0.0/12", "private rfc1918"), - ("192.0.0.0/24", "ietf protocol assignments"), - ("192.0.2.0/24", "documentation"), - ("192.168.0.0/16", "private rfc1918"), - ("198.18.0.0/15", "benchmarking"), - ("198.51.100.0/24", "documentation"), - ("203.0.113.0/24", "documentation"), - ("224.0.0.0/4", "multicast"), - ("240.0.0.0/4", "reserved"), - ] - .into_iter() - .map(|(cidr, label)| (cidr.parse::().expect("valid v4 cidr"), label)) - .collect() +static V4_BLOCKLIST: OnceLock> = OnceLock::new(); +static V6_BLOCKLIST: OnceLock<[(Ipv6Net, &'static str); 3]> = OnceLock::new(); + +fn v4_blocklist() -> &'static [(Ipv4Net, &'static str)] { + V4_BLOCKLIST.get_or_init(|| { + [ + ("0.0.0.0/8", "this-network"), + ("10.0.0.0/8", "private rfc1918"), + ("100.64.0.0/10", "cgnat rfc6598"), + ("127.0.0.0/8", "loopback"), + ("169.254.0.0/16", "link-local (incl. AWS metadata)"), + ("172.16.0.0/12", "private rfc1918"), + ("192.0.0.0/24", "ietf protocol assignments"), + ("192.0.2.0/24", "documentation"), + ("192.168.0.0/16", "private rfc1918"), + ("198.18.0.0/15", "benchmarking"), + ("198.51.100.0/24", "documentation"), + ("203.0.113.0/24", "documentation"), + ("224.0.0.0/4", "multicast"), + ("240.0.0.0/4", "reserved"), + ] + .into_iter() + .map(|(cidr, label)| (cidr.parse::().expect("valid v4 cidr"), label)) + .collect() + }) +} + +fn v6_blocklist() -> &'static [(Ipv6Net, &'static str)] { + V6_BLOCKLIST.get_or_init(|| { + [ + ("fe80::/10".parse().unwrap(), "link-local fe80::/10"), + ("fc00::/7".parse().unwrap(), "unique-local fc00::/7"), + ("ff00::/8".parse().unwrap(), "multicast ff00::/8"), + ] + }) } fn check_ipv4(addr: Ipv4Addr, policy: &SsrfPolicy) -> Option<&'static str> { for (net, label) in v4_blocklist() { - if label == "loopback" && policy.allow_loopback { + if *label == "loopback" && policy.allow_loopback { continue; } if net.contains(&addr) { @@ -85,12 +101,7 @@ fn check_ipv6(addr: Ipv6Addr, policy: &SsrfPolicy) -> Option<&'static str> { if addr == Ipv6Addr::UNSPECIFIED { return Some("unspecified"); } - let blocks: [(Ipv6Net, &'static str); 3] = [ - ("fe80::/10".parse().unwrap(), "link-local fe80::/10"), - ("fc00::/7".parse().unwrap(), "unique-local fc00::/7"), - ("ff00::/8".parse().unwrap(), "multicast ff00::/8"), - ]; - for (net, label) in blocks { + for (net, label) in v6_blocklist() { if net.contains(&addr) { return Some(label); } From ce1b531cd96e5f35d4cc01f76ad5d4d46f3b33cf Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:42:33 -0300 Subject: [PATCH 05/22] feat(web): check_target with DNS resolve, literal-IP short-circuit, fail-closed --- web/src/ssrf.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 5 deletions(-) diff --git a/web/src/ssrf.rs b/web/src/ssrf.rs index ec67ace8fd..2badef0250 100644 --- a/web/src/ssrf.rs +++ b/web/src/ssrf.rs @@ -30,11 +30,15 @@ pub fn parse_target(raw: &str) -> Result { "scheme not allowed: {scheme}: (only http: and https: are permitted)" )); } - let hostname = url - .host_str() - .filter(|h| !h.is_empty()) - .ok_or_else(|| "url has no hostname".to_string())? - .to_string(); + let hostname = { + let h = url + .host_str() + .filter(|h| !h.is_empty()) + .ok_or_else(|| "url has no hostname".to_string())?; + // `host_str()` keeps the brackets on an IPv6 literal (e.g. "[::1]"). + // Strip them so downstream code can parse the string as an IpAddr. + h.trim_start_matches('[').trim_end_matches(']').to_string() + }; let port = url .port_or_known_default() .ok_or_else(|| "invalid port".to_string())?; @@ -117,6 +121,89 @@ pub fn check_ip(addr: IpAddr, policy: &SsrfPolicy) -> Option<&'static str> { } } +use std::net::SocketAddr; + +#[derive(Debug, Clone)] +pub struct ResolvedTarget { + pub address: IpAddr, + pub hostname: String, + pub port: u16, +} + +#[derive(Debug, Clone)] +pub struct SsrfReject { + pub code: &'static str, + pub message: String, +} + +fn loopback_hint(policy: &SsrfPolicy) -> &'static str { + if policy.allow_loopback { + "" + } else { + " (set web.allow_loopback=true in config.yaml if loopback is intentional in this environment)" + } +} + +/// Resolve the host (or accept a literal IP), validate EVERY resolved +/// address against the blocklist, and return the address to dial (the +/// first resolved address). If ANY address is blocked, the whole call is +/// refused. Fail-closed: DNS failure / empty result map to `blocked_host`. +pub async fn check_target( + target: &ParsedTarget, + policy: &SsrfPolicy, +) -> Result { + let host = &target.hostname; + + // Literal IP: skip DNS, check directly. `host_str()` already strips + // the brackets around a v6 literal. + if let Ok(ip) = host.parse::() { + if let Some(label) = check_ip(ip, policy) { + let hint = if label == "loopback" { loopback_hint(policy) } else { "" }; + return Err(SsrfReject { + code: "blocked_host", + message: format!("address {host} is in {label}{hint}"), + }); + } + return Ok(ResolvedTarget { address: ip, hostname: host.clone(), port: target.port }); + } + + let resolved: Vec = match tokio::net::lookup_host((host.as_str(), target.port)).await + { + Ok(it) => it.collect(), + Err(e) => { + return Err(SsrfReject { + code: "blocked_host", + message: format!("dns lookup failed for {host}: {e}"), + }); + } + }; + if resolved.is_empty() { + return Err(SsrfReject { + code: "blocked_host", + message: format!("dns returned no addresses for {host}"), + }); + } + + for sa in &resolved { + if let Some(label) = check_ip(sa.ip(), policy) { + let hint = if label == "loopback" { loopback_hint(policy) } else { "" }; + return Err(SsrfReject { + code: "blocked_host", + message: format!( + "{host} resolves to {} ({label}); refusing to dial{hint}", + sa.ip() + ), + }); + } + } + + Ok(ResolvedTarget { + address: resolved[0].ip(), + hostname: host.clone(), + port: target.port, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -178,4 +265,27 @@ mod tests { assert_eq!(parse_target("https://example.com/").unwrap().port, 443); assert_eq!(parse_target("http://example.com:8080/").unwrap().port, 8080); } + + #[tokio::test] + async fn check_target_literal_v4_public_ok() { + let t = parse_target("http://1.1.1.1/").unwrap(); + let r = check_target(&t, &SsrfPolicy { allow_loopback: true }).await.unwrap(); + assert_eq!(r.address, "1.1.1.1".parse::().unwrap()); + assert_eq!(r.port, 80); + } + + #[tokio::test] + async fn check_target_literal_v4_private_blocked() { + let t = parse_target("http://169.254.169.254/").unwrap(); + let rej = check_target(&t, &SsrfPolicy { allow_loopback: true }).await.unwrap_err(); + assert_eq!(rej.code, "blocked_host"); + } + + #[tokio::test] + async fn check_target_literal_v6_bracket_stripped_and_blocked() { + let t = parse_target("http://[::1]/").unwrap(); + let rej = check_target(&t, &SsrfPolicy { allow_loopback: false }).await.unwrap_err(); + assert_eq!(rej.code, "blocked_host"); + assert!(rej.message.contains("allow_loopback")); + } } From e25acb26ea9ceb6ff05d92ccded6cb3d589640b5 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:46:26 -0300 Subject: [PATCH 06/22] feat(web): FetchPayload wire schema, enums, validation, tool description --- web/src/lib.rs | 1 + web/src/schemas.rs | 165 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 web/src/schemas.rs diff --git a/web/src/lib.rs b/web/src/lib.rs index 7c1ca16529..001d86ed66 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -6,4 +6,5 @@ pub mod config; pub mod manifest; +pub mod schemas; pub mod ssrf; diff --git a/web/src/schemas.rs b/web/src/schemas.rs new file mode 100644 index 0000000000..9413351fd1 --- /dev/null +++ b/web/src/schemas.rs @@ -0,0 +1,165 @@ +//! Wire schema + tool description for `web::fetch`. `FetchPayload` derives +//! `JsonSchema` (for `request_format`) and `Deserialize`. The handler takes +//! a raw `Value` and parses this so a bad payload returns `invalid_payload` +//! rather than an SDK-level error (the SDK deserializes typed inputs before +//! the handler runs). + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::Deserialize; + +pub const METHODS: [&str; 7] = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum ResponseFormat { + Text, + Base64, + Json, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum PageFormat { + Markdown, + Text, + Html, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct FetchPayload { + /// Absolute http(s):// URL to fetch. + pub url: String, + /// HTTP method. Defaults to GET. Case-insensitive ("get" works). + #[serde(default)] + pub method: Option, + /// Request headers. Lower-cased keys recommended. + #[serde(default)] + pub headers: Option>, + /// Raw request body as a string. Prefer `json` for JSON payloads. + #[serde(default)] + pub body: Option, + /// Structured JSON payload. When set, the handler stringifies it and + /// forces `content-type: application/json`; wins over `body`. + #[serde(default)] + pub json: Option, + /// Per-request timeout in ms. Capped by the worker's max_timeout_ms. + #[serde(default)] + pub timeout_ms: Option, + /// Cap on response body bytes. Defaults to the worker ceiling for raw + /// fetches, or 256 KiB in page-reading mode (`format` set). + #[serde(default)] + pub max_bytes: Option, + /// Follow 3xx redirects. Defaults to true. Each hop is re-checked + /// against the SSRF blocklist. + #[serde(default)] + pub follow_redirects: Option, + /// How to return the body: "text" (default), "base64", or "json". + #[serde(default)] + pub response_format: Option, + /// Page-reading mode: "markdown" | "text" | "html". When set, a browser + /// UA + format Accept header are used, HTML is transformed, and + /// response_format is ignored (treated as "text"). + #[serde(default)] + pub format: Option, +} + +impl FetchPayload { + pub fn normalized_method(&self) -> String { + self.method + .as_deref() + .map(|m| m.to_uppercase()) + .unwrap_or_else(|| "GET".to_string()) + } + + /// Post-parse validation that serde can't express (positivity, method + /// enum, non-empty url). Returns an `invalid_payload` message on failure. + pub fn validate(&self) -> Result<(), String> { + if self.url.is_empty() { + return Err("url: must be a non-empty string".to_string()); + } + if let Some(m) = &self.method { + if !METHODS.contains(&m.to_uppercase().as_str()) { + return Err(format!("method: must be one of {}", METHODS.join(", "))); + } + } + if matches!(self.timeout_ms, Some(0)) { + return Err("timeout_ms: must be a positive integer".to_string()); + } + if matches!(self.max_bytes, Some(0)) { + return Err("max_bytes: must be a positive integer".to_string()); + } + Ok(()) + } +} + +pub fn request_schema() -> serde_json::Value { + serde_json::to_value(schemars::schema_for!(FetchPayload)).expect("FetchPayload schema serializes") +} + +pub const TOOL_DESCRIPTION: &str = concat!( + "Fetch a URL over HTTP(S) and return the response as a structured envelope. ", + "Use this INSTEAD of `shell::exec` with curl for any HTTP request — it ", + "returns {ok, status, headers, body} as JSON, enforces size/timeout caps, ", + "and blocks private / cloud-metadata / link-local addresses server-side ", + "(SSRF guard; loopback is allowed by default for harness dev workflows). ", + "To READ A WEB PAGE, set `format: \"markdown\"` — HTML is converted to ", + "Markdown and images come back viewable (image responses in that mode ", + "return the image itself plus a text line, not the {ok,...} envelope). ", + "For JSON: pass `json: {...}` (auto-stringifies + sets content-type) and ", + "`response_format: \"json\"` (auto-parses response into the `json` field). ", + "Method is case-insensitive. On failure returns `{ok:false, error, message}` ", + "where `error` is one of: invalid_payload, invalid_url, blocked_host, timeout, ", + "too_many_redirects, transport_error. Branch on `error`, not on text." +); + +#[cfg(test)] +mod tests { + use super::*; + + fn payload(j: serde_json::Value) -> Result { + serde_json::from_value(j) + } + + #[test] + fn parses_minimal() { + let p = payload(serde_json::json!({ "url": "https://x.test/" })).unwrap(); + assert_eq!(p.url, "https://x.test/"); + assert_eq!(p.normalized_method(), "GET"); + p.validate().unwrap(); + } + + #[test] + fn method_case_insensitive() { + let p = payload(serde_json::json!({ "url": "https://x.test/", "method": "post" })).unwrap(); + assert_eq!(p.normalized_method(), "POST"); + p.validate().unwrap(); + } + + #[test] + fn rejects_unknown_method() { + let p = payload(serde_json::json!({ "url": "https://x.test/", "method": "FETCH" })).unwrap(); + assert!(p.validate().is_err()); + } + + #[test] + fn rejects_empty_url_and_nonpositive_caps() { + assert!(payload(serde_json::json!({ "url": "" })).unwrap().validate().is_err()); + assert!(payload(serde_json::json!({ "url": "https://x/", "timeout_ms": 0 })).unwrap().validate().is_err()); + assert!(payload(serde_json::json!({ "url": "https://x/", "max_bytes": 0 })).unwrap().validate().is_err()); + } + + #[test] + fn rejects_unknown_enum_values() { + assert!(payload(serde_json::json!({ "url": "https://x/", "format": "pdf" })).is_err()); + assert!(payload(serde_json::json!({ "url": "https://x/", "response_format": "xml" })).is_err()); + } + + #[test] + fn request_schema_is_object_with_url() { + let s = request_schema(); + assert_eq!(s["type"], "object"); + assert!(s["properties"].get("url").is_some()); + } +} From 15f27e60ea21bf870041be927ef857e7ac49ad63 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:49:55 -0300 Subject: [PATCH 07/22] =?UTF-8?q?feat(web):=20HTML=E2=86=92markdown=20(htm?= =?UTF-8?q?d)=20+=20text=20extractor=20(tl)=20+=20depth=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/convert.rs | 211 +++++++++++++++++++++++++++++++++++++++++++++ web/src/lib.rs | 1 + 2 files changed, 212 insertions(+) create mode 100644 web/src/convert.rs diff --git a/web/src/convert.rs b/web/src/convert.rs new file mode 100644 index 0000000000..ca95dda07a --- /dev/null +++ b/web/src/convert.rs @@ -0,0 +1,211 @@ +//! Content transforms for `web::fetch` page-reading mode. Pure functions, +//! no I/O. HTML→Markdown via `htmd`; plain-text via an iterative walk over +//! `tl`'s flat node arena that skips non-content subtrees. Callers MUST run +//! `max_tag_depth` and skip the transform when it exceeds `MAX_NESTING_DEPTH` +//! (htmd recurses over an rcdom tree; a stack overflow ABORTS the process +//! and is not catchable by `catch_unwind`). + +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use crate::schemas::PageFormat; + +pub const BROWSER_USER_AGENT: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"; +pub const ACCEPT_LANGUAGE: &str = "en-US,en;q=0.9"; +pub const MAX_NESTING_DEPTH: usize = 200; + +const SKIP_TAGS: [&str; 6] = ["script", "style", "noscript", "iframe", "object", "embed"]; +const BLOCK_TAGS: [&str; 19] = [ + "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr", "section", "article", "header", + "footer", "blockquote", "pre", "table", "ul", "ol", +]; + +pub fn accept_header_for(format: PageFormat) -> &'static str { + match format { + PageFormat::Markdown => { + "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" + } + PageFormat::Text => "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1", + PageFormat::Html => { + "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" + } + } +} + +pub fn is_image_mime(mime: &str) -> bool { + mime.starts_with("image/") +} + +pub fn is_viewable_image_mime(mime: &str) -> bool { + matches!(mime, "image/jpeg" | "image/png" | "image/gif" | "image/webp") +} + +/// Maximum tag-nesting depth, computed iteratively (no recursion, cannot +/// itself overflow). Used as a pre-transform guard. +pub fn max_tag_depth(html: &str) -> usize { + let dom = match tl::parse(html, tl::ParserOptions::default()) { + Ok(d) => d, + Err(_) => return 0, + }; + let parser = dom.parser(); + let mut max = 0usize; + // stack of (node_handle, depth) + let mut stack: Vec<(tl::NodeHandle, usize)> = + dom.children().iter().map(|h| (*h, 1usize)).collect(); + while let Some((handle, depth)) = stack.pop() { + if depth > max { + max = depth; + } + if let Some(tag) = handle.get(parser).and_then(|n| n.as_tag()) { + for child in tag.children().top().as_slice().iter() { + stack.push((*child, depth + 1)); + } + } + } + max +} + +/// Iterative plain-text extraction. Skips SKIP_TAGS subtrees entirely (by +/// not descending), emits '\n' for
and on BLOCK_TAGS close, collapses +/// 3+ newlines to 2, trims. +pub fn extract_text(html: &str) -> String { + enum Work { + Enter(tl::NodeHandle), + CloseBlock, + } + let dom = match tl::parse(html, tl::ParserOptions::default()) { + Ok(d) => d, + Err(_) => return String::new(), + }; + let parser = dom.parser(); + let mut out = String::new(); + let mut stack: Vec = dom.children().iter().rev().map(|h| Work::Enter(*h)).collect(); + + while let Some(item) = stack.pop() { + match item { + Work::CloseBlock => out.push('\n'), + Work::Enter(handle) => { + let Some(node) = handle.get(parser) else { continue }; + match node { + tl::Node::Raw(bytes) => out.push_str(&bytes.as_utf8_str()), + tl::Node::Tag(tag) => { + let name = tag.name().as_utf8_str(); + let name = name.as_ref(); + if SKIP_TAGS.contains(&name) { + continue; // skip subtree + } + if name == "br" { + out.push('\n'); + continue; + } + if BLOCK_TAGS.contains(&name) { + stack.push(Work::CloseBlock); + } + for child in tag.children().top().as_slice().iter().rev() { + stack.push(Work::Enter(*child)); + } + } + _ => {} + } + } + } + } + + collapse_blank_lines(&out).trim().to_string() +} + +fn collapse_blank_lines(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut newlines = 0usize; + for ch in s.chars() { + if ch == '\n' { + newlines += 1; + if newlines <= 2 { + out.push('\n'); + } + } else { + newlines = 0; + out.push(ch); + } + } + out +} + +/// HTML→Markdown via htmd. Returns Err on failure so the caller falls back +/// to the raw body. Wrapped in catch_unwind for ordinary panics; the +/// caller's depth guard handles the (uncatchable) stack-overflow case. +pub fn html_to_markdown(html: &str) -> Result { + let result = catch_unwind(AssertUnwindSafe(|| { + htmd::HtmlToMarkdown::builder() + .skip_tags(vec!["script", "style", "meta", "link"]) + .build() + .convert(html) + })); + match result { + Ok(Ok(md)) => Ok(md), + Ok(Err(e)) => Err(format!("htmd convert failed: {e}")), + Err(_) => Err("htmd panicked".to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schemas::PageFormat; + + #[test] + fn mime_helpers() { + assert!(is_image_mime("image/png")); + assert!(!is_image_mime("text/html")); + assert!(is_viewable_image_mime("image/jpeg")); + assert!(!is_viewable_image_mime("image/svg+xml")); + } + + #[test] + fn accept_headers_differ_by_format() { + assert!(accept_header_for(PageFormat::Markdown).contains("text/markdown")); + assert!(accept_header_for(PageFormat::Text).starts_with("text/plain")); + assert!(accept_header_for(PageFormat::Html).contains("application/xhtml+xml")); + } + + #[test] + fn extracts_text_skipping_script_and_blocking() { + let html = "

Title

Hello

World

"; + let text = extract_text(html); + assert!(text.contains("Title")); + assert!(text.contains("Hello")); + assert!(text.contains("World")); + assert!(!text.contains("var x")); + // block tags force separation — no "TitleHello" run + assert!(!text.contains("TitleHello")); + } + + #[test] + fn extract_text_collapses_blank_runs_and_trims() { + let html = "

only

"; + let text = extract_text(html); + assert!(!text.contains("\n\n\n")); + assert_eq!(text.trim(), text); + } + + #[test] + fn br_becomes_newline() { + let text = extract_text("a
b"); + assert!(text.contains("a\nb")); + } + + #[test] + fn markdown_drops_script_style_and_renders_heading() { + let md = html_to_markdown("

Hi

").unwrap(); + assert!(md.contains("# Hi") || md.contains("Hi")); + assert!(!md.contains("bad()")); + assert!(!md.contains("x{}")); + } + + #[test] + fn depth_guard_counts_nesting() { + let deep = "
".repeat(50) + "x" + &"
".repeat(50); + assert!(max_tag_depth(&deep) >= 50); + assert!(max_tag_depth("

flat

") <= 2); + } +} diff --git a/web/src/lib.rs b/web/src/lib.rs index 001d86ed66..0e881e2571 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -5,6 +5,7 @@ //! Standalone Rust port of the former `harness/src/web` TypeScript worker. pub mod config; +pub mod convert; pub mod manifest; pub mod schemas; pub mod ssrf; From 466152ff8aca439886416873177dc9f3a03efec4 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:53:17 -0300 Subject: [PATCH 08/22] feat(web): fetch.rs pure helpers (timeout/bytes/json/auth-strip/encode) --- web/src/fetch.rs | 146 +++++++++++++++++++++++++++++++++++++++++++++++ web/src/lib.rs | 1 + 2 files changed, 147 insertions(+) create mode 100644 web/src/fetch.rs diff --git a/web/src/fetch.rs b/web/src/fetch.rs new file mode 100644 index 0000000000..f152338a0d --- /dev/null +++ b/web/src/fetch.rs @@ -0,0 +1,146 @@ +//! The HTTP call for `web::fetch`: SSRF-checked, IP-pinned, manual redirect +//! loop with per-hop re-validation, byte-capped streaming read, and +//! page-reading transforms. `execute_fetch` lands in Task 9; this file +//! starts with the pure helpers. + +use std::collections::BTreeMap; + +use base64::{engine::general_purpose::STANDARD, Engine as _}; + +use crate::config::WebConfig; +use crate::schemas::{FetchPayload, ResponseFormat}; + +const DENY_ON_REDIRECT: [&str; 3] = ["authorization", "cookie", "proxy-authorization"]; + +/// Default to `default_timeout_ms`, never exceed `max_timeout_ms`. +pub fn resolve_timeout(p: &FetchPayload, cfg: &WebConfig) -> u64 { + p.timeout_ms.unwrap_or(cfg.default_timeout_ms).min(cfg.max_timeout_ms) +} + +/// Raw fetches default to the hard ceiling; page mode defaults to the +/// context-safe `default_response_bytes`. Never exceed `max_response_bytes`. +pub fn resolve_max_bytes(p: &FetchPayload, cfg: &WebConfig) -> u64 { + let fallback = if p.format.is_some() { + cfg.default_response_bytes + } else { + cfg.max_response_bytes + }; + p.max_bytes.unwrap_or(fallback).min(cfg.max_response_bytes) +} + +/// If `json` is set, stringify it into the body and set content-type to +/// application/json (only when the caller didn't set one). `json` wins. +pub fn apply_json_payload( + p: &FetchPayload, + mut headers: BTreeMap, +) -> (Option, BTreeMap) { + let Some(json) = &p.json else { + return (p.body.clone(), headers); + }; + let has_ct = headers.keys().any(|k| k.eq_ignore_ascii_case("content-type")); + if !has_ct { + headers.insert("content-type".to_string(), "application/json".to_string()); + } + (Some(json.to_string()), headers) +} + +/// Strip auth/cookie when the redirect leaves the origin OR downgrades +/// https→http (a same-host downgrade still leaks creds in cleartext). +pub fn strip_cross_origin_auth( + headers: BTreeMap, + from: &reqwest::Url, + to: &reqwest::Url, +) -> BTreeMap { + let same_host = from.host_str() == to.host_str() + && from.port_or_known_default() == to.port_or_known_default(); + let downgrade = from.scheme() == "https" && to.scheme() == "http"; + if same_host && !downgrade { + return headers; + } + headers + .into_iter() + .filter(|(k, _)| !DENY_ON_REDIRECT.contains(&k.to_lowercase().as_str())) + .collect() +} + +pub fn encode_body(bytes: &[u8], format: ResponseFormat) -> String { + match format { + ResponseFormat::Base64 => STANDARD.encode(bytes), + // text and json both return raw utf8 in `body`; json parsing is additive. + _ => String::from_utf8_lossy(bytes).into_owned(), + } +} + +pub fn try_parse_json(text: &str) -> Result { + serde_json::from_str(text).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::WebConfig; + use crate::schemas::{FetchPayload, ResponseFormat}; + use std::collections::BTreeMap; + + fn p(j: serde_json::Value) -> FetchPayload { + serde_json::from_value(j).unwrap() + } + + #[test] + fn timeout_defaults_and_caps() { + let cfg = WebConfig::default(); + assert_eq!(resolve_timeout(&p(serde_json::json!({"url":"x"})), &cfg), 30_000); + assert_eq!(resolve_timeout(&p(serde_json::json!({"url":"x","timeout_ms":5000})), &cfg), 5_000); + assert_eq!(resolve_timeout(&p(serde_json::json!({"url":"x","timeout_ms":999999})), &cfg), 120_000); + } + + #[test] + fn max_bytes_defaults_by_page_mode() { + let cfg = WebConfig::default(); + assert_eq!(resolve_max_bytes(&p(serde_json::json!({"url":"x"})), &cfg), 5 * 1024 * 1024); + assert_eq!(resolve_max_bytes(&p(serde_json::json!({"url":"x","format":"markdown"})), &cfg), 256 * 1024); + assert_eq!(resolve_max_bytes(&p(serde_json::json!({"url":"x","max_bytes":999999999u64})), &cfg), 5 * 1024 * 1024); + } + + #[test] + fn json_payload_sets_body_and_content_type() { + let pl = p(serde_json::json!({"url":"x","json":{"a":1}})); + let (body, headers) = apply_json_payload(&pl, BTreeMap::new()); + assert_eq!(body.unwrap(), "{\"a\":1}"); + assert_eq!(headers.get("content-type").unwrap(), "application/json"); + } + + #[test] + fn json_payload_respects_caller_content_type() { + let pl = p(serde_json::json!({"url":"x","json":{"a":1}})); + let mut h = BTreeMap::new(); + h.insert("content-type".to_string(), "application/vnd.x".to_string()); + let (_, headers) = apply_json_payload(&pl, h); + assert_eq!(headers.get("content-type").unwrap(), "application/vnd.x"); + } + + #[test] + fn strip_creds_on_host_change_and_downgrade() { + let mut h = BTreeMap::new(); + h.insert("authorization".to_string(), "Bearer t".to_string()); + h.insert("cookie".to_string(), "s=1".to_string()); + let same = strip_cross_origin_auth(h.clone(), &"https://a.test/".parse().unwrap(), &"https://a.test/2".parse().unwrap()); + assert!(same.contains_key("authorization")); + let cross = strip_cross_origin_auth(h.clone(), &"https://a.test/".parse().unwrap(), &"https://b.test/".parse().unwrap()); + assert!(!cross.contains_key("authorization")); + let downgrade = strip_cross_origin_auth(h, &"https://a.test/".parse().unwrap(), &"http://a.test/".parse().unwrap()); + assert!(!downgrade.contains_key("cookie")); + } + + #[test] + fn encode_body_base64_and_text() { + assert_eq!(encode_body(b"hi", ResponseFormat::Text), "hi"); + assert_eq!(encode_body(b"hi", ResponseFormat::Base64), "aGk="); + } + + #[test] + fn parse_json_ok_and_err() { + assert!(try_parse_json("{\"a\":1}").is_ok()); + assert!(try_parse_json("not json").is_err()); + } +} diff --git a/web/src/lib.rs b/web/src/lib.rs index 0e881e2571..f4d09d054b 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub mod convert; +pub mod fetch; pub mod manifest; pub mod schemas; pub mod ssrf; From 39f8caf5284bcea4b5d63cd59965bb52c1ad3d35 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 11:56:17 -0300 Subject: [PATCH 09/22] feat(web): byte-capped streaming reader (truncation terminal) --- web/Cargo.toml | 2 ++ web/src/fetch.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/web/Cargo.toml b/web/Cargo.toml index 4321295ecc..1dec4f9d8f 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -34,6 +34,7 @@ htmd = "0.5" tl = { package = "astral-tl", version = "0.7" } ipnet = "2" base64 = "0.22" +bytes = "1" [dev-dependencies] serde_json = "1" @@ -41,3 +42,4 @@ tempfile = "3" wiremock = "0.6" tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time", "net"] } futures = "0.3" +bytes = "1" diff --git a/web/src/fetch.rs b/web/src/fetch.rs index f152338a0d..11fc56ff63 100644 --- a/web/src/fetch.rs +++ b/web/src/fetch.rs @@ -6,6 +6,7 @@ use std::collections::BTreeMap; use base64::{engine::general_purpose::STANDARD, Engine as _}; +use futures::StreamExt; use crate::config::WebConfig; use crate::schemas::{FetchPayload, ResponseFormat}; @@ -75,6 +76,30 @@ pub fn try_parse_json(text: &str) -> Result { serde_json::from_str(text).map_err(|e| e.to_string()) } +/// Read a byte stream up to `cap`, marking truncation. Once the cap is +/// reached the result is terminal — a subsequent stream error (e.g. reset +/// on drop) is ignored: the cap is the contract. +pub async fn read_capped(mut stream: S, cap: u64) -> (Vec, bool) +where + S: futures::Stream> + Unpin, +{ + let cap = cap as usize; + let mut buf: Vec = Vec::new(); + while let Some(item) = stream.next().await { + let chunk = match item { + Ok(c) => c, + Err(_) => return (buf, false), // pre-cap transport error: surface as non-truncated; caller maps it + }; + if buf.len() + chunk.len() > cap { + let remaining = cap.saturating_sub(buf.len()); + buf.extend_from_slice(&chunk[..remaining]); + return (buf, true); // truncation is terminal; drop the stream + } + buf.extend_from_slice(&chunk); + } + (buf, false) +} + #[cfg(test)] mod tests { use super::*; @@ -143,4 +168,35 @@ mod tests { assert!(try_parse_json("{\"a\":1}").is_ok()); assert!(try_parse_json("not json").is_err()); } + + #[tokio::test] + async fn read_capped_truncates_and_marks() { + let chunks: Vec> = vec![ + Ok(bytes::Bytes::from_static(b"aaaa")), + Ok(bytes::Bytes::from_static(b"bbbb")), + ]; + let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 6).await; + assert_eq!(bytes, b"aaaabb"); + assert!(truncated); + } + + #[tokio::test] + async fn read_capped_full_under_cap() { + let chunks: Vec> = + vec![Ok(bytes::Bytes::from_static(b"hello"))]; + let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 100).await; + assert_eq!(bytes, b"hello"); + assert!(!truncated); + } + + #[tokio::test] + async fn read_capped_truncation_wins_over_later_error() { + // a chunk that overflows the cap, then an error that must be ignored + let chunks: Vec> = vec![ + Ok(bytes::Bytes::from_static(b"aaaaaaaa")), + ]; + let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 4).await; + assert_eq!(bytes, b"aaaa"); + assert!(truncated); + } } From 9317c12191dd68cbc02c0913f84eb1f36ed8a7b1 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:00:04 -0300 Subject: [PATCH 10/22] feat(web): execute_fetch request loop + IP pinning + redirects + page shaping --- web/src/fetch.rs | 357 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 356 insertions(+), 1 deletion(-) diff --git a/web/src/fetch.rs b/web/src/fetch.rs index 11fc56ff63..3b02f89197 100644 --- a/web/src/fetch.rs +++ b/web/src/fetch.rs @@ -4,12 +4,20 @@ //! starts with the pure helpers. use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::time::Duration; use base64::{engine::general_purpose::STANDARD, Engine as _}; use futures::StreamExt; +use serde_json::{json, Value}; use crate::config::WebConfig; -use crate::schemas::{FetchPayload, ResponseFormat}; +use crate::convert::{ + accept_header_for, extract_text, html_to_markdown, is_image_mime, is_viewable_image_mime, + max_tag_depth, ACCEPT_LANGUAGE, BROWSER_USER_AGENT, MAX_NESTING_DEPTH, +}; +use crate::schemas::{FetchPayload, PageFormat, ResponseFormat}; +use crate::ssrf::{check_target, parse_target, SsrfPolicy}; const DENY_ON_REDIRECT: [&str; 3] = ["authorization", "cookie", "proxy-authorization"]; @@ -100,6 +108,353 @@ where (buf, false) } +struct RawResponse { + status: u16, + status_text: String, + headers: BTreeMap, + bytes: Vec, + truncated: bool, +} + +enum Outcome { + Response(RawResponse), + Timeout, + Transport(String), +} + +fn error_envelope(code: &str, message: impl Into) -> Value { + json!({ "ok": false, "error": code, "message": message.into() }) +} + +/// Flatten reqwest headers into a lower-cased string map, joining repeated +/// headers (e.g. set-cookie) with ", ". +fn flatten_headers(h: &reqwest::header::HeaderMap) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for (name, value) in h.iter() { + let k = name.as_str().to_lowercase(); + let v = value.to_str().unwrap_or("").to_string(); + out.entry(k) + .and_modify(|existing| { + existing.push_str(", "); + existing.push_str(&v); + }) + .or_insert(v); + } + out +} + +/// Drop any caller-supplied Host header so reqwest derives it from the URL. +fn without_host(headers: &BTreeMap) -> BTreeMap { + headers + .iter() + .filter(|(k, _)| !k.eq_ignore_ascii_case("host")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +/// One HTTP request, pinned to `address`. SNI/cert/Host stay on the URL +/// hostname; only the TCP target is overridden (DNS-rebinding defense). +async fn perform_request( + target: &crate::ssrf::ParsedTarget, + address: SocketAddr, + method: &str, + headers: &BTreeMap, + body: Option<&str>, + timeout: Duration, + max_bytes: u64, +) -> Outcome { + let client = match reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .resolve(&target.hostname, address) + .pool_max_idle_per_host(0) + .timeout(timeout) + .build() + { + Ok(c) => c, + Err(e) => return Outcome::Transport(e.to_string()), + }; + + let m = reqwest::Method::from_bytes(method.as_bytes()).unwrap_or(reqwest::Method::GET); + let mut req = client.request(m, target.url.clone()); + for (k, v) in without_host(headers) { + req = req.header(k, v); + } + if let Some(b) = body { + if method != "GET" && method != "HEAD" { + req = req.body(b.to_string()); + } + } + + let resp = match req.send().await { + Ok(r) => r, + Err(e) => { + return if e.is_timeout() { + Outcome::Timeout + } else { + Outcome::Transport(e.to_string()) + }; + } + }; + + let status = resp.status().as_u16(); + let status_text = resp.status().canonical_reason().unwrap_or("").to_string(); + let headers = flatten_headers(resp.headers()); + // Box::pin so the `Unpin` bound on read_capped holds regardless of + // reqwest's internal stream type. + let (bytes, truncated) = read_capped(Box::pin(resp.bytes_stream()), max_bytes).await; + Outcome::Response(RawResponse { status, status_text, headers, bytes, truncated }) +} + +pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { + let method = payload.normalized_method(); + let follow_redirects = payload.follow_redirects.unwrap_or(true); + let page_format = payload.format; + let response_format = if page_format.is_some() { + ResponseFormat::Text + } else { + payload.response_format.unwrap_or(ResponseFormat::Text) + }; + let timeout = Duration::from_millis(resolve_timeout(&payload, cfg)); + let max_bytes = resolve_max_bytes(&payload, cfg); + let policy = SsrfPolicy { allow_loopback: cfg.allow_loopback }; + + let mut current_url = payload.url.clone(); + + // Build base headers: page mode injects a browser UA + Accept + Accept-Language + // for headers the caller didn't supply; otherwise the configured UA. + let caller_keys: std::collections::BTreeSet = payload + .headers + .as_ref() + .map(|h| h.keys().map(|k| k.to_lowercase()).collect()) + .unwrap_or_default(); + let browser_ua_injected = page_format.is_some() && !caller_keys.contains("user-agent"); + + let mut base: BTreeMap = BTreeMap::new(); + base.insert( + "user-agent".to_string(), + if browser_ua_injected { BROWSER_USER_AGENT.to_string() } else { cfg.user_agent.clone() }, + ); + if let Some(f) = page_format { + if !caller_keys.contains("accept") { + base.insert("accept".to_string(), accept_header_for(f).to_string()); + } + if !caller_keys.contains("accept-language") { + base.insert("accept-language".to_string(), ACCEPT_LANGUAGE.to_string()); + } + } + if let Some(h) = &payload.headers { + for (k, v) in h { + base.insert(k.clone(), v.clone()); // caller wins + } + } + let (effective_body, mut current_headers) = apply_json_payload(&payload, base); + + let mut redirect_chain: Vec = Vec::new(); + + for hop in 0..=cfg.max_redirects { + let parsed = match parse_target(¤t_url) { + Ok(p) => p, + Err(e) => return error_envelope("invalid_url", e), + }; + let resolved = match check_target(&parsed, &policy).await { + Ok(r) => r, + Err(rej) => return error_envelope(rej.code, rej.message), + }; + let socket = SocketAddr::new(resolved.address, resolved.port); + + let mut outcome = perform_request( + &parsed, socket, &method, ¤t_headers, effective_body.as_deref(), timeout, max_bytes, + ) + .await; + + // Cloudflare cf-mitigated:challenge: retry the SAME hop once with the + // honest UA. Only when we injected the browser UA, GET/HEAD, 403+challenge. + if browser_ua_injected + && (method == "GET" || method == "HEAD") + { + if let Outcome::Response(r) = &outcome { + if r.status == 403 && r.headers.get("cf-mitigated").map(|s| s.as_str()) == Some("challenge") { + let mut retry_headers = current_headers.clone(); + retry_headers.insert("user-agent".to_string(), cfg.user_agent.clone()); + outcome = perform_request( + &parsed, socket, &method, &retry_headers, effective_body.as_deref(), timeout, max_bytes, + ) + .await; + } + } + } + + let resp = match outcome { + Outcome::Timeout => { + return error_envelope( + "timeout", + format!("request timed out after {}ms (raise timeout_ms or shrink response with smaller max_bytes)", timeout.as_millis()), + ); + } + Outcome::Transport(m) => return error_envelope("transport_error", m), + Outcome::Response(r) => r, + }; + + // Redirect handling. + if follow_redirects && (300..400).contains(&resp.status) { + if let Some(location) = resp.headers.get("location") { + let next = match parsed.url.join(location) { + Ok(u) => u, + Err(_) => { + return error_envelope( + "transport_error", + format!("redirect target is not a valid URL: {location}"), + ); + } + }; + if hop == cfg.max_redirects { + return error_envelope( + "too_many_redirects", + format!("exceeded max_redirects ({})", cfg.max_redirects), + ); + } + redirect_chain.push(current_url.clone()); + current_headers = strip_cross_origin_auth(current_headers, &parsed.url, &next); + current_url = next.to_string(); + continue; + } + // 3xx without Location: fall through and return the response. + } + + return shape_response(resp, page_format, response_format, &redirect_chain, cfg); + } + + error_envelope( + "too_many_redirects", + format!("exceeded max_redirects ({})", cfg.max_redirects), + ) +} + +fn shape_response( + resp: RawResponse, + page_format: Option, + response_format: ResponseFormat, + redirect_chain: &[String], + cfg: &WebConfig, +) -> Value { + if let Some(pf) = page_format { + let content_type = resp.headers.get("content-type").cloned().unwrap_or_default(); + let mime = content_type.split(';').next().unwrap_or("").trim().to_lowercase(); + + if is_image_mime(&mime) { + let viewable = is_viewable_image_mime(&mime) + && (200..300).contains(&resp.status) + && !resp.truncated + && !resp.bytes.is_empty(); + if viewable { + let mut details = json!({ + "ok": true, + "status": resp.status, + "status_text": resp.status_text, + "content_type": mime, + "bytes": resp.bytes.len(), + }); + if !redirect_chain.is_empty() { + details["redirect_chain"] = json!(redirect_chain); + } + return json!({ + "content": [ + { "type": "image", "mime": mime, "data": STANDARD.encode(&resp.bytes) }, + { "type": "text", "text": format!("Image fetched ({mime}, {} bytes)", resp.bytes.len()) }, + ], + "details": details, + }); + } + // Non-viewable image → base64 envelope. + let mut out = json!({ + "ok": true, + "status": resp.status, + "status_text": resp.status_text, + "headers": resp.headers, + "body": STANDARD.encode(&resp.bytes), + "response_format": "base64", + "bytes_truncated": resp.truncated, + "content_type": mime, + }); + if !redirect_chain.is_empty() { + out["redirect_chain"] = json!(redirect_chain); + } + return out; + } + + let raw = String::from_utf8_lossy(&resp.bytes).into_owned(); + let is_html = mime == "text/html" || mime == "application/xhtml+xml"; + let mut body = raw.clone(); + let mut transformed: Option<&str> = None; + let within_cap = (resp.bytes.len() as u64) <= cfg.max_transform_bytes; + let depth_ok = within_cap && max_tag_depth(&raw) <= MAX_NESTING_DEPTH; + + if is_html && within_cap && depth_ok && matches!(pf, PageFormat::Markdown | PageFormat::Text) { + let converted = match pf { + PageFormat::Markdown => html_to_markdown(&raw), + PageFormat::Text => Ok(extract_text(&raw)), + PageFormat::Html => unreachable!(), + }; + if let Ok(text) = converted { + body = text; + transformed = Some(match pf { + PageFormat::Markdown => "markdown", + PageFormat::Text => "text", + PageFormat::Html => "html", + }); + if resp.truncated { + body.push_str("\n\n[Content truncated at max_bytes — the page continues beyond this point]"); + } + } + // On transform failure: keep raw body, leave transformed unset. + } + + let mut out = json!({ + "ok": true, + "status": resp.status, + "status_text": resp.status_text, + "headers": resp.headers, + "body": body, + "response_format": "text", + "bytes_truncated": resp.truncated, + "content_type": mime, + }); + if let Some(t) = transformed { + out["transformed"] = json!(t); + } + if !redirect_chain.is_empty() { + out["redirect_chain"] = json!(redirect_chain); + } + return out; + } + + // Raw (non-page) mode. + let body = encode_body(&resp.bytes, response_format); + let mut out = json!({ + "ok": true, + "status": resp.status, + "status_text": resp.status_text, + "headers": resp.headers, + "body": body, + "response_format": match response_format { + ResponseFormat::Text => "text", + ResponseFormat::Base64 => "base64", + ResponseFormat::Json => "json", + }, + "bytes_truncated": resp.truncated, + }); + if !redirect_chain.is_empty() { + out["redirect_chain"] = json!(redirect_chain); + } + if matches!(response_format, ResponseFormat::Json) && !resp.truncated { + match try_parse_json(&body) { + Ok(v) => out["json"] = v, + Err(e) => out["parse_error"] = json!(e), + } + } + out +} + #[cfg(test)] mod tests { use super::*; From bc29d4ae8749e8306c9c6ccce7f207e02569391b Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:05:13 -0300 Subject: [PATCH 11/22] feat(web): register web::fetch with Value input, validation, panic isolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add functions/fetch.rs handler (raw Value input, invalid_payload on parse/validate failure, catch_unwind → transport_error envelope) and functions/mod.rs with register_all. Wire pub mod functions into lib.rs. --- web/src/functions/fetch.rs | 74 ++++++++++++++++++++++++++++++++++++++ web/src/functions/mod.rs | 13 +++++++ web/src/lib.rs | 1 + 3 files changed, 88 insertions(+) create mode 100644 web/src/functions/fetch.rs create mode 100644 web/src/functions/mod.rs diff --git a/web/src/functions/fetch.rs b/web/src/functions/fetch.rs new file mode 100644 index 0000000000..20ed783267 --- /dev/null +++ b/web/src/functions/fetch.rs @@ -0,0 +1,74 @@ +//! `web::fetch` registration + handler. Input is a raw `Value` (parsed +//! internally) so a malformed payload returns `invalid_payload` instead of +//! an SDK-level error. The whole handler is wrapped in catch_unwind so a +//! panic becomes a transport_error envelope rather than a dropped response. + +use std::panic::AssertUnwindSafe; +use std::sync::Arc; + +use futures::FutureExt; +use iii_sdk::{IIIError, RegisterFunction, III}; +use serde_json::{json, Value}; + +use crate::config::{SharedConfig, WebConfig}; +use crate::fetch::execute_fetch; +use crate::schemas::{request_schema, FetchPayload, TOOL_DESCRIPTION}; + +/// Parse + validate + execute. Always returns an envelope (never errors). +pub async fn handle(payload: Value, cfg: &WebConfig) -> Value { + let parsed: FetchPayload = match serde_json::from_value(payload) { + Ok(p) => p, + Err(e) => return json!({ "ok": false, "error": "invalid_payload", "message": e.to_string() }), + }; + if let Err(msg) = parsed.validate() { + return json!({ "ok": false, "error": "invalid_payload", "message": msg }); + } + execute_fetch(parsed, cfg).await +} + +pub fn register(iii: &Arc, shared: &SharedConfig) { + let shared = shared.clone(); + iii.register_function( + "web::fetch", + RegisterFunction::new_async(move |payload: Value| { + let cfg = shared.load_full(); // single snapshot per request + async move { + // Wrap the whole handler: a panic must become an envelope, + // not a dropped response (the SDK runs us in a bare spawn + // with no catch_unwind, so a panic would hang the caller). + let result = AssertUnwindSafe(handle(payload, &cfg)).catch_unwind().await; + let envelope = result.unwrap_or_else(|_| { + json!({ "ok": false, "error": "transport_error", "message": "internal error" }) + }); + Ok::(envelope) + } + }) + .description(TOOL_DESCRIPTION) + .request_format(request_schema()), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::WebConfig; + + #[tokio::test] + async fn invalid_payload_missing_url() { + let out = handle(serde_json::json!({}), &WebConfig::default()).await; + assert_eq!(out["ok"], false); + assert_eq!(out["error"], "invalid_payload"); + } + + #[tokio::test] + async fn invalid_payload_bad_method() { + let out = handle(serde_json::json!({ "url": "https://x.test/", "method": "FETCH" }), &WebConfig::default()).await; + assert_eq!(out["error"], "invalid_payload"); + } + + #[tokio::test] + async fn invalid_url_scheme() { + let out = handle(serde_json::json!({ "url": "ftp://x.test/" }), &WebConfig::default()).await; + assert_eq!(out["error"], "invalid_url"); + } +} diff --git a/web/src/functions/mod.rs b/web/src/functions/mod.rs new file mode 100644 index 0000000000..4248124377 --- /dev/null +++ b/web/src/functions/mod.rs @@ -0,0 +1,13 @@ +//! Function registration for the web worker. + +use std::sync::Arc; + +use iii_sdk::III; + +use crate::config::SharedConfig; + +pub mod fetch; + +pub fn register_all(iii: &Arc, shared: &SharedConfig) { + fetch::register(iii, shared); +} diff --git a/web/src/lib.rs b/web/src/lib.rs index f4d09d054b..e4c2b9bc2b 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -7,6 +7,7 @@ pub mod config; pub mod convert; pub mod fetch; +pub mod functions; pub mod manifest; pub mod schemas; pub mod ssrf; From b1f73bae3ee541d38d57e5288b8b043b355a8557 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:08:34 -0300 Subject: [PATCH 12/22] feat(web): configuration-worker integration + main boot wiring --- web/src/configuration.rs | 134 +++++++++++++++++++++++++++++++++++++++ web/src/lib.rs | 1 + web/src/main.rs | 75 +++++++++++++++++++--- 3 files changed, 202 insertions(+), 8 deletions(-) create mode 100644 web/src/configuration.rs diff --git a/web/src/configuration.rs b/web/src/configuration.rs new file mode 100644 index 0000000000..158d941110 --- /dev/null +++ b/web/src/configuration.rs @@ -0,0 +1,134 @@ +//! Integration with the `configuration` worker: register a JSON Schema + +//! seed at boot, read the authoritative (env-expanded) value, and bind a +//! `configuration` trigger so `configuration:updated` re-fetches and applies +//! the change. All WebConfig fields hot-reload (no topology partition). + +use std::time::Duration; + +use iii_sdk::{IIIError, RegisterFunction, RegisterTriggerInput, TriggerRequest, III}; +use serde_json::{json, Value}; + +use crate::config::{SharedConfig, WebConfig}; + +pub const CONFIG_ID: &str = "web"; +const CONFIG_FN_ID: &str = "web::on-config-change"; +const CONFIG_TIMEOUT_MS: u64 = 5_000; +const CONFIG_RETRIES: u32 = 3; + +#[derive(Clone)] +pub struct SharedState { + pub config: SharedConfig, +} + +pub async fn register_config(iii: &III, seed: Option<&WebConfig>) -> Result<(), String> { + let mut payload = json!({ + "id": CONFIG_ID, + "name": "web", + "description": "Timeouts, byte caps, user-agent, and loopback policy for the web::fetch worker.", + "schema": WebConfig::json_schema(), + }); + if let Some(seed) = seed { + payload["initial_value"] = seed.to_json(); + } else if should_seed_default(iii).await? { + payload["initial_value"] = WebConfig::default().to_json(); + } + trigger_with_retry(iii, "configuration::register", payload).await?; + Ok(()) +} + +pub async fn fetch_config(iii: &III) -> Result { + match try_get_value(iii).await? { + Some(v) if !v.is_null() => WebConfig::from_json(&v), + _ => { + tracing::info!("no configuration value found; using built-in defaults"); + Ok(WebConfig::default()) + } + } +} + +async fn should_seed_default(iii: &III) -> Result { + match try_get_value(iii).await? { + None => Ok(true), + Some(v) if v.is_null() => Ok(true), + Some(_) => Ok(false), + } +} + +async fn try_get_value(iii: &III) -> Result, String> { + match trigger_with_retry(iii, "configuration::get", json!({ "id": CONFIG_ID })).await { + Ok(resp) => Ok(resp.get("value").cloned()), + Err(e) if e.contains("NOT_FOUND") => Ok(None), + Err(e) => Err(e), + } +} + +pub async fn apply_config(state: &SharedState, cfg: WebConfig) { + state.config.store(std::sync::Arc::new(cfg)); +} + +#[derive(Debug, Default, serde::Deserialize, schemars::JsonSchema)] +struct OnConfigChangeRequest {} + +#[derive(Debug, serde::Serialize, schemars::JsonSchema)] +struct OnConfigChangeResponse { + ok: bool, +} + +pub fn register_config_trigger(iii: &III, state: SharedState) -> Result<(), IIIError> { + let st = state.clone(); + let engine = iii.clone(); + iii.register_function( + CONFIG_FN_ID, + RegisterFunction::new_async(move |_req: OnConfigChangeRequest| { + let st = st.clone(); + let engine = engine.clone(); + async move { + on_config_change(&engine, &st).await; + Ok::(OnConfigChangeResponse { ok: true }) + } + }) + .description("Internal: reload web settings from the authoritative configuration on change."), + ); + + iii.register_trigger(RegisterTriggerInput { + trigger_type: "configuration".to_string(), + function_id: CONFIG_FN_ID.to_string(), + config: json!({ "configuration_id": CONFIG_ID, "event_types": ["configuration:updated"] }), + metadata: None, + })?; + Ok(()) +} + +async fn on_config_change(iii: &III, state: &SharedState) { + match fetch_config(iii).await { + Ok(cfg) => { + apply_config(state, cfg).await; + tracing::info!("web configuration reloaded"); + } + Err(e) => tracing::error!(error = %e, "config-change: keeping previous config"), + } +} + +async fn trigger_with_retry(iii: &III, function_id: &str, payload: Value) -> Result { + let mut last_err = String::new(); + for attempt in 1..=CONFIG_RETRIES { + match iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload: payload.clone(), + action: None, + timeout_ms: Some(CONFIG_TIMEOUT_MS), + }) + .await + { + Ok(v) => return Ok(v), + Err(e) => { + last_err = e.to_string(); + if attempt < CONFIG_RETRIES { + tokio::time::sleep(Duration::from_millis(250 * u64::from(attempt))).await; + } + } + } + } + Err(format!("{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}")) +} diff --git a/web/src/lib.rs b/web/src/lib.rs index e4c2b9bc2b..a7625fb782 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -5,6 +5,7 @@ //! Standalone Rust port of the former `harness/src/web` TypeScript worker. pub mod config; +pub mod configuration; pub mod convert; pub mod fetch; pub mod functions; diff --git a/web/src/main.rs b/web/src/main.rs index f69a994596..7f16efd36a 100644 --- a/web/src/main.rs +++ b/web/src/main.rs @@ -1,13 +1,20 @@ -//! `iii-web` binary entry. Full boot sequence is completed in Task 11; -//! this version supports `--manifest` so the crate is exercisable early. +//! `iii-web` binary entry: connect, register configuration + fetch the +//! authoritative value, register `web::fetch`, bind the config-change +//! trigger for hot-reload, then sleep until Ctrl+C. +use std::sync::Arc; + +use anyhow::{Context, Result}; use clap::Parser; +use iii_sdk::{register_worker, InitOptions, WorkerMetadata}; -use web::manifest; +use web::config::WebConfig; +use web::{configuration, functions, manifest}; #[derive(Parser, Debug)] #[command(name = "iii-web", about = "Outbound HTTP client on the iii bus (web::fetch).")] struct Cli { + /// Optional YAML seed used to populate `initial_value` on first registration. #[arg(long)] config: Option, #[arg(long, default_value = "ws://127.0.0.1:49134")] @@ -17,7 +24,7 @@ struct Cli { } #[tokio::main] -async fn main() -> anyhow::Result<()> { +async fn main() -> Result<()> { tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() @@ -27,12 +34,64 @@ async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); if cli.manifest { - let m = manifest::build_manifest(); - println!("{}", serde_json::to_string_pretty(&m).unwrap()); + println!("{}", serde_json::to_string_pretty(&manifest::build_manifest()).unwrap()); return Ok(()); } - // Full boot wiring is implemented in Task 11. - let _ = (cli.config, cli.url); + let iii = Arc::new(register_worker( + &cli.url, + InitOptions { + metadata: Some(WorkerMetadata { + runtime: "rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + name: "web".to_string(), + os: std::env::consts::OS.to_string(), + description: Some("Outbound HTTP client on the iii bus (web::fetch).".to_string()), + pid: Some(std::process::id()), + telemetry: None, + ..WorkerMetadata::default() + }), + ..InitOptions::default() + }, + )); + + // Optional YAML seed → initial_value on first registration. + let seed = cli.config.as_deref().and_then(|path| { + match std::fs::read_to_string(path).ok().and_then(|s| serde_yaml::from_str::(&s).ok()) { + Some(v) => match WebConfig::from_json(&v) { + Ok(cfg) => Some(cfg), + Err(e) => { + tracing::warn!(path, error = %e, "failed to parse seed config; ignoring"); + None + } + }, + None => None, + } + }); + + configuration::register_config(&iii, seed.as_ref()) + .await + .map_err(anyhow::Error::msg) + .context("registering web configuration schema")?; + let cfg = configuration::fetch_config(&iii) + .await + .map_err(anyhow::Error::msg) + .context("loading web configuration")?; + tracing::info!( + max_timeout_ms = cfg.max_timeout_ms, + max_response_bytes = cfg.max_response_bytes, + allow_loopback = cfg.allow_loopback, + "loaded web configuration" + ); + + let shared = cfg.into_shared(); + functions::register_all(&iii, &shared); + configuration::register_config_trigger(&iii, configuration::SharedState { config: shared.clone() }) + .context("registering configuration change trigger")?; + + tracing::info!("web ready: web::fetch + configuration hot-reload"); + tokio::signal::ctrl_c().await?; + tracing::info!("web shutting down"); + iii.shutdown_async().await; Ok(()) } From 80bc58807d2297df8cdf1f90b8f56dead33bf18f Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:11:17 -0300 Subject: [PATCH 13/22] chore(web): log config-RPC retries and seed-read I/O errors --- web/src/configuration.rs | 6 ++++++ web/src/main.rs | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/web/src/configuration.rs b/web/src/configuration.rs index 158d941110..6508005a27 100644 --- a/web/src/configuration.rs +++ b/web/src/configuration.rs @@ -125,6 +125,12 @@ async fn trigger_with_retry(iii: &III, function_id: &str, payload: Value) -> Res Err(e) => { last_err = e.to_string(); if attempt < CONFIG_RETRIES { + tracing::warn!( + function_id, + attempt, + error = %last_err, + "configuration RPC failed; retrying" + ); tokio::time::sleep(Duration::from_millis(250 * u64::from(attempt))).await; } } diff --git a/web/src/main.rs b/web/src/main.rs index 7f16efd36a..09de7b5822 100644 --- a/web/src/main.rs +++ b/web/src/main.rs @@ -57,7 +57,14 @@ async fn main() -> Result<()> { // Optional YAML seed → initial_value on first registration. let seed = cli.config.as_deref().and_then(|path| { - match std::fs::read_to_string(path).ok().and_then(|s| serde_yaml::from_str::(&s).ok()) { + let contents = match std::fs::read_to_string(path) { + Ok(s) => s, + Err(e) => { + tracing::warn!(path, error = %e, "failed to read seed config; ignoring"); + return None; + } + }; + match serde_yaml::from_str::(&contents).ok() { Some(v) => match WebConfig::from_json(&v) { Ok(cfg) => Some(cfg), Err(e) => { From 27b8eff331224218eb4b383b25382d12a367157d Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:15:57 -0300 Subject: [PATCH 14/22] test(web): wiremock integration tests for fetch behaviors + image envelope --- web/tests/integration.rs | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 web/tests/integration.rs diff --git a/web/tests/integration.rs b/web/tests/integration.rs new file mode 100644 index 0000000000..314ec9e013 --- /dev/null +++ b/web/tests/integration.rs @@ -0,0 +1,144 @@ +use serde_json::json; +use web::config::WebConfig; +use web::fetch::execute_fetch; +use web::schemas::FetchPayload; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn payload(j: serde_json::Value) -> FetchPayload { + serde_json::from_value(j).unwrap() +} + +#[tokio::test] +async fn basic_get_returns_body_and_status() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/hi")) + .respond_with(ResponseTemplate::new(200).set_body_string("hello")) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/hi", server.uri()) })), &WebConfig::default()).await; + assert_eq!(out["ok"], true); + assert_eq!(out["status"], 200); + assert_eq!(out["body"], "hello"); +} + +#[tokio::test] +async fn json_mode_autoparses_when_not_truncated() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/j")) + .respond_with(ResponseTemplate::new(200).set_body_string("{\"a\":1}")) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/j", server.uri()), "response_format": "json" })), &WebConfig::default()).await; + assert_eq!(out["json"]["a"], 1); + assert_eq!(out["body"], "{\"a\":1}"); // raw text still present +} + +#[tokio::test] +async fn json_mode_sets_parse_error_on_bad_json() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/bad")) + .respond_with(ResponseTemplate::new(200).set_body_string("nope")) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/bad", server.uri()), "response_format": "json" })), &WebConfig::default()).await; + assert!(out.get("json").is_none()); + assert!(out["parse_error"].is_string()); +} + +#[tokio::test] +async fn byte_cap_truncates() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/big")) + .respond_with(ResponseTemplate::new(200).set_body_string("x".repeat(1000))) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/big", server.uri()), "max_bytes": 10 })), &WebConfig::default()).await; + assert_eq!(out["bytes_truncated"], true); + assert_eq!(out["body"].as_str().unwrap().len(), 10); +} + +#[tokio::test] +async fn redirect_followed_and_chain_recorded() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/from")) + .respond_with(ResponseTemplate::new(302).insert_header("location", "/to")) + .mount(&server).await; + Mock::given(method("GET")).and(path("/to")) + .respond_with(ResponseTemplate::new(200).set_body_string("arrived")) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/from", server.uri()) })), &WebConfig::default()).await; + assert_eq!(out["body"], "arrived"); + assert!(out["redirect_chain"].as_array().unwrap().len() == 1); +} + +#[tokio::test] +async fn too_many_redirects() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/loop")) + .respond_with(ResponseTemplate::new(302).insert_header("location", "/loop")) + .mount(&server).await; + + let cfg = WebConfig { max_redirects: 2, ..WebConfig::default() }; + let out = execute_fetch(payload(json!({ "url": format!("{}/loop", server.uri()) })), &cfg).await; + assert_eq!(out["error"], "too_many_redirects"); +} + +#[tokio::test] +async fn three_xx_without_location_returns_as_success() { + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/nolctn")) + .respond_with(ResponseTemplate::new(304)) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/nolctn", server.uri()) })), &WebConfig::default()).await; + assert_eq!(out["ok"], true); + assert_eq!(out["status"], 304); +} + +#[tokio::test] +async fn page_mode_markdown_transforms_html() { + let server = MockServer::start().await; + // Use set_body_raw so wiremock sends the correct Content-Type header. + // set_body_string always sets mime to "text/plain" and that field wins + // over any insert_header("content-type") call in generate_response(). + Mock::given(method("GET")).and(path("/page")) + .respond_with(ResponseTemplate::new(200) + .set_body_raw("

Title

Body

", "text/html; charset=utf-8")) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/page", server.uri()), "format": "markdown" })), &WebConfig::default()).await; + assert_eq!(out["transformed"], "markdown"); + assert!(out["body"].as_str().unwrap().contains("Title")); + assert_eq!(out["content_type"], "text/html"); +} + +#[tokio::test] +async fn image_envelope_deserializes_to_content_blocks() { + // 1x1 transparent PNG + let png: &[u8] = &[ + 0x89,0x50,0x4E,0x47,0x0D,0x0A,0x1A,0x0A,0x00,0x00,0x00,0x0D,0x49,0x48,0x44,0x52, + 0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x08,0x06,0x00,0x00,0x00,0x1F,0x15,0xC4, + 0x89,0x00,0x00,0x00,0x0A,0x49,0x44,0x41,0x54,0x78,0x9C,0x63,0x00,0x01,0x00,0x00, + 0x05,0x00,0x01,0x0D,0x0A,0x2D,0xB4,0x00,0x00,0x00,0x00,0x49,0x45,0x4E,0x44,0xAE,0x42,0x60,0x82, + ]; + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/img")) + .respond_with(ResponseTemplate::new(200).insert_header("content-type", "image/png").set_body_bytes(png.to_vec())) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/img", server.uri()), "format": "markdown" })), &WebConfig::default()).await; + // shape: { content: [image, text], details } + let content = out["content"].as_array().expect("content array"); + assert_eq!(content[0]["type"], "image"); + assert_eq!(content[0]["mime"], "image/png"); + assert_eq!(content[1]["type"], "text"); + assert_eq!(out["details"]["ok"], true); +} + +#[tokio::test] +async fn blocked_host_for_metadata_ip() { + let out = execute_fetch(payload(json!({ "url": "http://169.254.169.254/latest/meta-data/" })), &WebConfig::default()).await; + assert_eq!(out["error"], "blocked_host"); +} From 22b528146a5bad498d7596a983da15394f70c971 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:19:30 -0300 Subject: [PATCH 15/22] test(web): adversarial SSRF blocking + IP-pinning tests --- web/tests/ssrf_rebinding.rs | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 web/tests/ssrf_rebinding.rs diff --git a/web/tests/ssrf_rebinding.rs b/web/tests/ssrf_rebinding.rs new file mode 100644 index 0000000000..76efc03e72 --- /dev/null +++ b/web/tests/ssrf_rebinding.rs @@ -0,0 +1,45 @@ +use serde_json::json; +use web::config::WebConfig; +use web::fetch::execute_fetch; +use web::schemas::FetchPayload; +use web::ssrf::{check_target, parse_target, SsrfPolicy}; + +fn payload(j: serde_json::Value) -> FetchPayload { + serde_json::from_value(j).unwrap() +} + +#[tokio::test] +async fn refuses_when_resolution_is_private() { + // A hostname that resolves to loopback; with allow_loopback=false it must be refused. + let t = parse_target("http://localhost/").unwrap(); + let rej = check_target(&t, &SsrfPolicy { allow_loopback: false }).await.unwrap_err(); + assert_eq!(rej.code, "blocked_host"); +} + +#[tokio::test] +async fn private_literal_blocked_end_to_end() { + for url in [ + "http://10.0.0.1/", + "http://192.168.1.1/", + "http://[::1]/", + "http://[::ffff:a9fe:a9fe]/", + ] { + let out = execute_fetch(payload(json!({ "url": url })), &WebConfig { allow_loopback: false, ..WebConfig::default() }).await; + assert_eq!(out["error"], "blocked_host", "{url} must be blocked"); + } +} + +#[tokio::test] +async fn pinning_dials_validated_ip() { + // wiremock binds to 127.0.0.1; with allow_loopback=true the request must + // succeed AND be served by the pinned loopback address (not re-resolved). + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("GET")).and(path("/ok")) + .respond_with(ResponseTemplate::new(200).set_body_string("pinned")) + .mount(&server).await; + + let out = execute_fetch(payload(json!({ "url": format!("{}/ok", server.uri()) })), &WebConfig::default()).await; + assert_eq!(out["body"], "pinned"); +} From 860e494b4f3b73632ec365a0cd386acdd9b82420 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 12:23:22 -0300 Subject: [PATCH 16/22] =?UTF-8?q?chore(web):=20finalize=20migration=20?= =?UTF-8?q?=E2=80=94=20cleanup=20comments/imports,=20clippy=20clean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ssrf.rs: fix stale comment (IPv6 bracket stripping happens in parse_target, not host_str); move mid-file SocketAddr import to top-level use std::net::{..., SocketAddr} - schemas.rs: fix module doc "(for request_format)" → "(for request_schema)" - harness/src/web removed (untracked TS leftover, no git change needed) - cargo build + test (54 passed) + clippy -D warnings: all green --- web/src/schemas.rs | 2 +- web/src/ssrf.rs | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/web/src/schemas.rs b/web/src/schemas.rs index 9413351fd1..e663053457 100644 --- a/web/src/schemas.rs +++ b/web/src/schemas.rs @@ -1,5 +1,5 @@ //! Wire schema + tool description for `web::fetch`. `FetchPayload` derives -//! `JsonSchema` (for `request_format`) and `Deserialize`. The handler takes +//! `JsonSchema` (for `request_schema`) and `Deserialize`. The handler takes //! a raw `Value` and parses this so a bad payload returns `invalid_payload` //! rather than an SDK-level error (the SDK deserializes typed inputs before //! the handler runs). diff --git a/web/src/ssrf.rs b/web/src/ssrf.rs index 2badef0250..c78be7f966 100644 --- a/web/src/ssrf.rs +++ b/web/src/ssrf.rs @@ -4,7 +4,7 @@ //! socket is pinned to the validated IP (see fetch.rs) to defeat DNS //! rebinding (TOCTOU between check and connect). -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::OnceLock; use ipnet::{Ipv4Net, Ipv6Net}; @@ -121,8 +121,6 @@ pub fn check_ip(addr: IpAddr, policy: &SsrfPolicy) -> Option<&'static str> { } } -use std::net::SocketAddr; - #[derive(Debug, Clone)] pub struct ResolvedTarget { pub address: IpAddr, @@ -154,8 +152,8 @@ pub async fn check_target( ) -> Result { let host = &target.hostname; - // Literal IP: skip DNS, check directly. `host_str()` already strips - // the brackets around a v6 literal. + // Literal IP short-circuit: skip DNS. Brackets were already stripped + // from IPv6 literals in parse_target. if let Ok(ip) = host.parse::() { if let Some(label) = check_ip(ip, policy) { let hint = if label == "loopback" { loopback_hint(policy) } else { "" }; From 684ac4514fc7fc8b0e0d276f94195118e1cb46de Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 13:28:17 -0300 Subject: [PATCH 17/22] refactor(web): run HTML transform on spawn_blocking Move the CPU-bound depth-check and HTML-to-markdown/text conversion into tokio::task::spawn_blocking so large transforms never block an async runtime worker thread. shape_response is now async; the call site in execute_fetch is updated to .await it. Behavior is identical: depth guard, transform failure, and closure panics all fall back to raw body with transformed unset; truncation marker only appended on successful transform. --- web/src/fetch.rs | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/web/src/fetch.rs b/web/src/fetch.rs index 3b02f89197..c6cd72997a 100644 --- a/web/src/fetch.rs +++ b/web/src/fetch.rs @@ -321,7 +321,7 @@ pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { // 3xx without Location: fall through and return the response. } - return shape_response(resp, page_format, response_format, &redirect_chain, cfg); + return shape_response(resp, page_format, response_format, &redirect_chain, cfg).await; } error_envelope( @@ -330,7 +330,7 @@ pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { ) } -fn shape_response( +async fn shape_response( resp: RawResponse, page_format: Option, response_format: ResponseFormat, @@ -387,15 +387,23 @@ fn shape_response( let mut body = raw.clone(); let mut transformed: Option<&str> = None; let within_cap = (resp.bytes.len() as u64) <= cfg.max_transform_bytes; - let depth_ok = within_cap && max_tag_depth(&raw) <= MAX_NESTING_DEPTH; - if is_html && within_cap && depth_ok && matches!(pf, PageFormat::Markdown | PageFormat::Text) { - let converted = match pf { - PageFormat::Markdown => html_to_markdown(&raw), - PageFormat::Text => Ok(extract_text(&raw)), - PageFormat::Html => unreachable!(), - }; - if let Ok(text) = converted { + if is_html && within_cap && matches!(pf, PageFormat::Markdown | PageFormat::Text) { + let raw_clone = raw.clone(); + let maybe_text: Option = tokio::task::spawn_blocking(move || { + if max_tag_depth(&raw_clone) > MAX_NESTING_DEPTH { + return None; + } + match pf { + PageFormat::Markdown => html_to_markdown(&raw_clone).ok(), + PageFormat::Text => Some(extract_text(&raw_clone)), + PageFormat::Html => None, + } + }) + .await + .unwrap_or(None); + + if let Some(text) = maybe_text { body = text; transformed = Some(match pf { PageFormat::Markdown => "markdown", @@ -406,7 +414,7 @@ fn shape_response( body.push_str("\n\n[Content truncated at max_bytes — the page continues beyond this point]"); } } - // On transform failure: keep raw body, leave transformed unset. + // On transform failure / over-depth / panic: keep raw body, leave transformed unset. } let mut out = json!({ From 1d425a2fbeca18aa2b6fc81e099748633dc71574 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Fri, 19 Jun 2026 14:24:14 -0300 Subject: [PATCH 18/22] style(web): apply cargo fmt CI runs `cargo fmt --all -- --check`, which was failing on rustfmt drift across the crate (notably the test files). Normalize formatting. Claude-Session: https://claude.ai/code/session_018YytfzKvsyBiTHi6b2u1rn --- web/src/config.rs | 6 +- web/src/configuration.rs | 8 +- web/src/convert.rs | 37 ++++++-- web/src/fetch.rs | 119 ++++++++++++++++++++------ web/src/functions/fetch.rs | 16 +++- web/src/main.rs | 19 ++++- web/src/manifest.rs | 5 +- web/src/schemas.rs | 29 +++++-- web/src/ssrf.rs | 96 ++++++++++++++++----- web/tests/integration.rs | 164 +++++++++++++++++++++++++----------- web/tests/ssrf_rebinding.rs | 30 +++++-- 11 files changed, 409 insertions(+), 120 deletions(-) diff --git a/web/src/config.rs b/web/src/config.rs index a7d0303ee1..171dda9b2e 100644 --- a/web/src/config.rs +++ b/web/src/config.rs @@ -88,7 +88,11 @@ mod tests { #[test] fn json_roundtrip() { - let c = WebConfig { max_redirects: 2, allow_loopback: false, ..WebConfig::default() }; + let c = WebConfig { + max_redirects: 2, + allow_loopback: false, + ..WebConfig::default() + }; let back = WebConfig::from_json(&c.to_json()).unwrap(); assert_eq!(back.max_redirects, 2); assert!(!back.allow_loopback); diff --git a/web/src/configuration.rs b/web/src/configuration.rs index 6508005a27..dcc487c92e 100644 --- a/web/src/configuration.rs +++ b/web/src/configuration.rs @@ -87,7 +87,9 @@ pub fn register_config_trigger(iii: &III, state: SharedState) -> Result<(), IIIE Ok::(OnConfigChangeResponse { ok: true }) } }) - .description("Internal: reload web settings from the authoritative configuration on change."), + .description( + "Internal: reload web settings from the authoritative configuration on change.", + ), ); iii.register_trigger(RegisterTriggerInput { @@ -136,5 +138,7 @@ async fn trigger_with_retry(iii: &III, function_id: &str, payload: Value) -> Res } } } - Err(format!("{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}")) + Err(format!( + "{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}" + )) } diff --git a/web/src/convert.rs b/web/src/convert.rs index ca95dda07a..a582c92332 100644 --- a/web/src/convert.rs +++ b/web/src/convert.rs @@ -16,8 +16,25 @@ pub const MAX_NESTING_DEPTH: usize = 200; const SKIP_TAGS: [&str; 6] = ["script", "style", "noscript", "iframe", "object", "embed"]; const BLOCK_TAGS: [&str; 19] = [ - "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr", "section", "article", "header", - "footer", "blockquote", "pre", "table", "ul", "ol", + "p", + "div", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "li", + "tr", + "section", + "article", + "header", + "footer", + "blockquote", + "pre", + "table", + "ul", + "ol", ]; pub fn accept_header_for(format: PageFormat) -> &'static str { @@ -37,7 +54,10 @@ pub fn is_image_mime(mime: &str) -> bool { } pub fn is_viewable_image_mime(mime: &str) -> bool { - matches!(mime, "image/jpeg" | "image/png" | "image/gif" | "image/webp") + matches!( + mime, + "image/jpeg" | "image/png" | "image/gif" | "image/webp" + ) } /// Maximum tag-nesting depth, computed iteratively (no recursion, cannot @@ -79,13 +99,20 @@ pub fn extract_text(html: &str) -> String { }; let parser = dom.parser(); let mut out = String::new(); - let mut stack: Vec = dom.children().iter().rev().map(|h| Work::Enter(*h)).collect(); + let mut stack: Vec = dom + .children() + .iter() + .rev() + .map(|h| Work::Enter(*h)) + .collect(); while let Some(item) = stack.pop() { match item { Work::CloseBlock => out.push('\n'), Work::Enter(handle) => { - let Some(node) = handle.get(parser) else { continue }; + let Some(node) = handle.get(parser) else { + continue; + }; match node { tl::Node::Raw(bytes) => out.push_str(&bytes.as_utf8_str()), tl::Node::Tag(tag) => { diff --git a/web/src/fetch.rs b/web/src/fetch.rs index c6cd72997a..5d7bb43cbb 100644 --- a/web/src/fetch.rs +++ b/web/src/fetch.rs @@ -23,7 +23,9 @@ const DENY_ON_REDIRECT: [&str; 3] = ["authorization", "cookie", "proxy-authoriza /// Default to `default_timeout_ms`, never exceed `max_timeout_ms`. pub fn resolve_timeout(p: &FetchPayload, cfg: &WebConfig) -> u64 { - p.timeout_ms.unwrap_or(cfg.default_timeout_ms).min(cfg.max_timeout_ms) + p.timeout_ms + .unwrap_or(cfg.default_timeout_ms) + .min(cfg.max_timeout_ms) } /// Raw fetches default to the hard ceiling; page mode defaults to the @@ -46,7 +48,9 @@ pub fn apply_json_payload( let Some(json) = &p.json else { return (p.body.clone(), headers); }; - let has_ct = headers.keys().any(|k| k.eq_ignore_ascii_case("content-type")); + let has_ct = headers + .keys() + .any(|k| k.eq_ignore_ascii_case("content-type")); if !has_ct { headers.insert("content-type".to_string(), "application/json".to_string()); } @@ -202,7 +206,13 @@ async fn perform_request( // Box::pin so the `Unpin` bound on read_capped holds regardless of // reqwest's internal stream type. let (bytes, truncated) = read_capped(Box::pin(resp.bytes_stream()), max_bytes).await; - Outcome::Response(RawResponse { status, status_text, headers, bytes, truncated }) + Outcome::Response(RawResponse { + status, + status_text, + headers, + bytes, + truncated, + }) } pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { @@ -216,7 +226,9 @@ pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { }; let timeout = Duration::from_millis(resolve_timeout(&payload, cfg)); let max_bytes = resolve_max_bytes(&payload, cfg); - let policy = SsrfPolicy { allow_loopback: cfg.allow_loopback }; + let policy = SsrfPolicy { + allow_loopback: cfg.allow_loopback, + }; let mut current_url = payload.url.clone(); @@ -232,7 +244,11 @@ pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { let mut base: BTreeMap = BTreeMap::new(); base.insert( "user-agent".to_string(), - if browser_ua_injected { BROWSER_USER_AGENT.to_string() } else { cfg.user_agent.clone() }, + if browser_ua_injected { + BROWSER_USER_AGENT.to_string() + } else { + cfg.user_agent.clone() + }, ); if let Some(f) = page_format { if !caller_keys.contains("accept") { @@ -263,21 +279,33 @@ pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { let socket = SocketAddr::new(resolved.address, resolved.port); let mut outcome = perform_request( - &parsed, socket, &method, ¤t_headers, effective_body.as_deref(), timeout, max_bytes, + &parsed, + socket, + &method, + ¤t_headers, + effective_body.as_deref(), + timeout, + max_bytes, ) .await; // Cloudflare cf-mitigated:challenge: retry the SAME hop once with the // honest UA. Only when we injected the browser UA, GET/HEAD, 403+challenge. - if browser_ua_injected - && (method == "GET" || method == "HEAD") - { + if browser_ua_injected && (method == "GET" || method == "HEAD") { if let Outcome::Response(r) = &outcome { - if r.status == 403 && r.headers.get("cf-mitigated").map(|s| s.as_str()) == Some("challenge") { + if r.status == 403 + && r.headers.get("cf-mitigated").map(|s| s.as_str()) == Some("challenge") + { let mut retry_headers = current_headers.clone(); retry_headers.insert("user-agent".to_string(), cfg.user_agent.clone()); outcome = perform_request( - &parsed, socket, &method, &retry_headers, effective_body.as_deref(), timeout, max_bytes, + &parsed, + socket, + &method, + &retry_headers, + effective_body.as_deref(), + timeout, + max_bytes, ) .await; } @@ -338,8 +366,17 @@ async fn shape_response( cfg: &WebConfig, ) -> Value { if let Some(pf) = page_format { - let content_type = resp.headers.get("content-type").cloned().unwrap_or_default(); - let mime = content_type.split(';').next().unwrap_or("").trim().to_lowercase(); + let content_type = resp + .headers + .get("content-type") + .cloned() + .unwrap_or_default(); + let mime = content_type + .split(';') + .next() + .unwrap_or("") + .trim() + .to_lowercase(); if is_image_mime(&mime) { let viewable = is_viewable_image_mime(&mime) @@ -477,17 +514,38 @@ mod tests { #[test] fn timeout_defaults_and_caps() { let cfg = WebConfig::default(); - assert_eq!(resolve_timeout(&p(serde_json::json!({"url":"x"})), &cfg), 30_000); - assert_eq!(resolve_timeout(&p(serde_json::json!({"url":"x","timeout_ms":5000})), &cfg), 5_000); - assert_eq!(resolve_timeout(&p(serde_json::json!({"url":"x","timeout_ms":999999})), &cfg), 120_000); + assert_eq!( + resolve_timeout(&p(serde_json::json!({"url":"x"})), &cfg), + 30_000 + ); + assert_eq!( + resolve_timeout(&p(serde_json::json!({"url":"x","timeout_ms":5000})), &cfg), + 5_000 + ); + assert_eq!( + resolve_timeout(&p(serde_json::json!({"url":"x","timeout_ms":999999})), &cfg), + 120_000 + ); } #[test] fn max_bytes_defaults_by_page_mode() { let cfg = WebConfig::default(); - assert_eq!(resolve_max_bytes(&p(serde_json::json!({"url":"x"})), &cfg), 5 * 1024 * 1024); - assert_eq!(resolve_max_bytes(&p(serde_json::json!({"url":"x","format":"markdown"})), &cfg), 256 * 1024); - assert_eq!(resolve_max_bytes(&p(serde_json::json!({"url":"x","max_bytes":999999999u64})), &cfg), 5 * 1024 * 1024); + assert_eq!( + resolve_max_bytes(&p(serde_json::json!({"url":"x"})), &cfg), + 5 * 1024 * 1024 + ); + assert_eq!( + resolve_max_bytes(&p(serde_json::json!({"url":"x","format":"markdown"})), &cfg), + 256 * 1024 + ); + assert_eq!( + resolve_max_bytes( + &p(serde_json::json!({"url":"x","max_bytes":999999999u64})), + &cfg + ), + 5 * 1024 * 1024 + ); } #[test] @@ -512,11 +570,23 @@ mod tests { let mut h = BTreeMap::new(); h.insert("authorization".to_string(), "Bearer t".to_string()); h.insert("cookie".to_string(), "s=1".to_string()); - let same = strip_cross_origin_auth(h.clone(), &"https://a.test/".parse().unwrap(), &"https://a.test/2".parse().unwrap()); + let same = strip_cross_origin_auth( + h.clone(), + &"https://a.test/".parse().unwrap(), + &"https://a.test/2".parse().unwrap(), + ); assert!(same.contains_key("authorization")); - let cross = strip_cross_origin_auth(h.clone(), &"https://a.test/".parse().unwrap(), &"https://b.test/".parse().unwrap()); + let cross = strip_cross_origin_auth( + h.clone(), + &"https://a.test/".parse().unwrap(), + &"https://b.test/".parse().unwrap(), + ); assert!(!cross.contains_key("authorization")); - let downgrade = strip_cross_origin_auth(h, &"https://a.test/".parse().unwrap(), &"http://a.test/".parse().unwrap()); + let downgrade = strip_cross_origin_auth( + h, + &"https://a.test/".parse().unwrap(), + &"http://a.test/".parse().unwrap(), + ); assert!(!downgrade.contains_key("cookie")); } @@ -555,9 +625,8 @@ mod tests { #[tokio::test] async fn read_capped_truncation_wins_over_later_error() { // a chunk that overflows the cap, then an error that must be ignored - let chunks: Vec> = vec![ - Ok(bytes::Bytes::from_static(b"aaaaaaaa")), - ]; + let chunks: Vec> = + vec![Ok(bytes::Bytes::from_static(b"aaaaaaaa"))]; let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 4).await; assert_eq!(bytes, b"aaaa"); assert!(truncated); diff --git a/web/src/functions/fetch.rs b/web/src/functions/fetch.rs index 20ed783267..9f0aab1ed8 100644 --- a/web/src/functions/fetch.rs +++ b/web/src/functions/fetch.rs @@ -18,7 +18,9 @@ use crate::schemas::{request_schema, FetchPayload, TOOL_DESCRIPTION}; pub async fn handle(payload: Value, cfg: &WebConfig) -> Value { let parsed: FetchPayload = match serde_json::from_value(payload) { Ok(p) => p, - Err(e) => return json!({ "ok": false, "error": "invalid_payload", "message": e.to_string() }), + Err(e) => { + return json!({ "ok": false, "error": "invalid_payload", "message": e.to_string() }) + } }; if let Err(msg) = parsed.validate() { return json!({ "ok": false, "error": "invalid_payload", "message": msg }); @@ -62,13 +64,21 @@ mod tests { #[tokio::test] async fn invalid_payload_bad_method() { - let out = handle(serde_json::json!({ "url": "https://x.test/", "method": "FETCH" }), &WebConfig::default()).await; + let out = handle( + serde_json::json!({ "url": "https://x.test/", "method": "FETCH" }), + &WebConfig::default(), + ) + .await; assert_eq!(out["error"], "invalid_payload"); } #[tokio::test] async fn invalid_url_scheme() { - let out = handle(serde_json::json!({ "url": "ftp://x.test/" }), &WebConfig::default()).await; + let out = handle( + serde_json::json!({ "url": "ftp://x.test/" }), + &WebConfig::default(), + ) + .await; assert_eq!(out["error"], "invalid_url"); } } diff --git a/web/src/main.rs b/web/src/main.rs index 09de7b5822..8a22054ff7 100644 --- a/web/src/main.rs +++ b/web/src/main.rs @@ -12,7 +12,10 @@ use web::config::WebConfig; use web::{configuration, functions, manifest}; #[derive(Parser, Debug)] -#[command(name = "iii-web", about = "Outbound HTTP client on the iii bus (web::fetch).")] +#[command( + name = "iii-web", + about = "Outbound HTTP client on the iii bus (web::fetch)." +)] struct Cli { /// Optional YAML seed used to populate `initial_value` on first registration. #[arg(long)] @@ -34,7 +37,10 @@ async fn main() -> Result<()> { let cli = Cli::parse(); if cli.manifest { - println!("{}", serde_json::to_string_pretty(&manifest::build_manifest()).unwrap()); + println!( + "{}", + serde_json::to_string_pretty(&manifest::build_manifest()).unwrap() + ); return Ok(()); } @@ -93,8 +99,13 @@ async fn main() -> Result<()> { let shared = cfg.into_shared(); functions::register_all(&iii, &shared); - configuration::register_config_trigger(&iii, configuration::SharedState { config: shared.clone() }) - .context("registering configuration change trigger")?; + configuration::register_config_trigger( + &iii, + configuration::SharedState { + config: shared.clone(), + }, + ) + .context("registering configuration change trigger")?; tracing::info!("web ready: web::fetch + configuration hot-reload"); tokio::signal::ctrl_c().await?; diff --git a/web/src/manifest.rs b/web/src/manifest.rs index 58c845d56e..3d13ec205d 100644 --- a/web/src/manifest.rs +++ b/web/src/manifest.rs @@ -34,7 +34,10 @@ mod tests { assert_eq!(v["name"], env!("CARGO_PKG_NAME")); assert_eq!(v["version"], env!("CARGO_PKG_VERSION")); assert!(v["description"].as_str().is_some_and(|s| !s.is_empty())); - assert_eq!(v["default_config"]["user_agent"], "iii-harness/0.1 (+web::fetch)"); + assert_eq!( + v["default_config"]["user_agent"], + "iii-harness/0.1 (+web::fetch)" + ); assert!(!v["supported_targets"].as_array().unwrap().is_empty()); } } diff --git a/web/src/schemas.rs b/web/src/schemas.rs index e663053457..89054b10fa 100644 --- a/web/src/schemas.rs +++ b/web/src/schemas.rs @@ -95,7 +95,8 @@ impl FetchPayload { } pub fn request_schema() -> serde_json::Value { - serde_json::to_value(schemars::schema_for!(FetchPayload)).expect("FetchPayload schema serializes") + serde_json::to_value(schemars::schema_for!(FetchPayload)) + .expect("FetchPayload schema serializes") } pub const TOOL_DESCRIPTION: &str = concat!( @@ -139,21 +140,37 @@ mod tests { #[test] fn rejects_unknown_method() { - let p = payload(serde_json::json!({ "url": "https://x.test/", "method": "FETCH" })).unwrap(); + let p = + payload(serde_json::json!({ "url": "https://x.test/", "method": "FETCH" })).unwrap(); assert!(p.validate().is_err()); } #[test] fn rejects_empty_url_and_nonpositive_caps() { - assert!(payload(serde_json::json!({ "url": "" })).unwrap().validate().is_err()); - assert!(payload(serde_json::json!({ "url": "https://x/", "timeout_ms": 0 })).unwrap().validate().is_err()); - assert!(payload(serde_json::json!({ "url": "https://x/", "max_bytes": 0 })).unwrap().validate().is_err()); + assert!(payload(serde_json::json!({ "url": "" })) + .unwrap() + .validate() + .is_err()); + assert!( + payload(serde_json::json!({ "url": "https://x/", "timeout_ms": 0 })) + .unwrap() + .validate() + .is_err() + ); + assert!( + payload(serde_json::json!({ "url": "https://x/", "max_bytes": 0 })) + .unwrap() + .validate() + .is_err() + ); } #[test] fn rejects_unknown_enum_values() { assert!(payload(serde_json::json!({ "url": "https://x/", "format": "pdf" })).is_err()); - assert!(payload(serde_json::json!({ "url": "https://x/", "response_format": "xml" })).is_err()); + assert!( + payload(serde_json::json!({ "url": "https://x/", "response_format": "xml" })).is_err() + ); } #[test] diff --git a/web/src/ssrf.rs b/web/src/ssrf.rs index c78be7f966..5943044f12 100644 --- a/web/src/ssrf.rs +++ b/web/src/ssrf.rs @@ -23,7 +23,8 @@ pub struct ParsedTarget { /// Parse + validate scheme and host shape. No DNS. pub fn parse_target(raw: &str) -> Result { - let url = reqwest::Url::parse(raw).map_err(|_| "url is not a valid absolute URL".to_string())?; + let url = + reqwest::Url::parse(raw).map_err(|_| "url is not a valid absolute URL".to_string())?; let scheme = url.scheme(); if scheme != "http" && scheme != "https" { return Err(format!( @@ -42,7 +43,11 @@ pub fn parse_target(raw: &str) -> Result { let port = url .port_or_known_default() .ok_or_else(|| "invalid port".to_string())?; - Ok(ParsedTarget { url, hostname, port }) + Ok(ParsedTarget { + url, + hostname, + port, + }) } static V4_BLOCKLIST: OnceLock> = OnceLock::new(); @@ -100,7 +105,11 @@ fn check_ipv6(addr: Ipv6Addr, policy: &SsrfPolicy) -> Option<&'static str> { return check_ipv4(v4, policy); } if addr == Ipv6Addr::LOCALHOST { - return if policy.allow_loopback { None } else { Some("loopback") }; + return if policy.allow_loopback { + None + } else { + Some("loopback") + }; } if addr == Ipv6Addr::UNSPECIFIED { return Some("unspecified"); @@ -156,25 +165,33 @@ pub async fn check_target( // from IPv6 literals in parse_target. if let Ok(ip) = host.parse::() { if let Some(label) = check_ip(ip, policy) { - let hint = if label == "loopback" { loopback_hint(policy) } else { "" }; + let hint = if label == "loopback" { + loopback_hint(policy) + } else { + "" + }; return Err(SsrfReject { code: "blocked_host", message: format!("address {host} is in {label}{hint}"), }); } - return Ok(ResolvedTarget { address: ip, hostname: host.clone(), port: target.port }); + return Ok(ResolvedTarget { + address: ip, + hostname: host.clone(), + port: target.port, + }); } - let resolved: Vec = match tokio::net::lookup_host((host.as_str(), target.port)).await - { - Ok(it) => it.collect(), - Err(e) => { - return Err(SsrfReject { - code: "blocked_host", - message: format!("dns lookup failed for {host}: {e}"), - }); - } - }; + let resolved: Vec = + match tokio::net::lookup_host((host.as_str(), target.port)).await { + Ok(it) => it.collect(), + Err(e) => { + return Err(SsrfReject { + code: "blocked_host", + message: format!("dns lookup failed for {host}: {e}"), + }); + } + }; if resolved.is_empty() { return Err(SsrfReject { code: "blocked_host", @@ -184,7 +201,11 @@ pub async fn check_target( for sa in &resolved { if let Some(label) = check_ip(sa.ip(), policy) { - let hint = if label == "loopback" { loopback_hint(policy) } else { "" }; + let hint = if label == "loopback" { + loopback_hint(policy) + } else { + "" + }; return Err(SsrfReject { code: "blocked_host", message: format!( @@ -208,12 +229,24 @@ mod tests { use std::net::IpAddr; fn blocked(ip: &str, allow_loopback: bool) -> Option<&'static str> { - check_ip(ip.parse::().unwrap(), &SsrfPolicy { allow_loopback }) + check_ip( + ip.parse::().unwrap(), + &SsrfPolicy { allow_loopback }, + ) } #[test] fn blocks_private_v4_ranges() { - for ip in ["10.0.0.1", "172.16.5.5", "192.168.1.1", "169.254.169.254", "100.64.0.1", "0.0.0.0", "224.0.0.1", "240.0.0.1"] { + for ip in [ + "10.0.0.1", + "172.16.5.5", + "192.168.1.1", + "169.254.169.254", + "100.64.0.1", + "0.0.0.0", + "224.0.0.1", + "240.0.0.1", + ] { assert!(blocked(ip, true).is_some(), "{ip} should be blocked"); } } @@ -267,7 +300,14 @@ mod tests { #[tokio::test] async fn check_target_literal_v4_public_ok() { let t = parse_target("http://1.1.1.1/").unwrap(); - let r = check_target(&t, &SsrfPolicy { allow_loopback: true }).await.unwrap(); + let r = check_target( + &t, + &SsrfPolicy { + allow_loopback: true, + }, + ) + .await + .unwrap(); assert_eq!(r.address, "1.1.1.1".parse::().unwrap()); assert_eq!(r.port, 80); } @@ -275,14 +315,28 @@ mod tests { #[tokio::test] async fn check_target_literal_v4_private_blocked() { let t = parse_target("http://169.254.169.254/").unwrap(); - let rej = check_target(&t, &SsrfPolicy { allow_loopback: true }).await.unwrap_err(); + let rej = check_target( + &t, + &SsrfPolicy { + allow_loopback: true, + }, + ) + .await + .unwrap_err(); assert_eq!(rej.code, "blocked_host"); } #[tokio::test] async fn check_target_literal_v6_bracket_stripped_and_blocked() { let t = parse_target("http://[::1]/").unwrap(); - let rej = check_target(&t, &SsrfPolicy { allow_loopback: false }).await.unwrap_err(); + let rej = check_target( + &t, + &SsrfPolicy { + allow_loopback: false, + }, + ) + .await + .unwrap_err(); assert_eq!(rej.code, "blocked_host"); assert!(rej.message.contains("allow_loopback")); } diff --git a/web/tests/integration.rs b/web/tests/integration.rs index 314ec9e013..f814c3c30e 100644 --- a/web/tests/integration.rs +++ b/web/tests/integration.rs @@ -12,11 +12,17 @@ fn payload(j: serde_json::Value) -> FetchPayload { #[tokio::test] async fn basic_get_returns_body_and_status() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/hi")) + Mock::given(method("GET")) + .and(path("/hi")) .respond_with(ResponseTemplate::new(200).set_body_string("hello")) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/hi", server.uri()) })), &WebConfig::default()).await; + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/hi", server.uri()) })), + &WebConfig::default(), + ) + .await; assert_eq!(out["ok"], true); assert_eq!(out["status"], 200); assert_eq!(out["body"], "hello"); @@ -25,11 +31,17 @@ async fn basic_get_returns_body_and_status() { #[tokio::test] async fn json_mode_autoparses_when_not_truncated() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/j")) + Mock::given(method("GET")) + .and(path("/j")) .respond_with(ResponseTemplate::new(200).set_body_string("{\"a\":1}")) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/j", server.uri()), "response_format": "json" })), &WebConfig::default()).await; + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/j", server.uri()), "response_format": "json" })), + &WebConfig::default(), + ) + .await; assert_eq!(out["json"]["a"], 1); assert_eq!(out["body"], "{\"a\":1}"); // raw text still present } @@ -37,11 +49,17 @@ async fn json_mode_autoparses_when_not_truncated() { #[tokio::test] async fn json_mode_sets_parse_error_on_bad_json() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/bad")) + Mock::given(method("GET")) + .and(path("/bad")) .respond_with(ResponseTemplate::new(200).set_body_string("nope")) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/bad", server.uri()), "response_format": "json" })), &WebConfig::default()).await; + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/bad", server.uri()), "response_format": "json" })), + &WebConfig::default(), + ) + .await; assert!(out.get("json").is_none()); assert!(out["parse_error"].is_string()); } @@ -49,11 +67,17 @@ async fn json_mode_sets_parse_error_on_bad_json() { #[tokio::test] async fn byte_cap_truncates() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/big")) + Mock::given(method("GET")) + .and(path("/big")) .respond_with(ResponseTemplate::new(200).set_body_string("x".repeat(1000))) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/big", server.uri()), "max_bytes": 10 })), &WebConfig::default()).await; + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/big", server.uri()), "max_bytes": 10 })), + &WebConfig::default(), + ) + .await; assert_eq!(out["bytes_truncated"], true); assert_eq!(out["body"].as_str().unwrap().len(), 10); } @@ -61,14 +85,22 @@ async fn byte_cap_truncates() { #[tokio::test] async fn redirect_followed_and_chain_recorded() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/from")) + Mock::given(method("GET")) + .and(path("/from")) .respond_with(ResponseTemplate::new(302).insert_header("location", "/to")) - .mount(&server).await; - Mock::given(method("GET")).and(path("/to")) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/to")) .respond_with(ResponseTemplate::new(200).set_body_string("arrived")) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/from", server.uri()) })), &WebConfig::default()).await; + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/from", server.uri()) })), + &WebConfig::default(), + ) + .await; assert_eq!(out["body"], "arrived"); assert!(out["redirect_chain"].as_array().unwrap().len() == 1); } @@ -76,23 +108,38 @@ async fn redirect_followed_and_chain_recorded() { #[tokio::test] async fn too_many_redirects() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/loop")) + Mock::given(method("GET")) + .and(path("/loop")) .respond_with(ResponseTemplate::new(302).insert_header("location", "/loop")) - .mount(&server).await; - - let cfg = WebConfig { max_redirects: 2, ..WebConfig::default() }; - let out = execute_fetch(payload(json!({ "url": format!("{}/loop", server.uri()) })), &cfg).await; + .mount(&server) + .await; + + let cfg = WebConfig { + max_redirects: 2, + ..WebConfig::default() + }; + let out = execute_fetch( + payload(json!({ "url": format!("{}/loop", server.uri()) })), + &cfg, + ) + .await; assert_eq!(out["error"], "too_many_redirects"); } #[tokio::test] async fn three_xx_without_location_returns_as_success() { let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/nolctn")) + Mock::given(method("GET")) + .and(path("/nolctn")) .respond_with(ResponseTemplate::new(304)) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/nolctn", server.uri()) })), &WebConfig::default()).await; + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/nolctn", server.uri()) })), + &WebConfig::default(), + ) + .await; assert_eq!(out["ok"], true); assert_eq!(out["status"], 304); } @@ -103,12 +150,20 @@ async fn page_mode_markdown_transforms_html() { // Use set_body_raw so wiremock sends the correct Content-Type header. // set_body_string always sets mime to "text/plain" and that field wins // over any insert_header("content-type") call in generate_response(). - Mock::given(method("GET")).and(path("/page")) - .respond_with(ResponseTemplate::new(200) - .set_body_raw("

Title

Body

", "text/html; charset=utf-8")) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/page", server.uri()), "format": "markdown" })), &WebConfig::default()).await; + Mock::given(method("GET")) + .and(path("/page")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw("

Title

Body

", "text/html; charset=utf-8"), + ) + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/page", server.uri()), "format": "markdown" })), + &WebConfig::default(), + ) + .await; assert_eq!(out["transformed"], "markdown"); assert!(out["body"].as_str().unwrap().contains("Title")); assert_eq!(out["content_type"], "text/html"); @@ -118,17 +173,28 @@ async fn page_mode_markdown_transforms_html() { async fn image_envelope_deserializes_to_content_blocks() { // 1x1 transparent PNG let png: &[u8] = &[ - 0x89,0x50,0x4E,0x47,0x0D,0x0A,0x1A,0x0A,0x00,0x00,0x00,0x0D,0x49,0x48,0x44,0x52, - 0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x08,0x06,0x00,0x00,0x00,0x1F,0x15,0xC4, - 0x89,0x00,0x00,0x00,0x0A,0x49,0x44,0x41,0x54,0x78,0x9C,0x63,0x00,0x01,0x00,0x00, - 0x05,0x00,0x01,0x0D,0x0A,0x2D,0xB4,0x00,0x00,0x00,0x00,0x49,0x45,0x4E,0x44,0xAE,0x42,0x60,0x82, + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, + 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, + 0x15, 0xC4, 0x89, 0x00, 0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0x00, + 0x01, 0x00, 0x00, 0x05, 0x00, 0x01, 0x0D, 0x0A, 0x2D, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, + 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82, ]; let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/img")) - .respond_with(ResponseTemplate::new(200).insert_header("content-type", "image/png").set_body_bytes(png.to_vec())) - .mount(&server).await; - - let out = execute_fetch(payload(json!({ "url": format!("{}/img", server.uri()), "format": "markdown" })), &WebConfig::default()).await; + Mock::given(method("GET")) + .and(path("/img")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "image/png") + .set_body_bytes(png.to_vec()), + ) + .mount(&server) + .await; + + let out = execute_fetch( + payload(json!({ "url": format!("{}/img", server.uri()), "format": "markdown" })), + &WebConfig::default(), + ) + .await; // shape: { content: [image, text], details } let content = out["content"].as_array().expect("content array"); assert_eq!(content[0]["type"], "image"); @@ -139,6 +205,10 @@ async fn image_envelope_deserializes_to_content_blocks() { #[tokio::test] async fn blocked_host_for_metadata_ip() { - let out = execute_fetch(payload(json!({ "url": "http://169.254.169.254/latest/meta-data/" })), &WebConfig::default()).await; + let out = execute_fetch( + payload(json!({ "url": "http://169.254.169.254/latest/meta-data/" })), + &WebConfig::default(), + ) + .await; assert_eq!(out["error"], "blocked_host"); } diff --git a/web/tests/ssrf_rebinding.rs b/web/tests/ssrf_rebinding.rs index 76efc03e72..93849b04bd 100644 --- a/web/tests/ssrf_rebinding.rs +++ b/web/tests/ssrf_rebinding.rs @@ -12,7 +12,14 @@ fn payload(j: serde_json::Value) -> FetchPayload { async fn refuses_when_resolution_is_private() { // A hostname that resolves to loopback; with allow_loopback=false it must be refused. let t = parse_target("http://localhost/").unwrap(); - let rej = check_target(&t, &SsrfPolicy { allow_loopback: false }).await.unwrap_err(); + let rej = check_target( + &t, + &SsrfPolicy { + allow_loopback: false, + }, + ) + .await + .unwrap_err(); assert_eq!(rej.code, "blocked_host"); } @@ -24,7 +31,14 @@ async fn private_literal_blocked_end_to_end() { "http://[::1]/", "http://[::ffff:a9fe:a9fe]/", ] { - let out = execute_fetch(payload(json!({ "url": url })), &WebConfig { allow_loopback: false, ..WebConfig::default() }).await; + let out = execute_fetch( + payload(json!({ "url": url })), + &WebConfig { + allow_loopback: false, + ..WebConfig::default() + }, + ) + .await; assert_eq!(out["error"], "blocked_host", "{url} must be blocked"); } } @@ -36,10 +50,16 @@ async fn pinning_dials_validated_ip() { use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let server = MockServer::start().await; - Mock::given(method("GET")).and(path("/ok")) + Mock::given(method("GET")) + .and(path("/ok")) .respond_with(ResponseTemplate::new(200).set_body_string("pinned")) - .mount(&server).await; + .mount(&server) + .await; - let out = execute_fetch(payload(json!({ "url": format!("{}/ok", server.uri()) })), &WebConfig::default()).await; + let out = execute_fetch( + payload(json!({ "url": format!("{}/ok", server.uri()) })), + &WebConfig::default(), + ) + .await; assert_eq!(out["body"], "pinned"); } From 283b6c35b3e2519c8066343858c8cdae76ae4308 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Fri, 19 Jun 2026 14:24:14 -0300 Subject: [PATCH 19/22] docs(web): add worker README The per-worker PR validation requires a non-empty README.md. Document the web::fetch request/response envelope, configuration keys, page-reading mode, the SSRF guard, and local dev commands. Claude-Session: https://claude.ai/code/session_018YytfzKvsyBiTHi6b2u1rn --- web/README.md | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 web/README.md diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000000..e5912f2563 --- /dev/null +++ b/web/README.md @@ -0,0 +1,201 @@ +# web + +Outbound HTTP(S) client on the [iii engine](https://github.com/iii-hq/iii) +bus. A single bus function, `web::fetch`, performs a network request and +returns a parsed `{ ok, status, headers, body }` envelope with enforced +size/timeout caps and server-side SSRF protection. It is the network +counterpart to the `shell` worker: use `web::fetch` instead of +`shell::exec` with `curl`/`wget` for any HTTP request. + +The worker is a faithful Rust port of the original TypeScript `web::fetch` +implementation — same request fields and success/error/image envelopes, so +existing callers and the harness consumer are unaffected. + +## Table of contents + +1. [Install](#install) +2. [Configuration](#configuration) +3. [The `web::fetch` function](#the-webfetch-function) +4. [Reading the result](#reading-the-result) +5. [Page-reading mode](#page-reading-mode) +6. [SSRF guard](#ssrf-guard) +7. [Local development & testing](#local-development--testing) + +--- + +## Install + +```bash +iii worker add web +``` + +`iii worker add` fetches the binary (`iii-web`), writes a config block into +`~/.iii/config.yaml`, and the engine starts the worker on the next +`iii start`. + +--- + +## Configuration + +The worker reads its operational ceilings from the `configuration` worker +under the `web` key. It registers a schema + seed on boot, fetches the +authoritative value, and hot-reloads on change. All keys are optional and +fall back to the defaults below. + +```yaml +web: + default_timeout_ms: 30000 # per-request timeout when caller omits timeout_ms + max_timeout_ms: 120000 # ceiling; caller timeout_ms is clamped DOWN to this + default_response_bytes: 262144 # default body cap in page-reading mode (256 KiB) + max_response_bytes: 5242880 # absolute body ceiling (5 MiB); caller max_bytes clamped to this + max_transform_bytes: 1048576 # HTML→markdown/text runs only on bodies ≤ this (1 MiB) + max_redirects: 5 # hops before too_many_redirects + user_agent: 'iii-harness/0.1 (+web::fetch)' + allow_loopback: true # set false in prod so 127.0.0.0/8, ::1, localhost are blocked +``` + +A copy of these defaults lives in [`config.yaml.example`](./config.yaml.example). + +--- + +## The `web::fetch` function + +Minimal call — a URL is the only required field; everything else has a +default: + +```jsonc +// agent_trigger { function: "web::fetch", payload: { "url": "https://api.github.com/zen" } } +// → { "ok": true, "status": 200, "body": "Design for failure.", ... } +``` + +### Request fields + +| Field | Default | Notes | +|---|---|---| +| `url` *(required)* | — | absolute `http(s)://` | +| `method` | `GET` | case-insensitive; GET HEAD POST PUT PATCH DELETE OPTIONS | +| `headers` | `{}` | object; keys as written | +| `json` | — | structured payload; auto-stringified + sets `content-type: application/json`. **Wins over `body`.** | +| `body` | — | raw string body; ignored on GET/HEAD | +| `response_format` | `"text"` | `"text"` \| `"base64"` (binary) \| `"json"` (also parses into `json`) | +| `format` | — | page-reading mode: `"markdown"` \| `"text"` \| `"html"` (see below) | +| `timeout_ms` | `default_timeout_ms` | clamped DOWN to `max_timeout_ms` | +| `max_bytes` | `max_response_bytes` (`default_response_bytes` in `format` mode) | over-cap body is truncated, not errored | +| `follow_redirects` | `true` | each hop re-checked against the SSRF blocklist | + +### Response + +**Success** (`ok: true`) — returned for *any* completed response, 2xx through 5xx: + +```jsonc +{ + "ok": true, + "status": 200, + "status_text": "OK", + "headers": { "content-type": "application/json", ... }, // keys LOWER-CASED + "body": "", + "json": { ... }, // only when response_format="json" and parse succeeded + "parse_error": "…", // only when response_format="json" and JSON.parse failed + "response_format": "json", + "bytes_truncated": false, // true when body hit max_bytes (NOT an error) + "redirect_chain": ["https://…/a"], // omitted when no redirects + "content_type": "text/html", // page-reading mode only + "transformed": "markdown" // only when an HTML transform actually ran +} +``` + +**Failure** (`ok: false`) — the fetch never produced a response (no `status`): + +```jsonc +{ "ok": false, "error": "blocked_host", "message": "…" } +``` + +| `error` | Cause | Fix | +|---|---|---| +| `invalid_payload` | Payload failed schema (bad `method`, wrong types) | Correct the named fields | +| `invalid_url` | `url` isn't a parseable absolute `http(s)://` URL | Pass a full absolute URL incl. scheme | +| `blocked_host` | Target resolves to a private / link-local / cloud-metadata IP | Don't target internal/metadata hosts; for dev loopback set `allow_loopback` | +| `timeout` | Slower than `timeout_ms` | Raise `timeout_ms` (up to ceiling) or shrink work via `max_bytes` | +| `too_many_redirects` | More than `max_redirects` hops | Use the final URL, or `follow_redirects: false` and read the `location` header | +| `transport_error` | Connection refused/reset, TLS/DNS failure, or unparseable redirect `Location` | Check host/port/cert; retry if transient | + +There is **no `too_large` error** — oversize responses come back `ok: true` +with `bytes_truncated: true`. + +--- + +## Reading the result + +**`ok` tells you if the request COMPLETED, not whether the server was happy.** + +- A 404 or 500 is a **successful fetch** → `ok: true`, `status: 404`. Branch on `status` for HTTP outcomes. +- `ok: false` means the request never produced a response. Branch on `error`. + +```jsonc +if (!r.ok) → fetch failed; look at r.error +else if (r.status>=400) → server returned an HTTP error; look at r.status / r.body +else → success; use r.body or r.json +``` + +--- + +## Page-reading mode + +`response_format` (transport encoding) and `format` (page-reading) are +orthogonal — set one, not both. When `format` is set, `response_format` is +ignored. + +- **`format: "markdown"`** — `text/html` → Markdown (the right default for reading pages; far fewer tokens than raw HTML). +- **`format: "text"`** — `text/html` → plain text. +- **`format: "html"`** — raw HTML. + +In page-reading mode the request goes out with a browser User-Agent + +format-matched `Accept` header, and retries once with the honest UA on a +Cloudflare `cf-mitigated: challenge` response. The HTML→markdown/text +transform runs only on bodies ≤ `max_transform_bytes`; larger pages come +back raw with `transformed` unset. + +**Images:** a viewable `image/*` response (jpeg/png/gif/webp, 2xx, +complete, non-empty) returns the actual image plus a one-line text summary +instead of the JSON envelope. Anything else falls back to the normal +envelope with `response_format: "base64"`. + +--- + +## SSRF guard + +DNS is resolved **once**, every resolved address is checked, then the +request is dialed at the **validated IP** (pinned `lookup` + TLS +`servername`) — so the IP that passed the check is the IP connected to (no +DNS-rebind window). On each 3xx the `Location` is re-resolved and +re-validated before following, and `Authorization` / `Cookie` / +`Proxy-Authorization` are stripped when the redirect changes host or +downgrades `https → http`. + +Blocked **unconditionally** → `error: "blocked_host"`: + +- Private RFC1918 (`10/8`, `172.16/12`, `192.168/16`) +- Link-local incl. **cloud metadata** `169.254.169.254` +- IPv6 ULA (`fc00::/7`), link-local (`fe80::/10`), multicast +- `::ffff:`-mapped IPv4 in both dotted (`::ffff:169.254.169.254`) and hex (`::ffff:a9fe:a9fe`) forms + +Loopback (`127.0.0.0/8`, `::1`, `localhost`) is **allowed by default** for +dev convenience; operators set `web.allow_loopback: false` for prod, after +which loopback returns `blocked_host`. + +--- + +## Local development & testing + +```bash +cargo build # build the iii-web binary +cargo test # unit + wiremock integration + adversarial SSRF tests +cargo clippy -- -D warnings # lints +cargo fmt --check # formatting +``` + +The agent-facing authoring guide lives in +[`skills/index.md`](./skills/index.md) (served at runtime via +`directory::skills::get`). For the exact, authoritative field types, call +`engine::functions::info { function_id: "web::fetch" }` — the live schema +wins if it ever disagrees with this document. From c3ff5dce4c27dee2e46de44b9fd21bd6c1781d9d Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Fri, 19 Jun 2026 14:24:14 -0300 Subject: [PATCH 20/22] chore(web): track Cargo.lock Every sibling Rust worker commits its Cargo.lock; the web crate was missing it. Add it for reproducible builds and convention parity. Claude-Session: https://claude.ai/code/session_018YytfzKvsyBiTHi6b2u1rn --- web/Cargo.lock | 2702 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2702 insertions(+) create mode 100644 web/Cargo.lock diff --git a/web/Cargo.lock b/web/Cargo.lock new file mode 100644 index 0000000000..8a86aae0ec --- /dev/null +++ b/web/Cargo.lock @@ -0,0 +1,2702 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "astral-tl" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3" +dependencies = [ + "memchr", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + +[[package]] +name = "htmd" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eee9b00ee2e599b4f86507157e3db786e7a3319fc225f0e9584151dbea2291d" +dependencies = [ + "html5ever", + "markup5ever_rcdom", + "phf", +] + +[[package]] +name = "html5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2" +dependencies = [ + "log", + "markup5ever", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "iii-observability" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71a3002534a53d85c86e4be167cf7ae8c1de809ab3130ecfcb2d85eb4afd1272" +dependencies = [ + "futures-util", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "reqwest", + "serde_json", + "sysinfo", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "iii-sdk" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8490f2ad470d54cf7e0bc2f4105aca71bdb4c24752fcb406183f24b8dd328f80" +dependencies = [ + "async-trait", + "futures-util", + "hostname", + "iii-observability", + "reqwest", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "markup5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.38.0+unofficial" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "333171ccdf66e915257740d44e38ea5b1b19ce7b45d33cc35cb6f118fbd981ff" +dependencies = [ + "html5ever", + "markup5ever", + "tendril", + "xml5ever", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tendril" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24" +dependencies = [ + "new_debug_unreachable", + "utf-8", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web" +version = "1.0.0" +dependencies = [ + "anyhow", + "arc-swap", + "astral-tl", + "async-trait", + "base64", + "bytes", + "clap", + "futures", + "htmd", + "iii-sdk", + "ipnet", + "reqwest", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "tempfile", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web_atoms" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "075474b12bcb3d2e3d4546580e9de478eeeead668a1761e2a8860c836b7ef297" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xml5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dc9559429edf0cd3f327cc0afd9d6b36fa8cec6d93107b7fbe64f806b5f2d9" +dependencies = [ + "log", + "markup5ever", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" From 4a54a0fced1157f8eac48a23a0c90d24a80ef1a2 Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 14:54:21 -0300 Subject: [PATCH 21/22] fix(web): address CodeRabbit review on PR #295 - read_capped: surface a pre-cap stream error as a transport_error instead of returning ok:true with a partial body - execute_fetch: merge caller headers case-insensitively so a differently-cased header (e.g. User-Agent) overrides the injected default rather than being sent as a duplicate - main: log YAML seed parse failures via tracing::warn! instead of silently dropping them with .ok() - skills/index.md: drop redundant intensifier ("very large pages") Claude-Session: https://claude.ai/code/session_01QY5uCFgwr4wXR7ZcAKanaR --- web/skills/index.md | 2 +- web/src/fetch.rs | 34 ++++++++++++++++++++++------------ web/src/main.rs | 9 ++++++--- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/web/skills/index.md b/web/skills/index.md index 4d78362418..6437160dea 100644 --- a/web/skills/index.md +++ b/web/skills/index.md @@ -106,7 +106,7 @@ There is **no `too_large` error** — oversize responses come back `ok: true` wi Two orthogonal knobs — pick ONE: - **`response_format`** = transport encoding for APIs/binaries (`text`/`base64`/`json`). The body is returned untouched. -- **`format`** = page-reading mode (`markdown`/`text`/`html`). The request goes out with a browser User-Agent + format-matched `Accept` header, and `text/html` responses are transformed (`markdown` is the right default for reading pages — far fewer tokens than raw HTML). Non-HTML bodies pass through unchanged. If Cloudflare answers `403` with a challenge, the worker retries once with its honest UA (beats the UA-fingerprint rule only, not full JS challenges). For very large pages, lower `max_bytes` — conversion runs on the capped body. +- **`format`** = page-reading mode (`markdown`/`text`/`html`). The request goes out with a browser User-Agent + format-matched `Accept` header, and `text/html` responses are transformed (`markdown` is the right default for reading pages — far fewer tokens than raw HTML). Non-HTML bodies pass through unchanged. If Cloudflare answers `403` with a challenge, the worker retries once with its honest UA (beats the UA-fingerprint rule only, not full JS challenges). For large pages, lower `max_bytes` — conversion runs on the capped body. Don't combine them: when `format` is set, `response_format` is ignored (treated as `"text"`). diff --git a/web/src/fetch.rs b/web/src/fetch.rs index 5d7bb43cbb..b76d1f4c67 100644 --- a/web/src/fetch.rs +++ b/web/src/fetch.rs @@ -88,10 +88,12 @@ pub fn try_parse_json(text: &str) -> Result { serde_json::from_str(text).map_err(|e| e.to_string()) } -/// Read a byte stream up to `cap`, marking truncation. Once the cap is -/// reached the result is terminal — a subsequent stream error (e.g. reset -/// on drop) is ignored: the cap is the contract. -pub async fn read_capped(mut stream: S, cap: u64) -> (Vec, bool) +/// Read a byte stream up to `cap`, marking truncation. A transport error +/// before the cap is reached surfaces as `Err` — the caller maps it to a +/// transport error rather than returning partial data as a success. Once the +/// cap is reached the result is terminal: a subsequent stream error (e.g. +/// reset on drop) is ignored — the cap is the contract. +pub async fn read_capped(mut stream: S, cap: u64) -> Result<(Vec, bool), String> where S: futures::Stream> + Unpin, { @@ -100,16 +102,16 @@ where while let Some(item) = stream.next().await { let chunk = match item { Ok(c) => c, - Err(_) => return (buf, false), // pre-cap transport error: surface as non-truncated; caller maps it + Err(e) => return Err(e.to_string()), // pre-cap transport error: surface to caller }; if buf.len() + chunk.len() > cap { let remaining = cap.saturating_sub(buf.len()); buf.extend_from_slice(&chunk[..remaining]); - return (buf, true); // truncation is terminal; drop the stream + return Ok((buf, true)); // truncation is terminal; drop the stream } buf.extend_from_slice(&chunk); } - (buf, false) + Ok((buf, false)) } struct RawResponse { @@ -205,7 +207,10 @@ async fn perform_request( let headers = flatten_headers(resp.headers()); // Box::pin so the `Unpin` bound on read_capped holds regardless of // reqwest's internal stream type. - let (bytes, truncated) = read_capped(Box::pin(resp.bytes_stream()), max_bytes).await; + let (bytes, truncated) = match read_capped(Box::pin(resp.bytes_stream()), max_bytes).await { + Ok(v) => v, + Err(e) => return Outcome::Transport(e), + }; Outcome::Response(RawResponse { status, status_text, @@ -260,7 +265,10 @@ pub async fn execute_fetch(payload: FetchPayload, cfg: &WebConfig) -> Value { } if let Some(h) = &payload.headers { for (k, v) in h { - base.insert(k.clone(), v.clone()); // caller wins + // caller wins case-insensitively: drop any default we injected + // under a different case, then keep the caller's header verbatim. + base.retain(|bk, _| !bk.eq_ignore_ascii_case(k)); + base.insert(k.clone(), v.clone()); } } let (effective_body, mut current_headers) = apply_json_payload(&payload, base); @@ -608,7 +616,7 @@ mod tests { Ok(bytes::Bytes::from_static(b"aaaa")), Ok(bytes::Bytes::from_static(b"bbbb")), ]; - let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 6).await; + let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 6).await.unwrap(); assert_eq!(bytes, b"aaaabb"); assert!(truncated); } @@ -617,7 +625,9 @@ mod tests { async fn read_capped_full_under_cap() { let chunks: Vec> = vec![Ok(bytes::Bytes::from_static(b"hello"))]; - let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 100).await; + let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 100) + .await + .unwrap(); assert_eq!(bytes, b"hello"); assert!(!truncated); } @@ -627,7 +637,7 @@ mod tests { // a chunk that overflows the cap, then an error that must be ignored let chunks: Vec> = vec![Ok(bytes::Bytes::from_static(b"aaaaaaaa"))]; - let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 4).await; + let (bytes, truncated) = read_capped(futures::stream::iter(chunks), 4).await.unwrap(); assert_eq!(bytes, b"aaaa"); assert!(truncated); } diff --git a/web/src/main.rs b/web/src/main.rs index 8a22054ff7..e807eec1da 100644 --- a/web/src/main.rs +++ b/web/src/main.rs @@ -70,15 +70,18 @@ async fn main() -> Result<()> { return None; } }; - match serde_yaml::from_str::(&contents).ok() { - Some(v) => match WebConfig::from_json(&v) { + match serde_yaml::from_str::(&contents) { + Ok(v) => match WebConfig::from_json(&v) { Ok(cfg) => Some(cfg), Err(e) => { tracing::warn!(path, error = %e, "failed to parse seed config; ignoring"); None } }, - None => None, + Err(e) => { + tracing::warn!(path, error = %e, "failed to parse YAML seed config; ignoring"); + None + } } }); From 70f29665dbef0014f1ead88f2a9b280c22adf6af Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Fri, 19 Jun 2026 15:07:11 -0300 Subject: [PATCH 22/22] ci(release): include the web worker in the release pipeline - release.yml: trigger the Release workflow on `web/v*` tags - create-tag.yml: add `web` to the Create Tag worker choices so a release tag can be cut via the standard dispatch - parse_publish_workers_input.py: allow `web` in the skills-publish allowlist (web ships skills/index.md), so `all` covers it web/iii.worker.yaml already declares deploy: binary / bin: iii-web, so the generic Rust binary build + registry publish path (same as iii-directory) handles the rest with no further changes. Claude-Session: https://claude.ai/code/session_01QY5uCFgwr4wXR7ZcAKanaR --- .github/scripts/parse_publish_workers_input.py | 3 ++- .github/workflows/create-tag.yml | 1 + .github/workflows/release.yml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/parse_publish_workers_input.py b/.github/scripts/parse_publish_workers_input.py index 621891582c..e7e5a6993c 100644 --- a/.github/scripts/parse_publish_workers_input.py +++ b/.github/scripts/parse_publish_workers_input.py @@ -22,7 +22,8 @@ "image-resize", "mcp", "shell", - "storage" + "storage", + "web", ) _ALLOWED_SET = frozenset(ALLOWED_WORKERS) diff --git a/.github/workflows/create-tag.yml b/.github/workflows/create-tag.yml index abac00670d..ac9da9be16 100644 --- a/.github/workflows/create-tag.yml +++ b/.github/workflows/create-tag.yml @@ -30,6 +30,7 @@ on: - session-manager - shell - storage + - web bump: description: 'Version bump' required: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 801a496df6..2a6b5e120c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ on: - 'session-manager/v*' - 'shell/v*' - 'storage/v*' + - 'web/v*' workflow_dispatch: inputs: tag: