http: replace picohttpparser with a Rust parser; classify headers once and build fetch Response headers lazily

.github/workflows/miri.yml

@@ -19,6 +19,7 @@ on:
       - "src/http_types/**"
       - "src/md/**"
       - "src/paths/**"
+      - "src/picohttp/**"
       - "src/ptr/**"
       - "src/resolve_builtins/**"
       - "src/shell_parser/**"

.gitignore

@@ -159,7 +159,6 @@ codegen-for-zig-team.tar.gz
 /src/deps/lol*html
 /src/deps/ls*hpack
 /src/deps/mimalloc
-/src/deps/picohttpparser
 /src/deps/tinycc
 /src/deps/WebKit
 /src/deps/zig

CLAUDE.md

@@ -147,7 +147,7 @@ The Rust side is a Cargo workspace of ~200 crates rooted at `Cargo.toml`. The ke
 
 #### Vendored Dependencies (`vendor/`)
 
-Third-party C/C++ libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), hdrhistogram, highway (SIMD), libarchive (tar/zip), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`.
+Third-party C/C++ libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), hdrhistogram, highway (SIMD), libarchive (tar/zip), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`.
 
 ### JavaScript Class Implementation (C++)
 

LICENSE.md

@@ -28,7 +28,6 @@ Bun statically links these libraries:
 | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT |
 | [`lsquic`](https://github.com/litespeedtech/lsquic) | MIT (portions derived from [Chromium proto-quic](https://github.com/litespeedtech/lsquic/blob/master/LICENSE.chrome), BSD 3-Clause) |
 | [`mimalloc`](https://github.com/microsoft/mimalloc) | MIT |
-| [`picohttp`](https://github.com/h2o/picohttpparser) | dual-licensed under the Perl License or the MIT License |
 | [`zstd`](https://github.com/facebook/zstd) | dual-licensed under the BSD License or GPLv2 license |
 | [`simdutf`](https://github.com/simdutf/simdutf) | Apache 2.0 |
 | [`tinycc`](https://github.com/tinycc/tinycc) | LGPL v2.1 |
@@ -45,6 +44,7 @@ Bun statically links these libraries:
 | [`highway`](https://github.com/google/highway) | Apache 2.0 |
 | [`uucode`](https://github.com/jacobsandlund/uucode) | MIT |
 | A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets) | Apache 2.0 licensed |
+| HTTP/1.1 chunked-encoding decoder derived from [`picohttpparser`](https://github.com/h2o/picohttpparser) | MIT licensed |
 | Parts of [Tigerbeetle's IO code](https://github.com/tigerbeetle/tigerbeetle/blob/532c8b70b9142c17e07737ab6d3da68d7500cbca/src/io/windows.zig#L1) | Apache 2.0 licensed |
 | `__cxa_thread_atexit` fallback from [LLVM libc++abi](https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/libcxxabi/src/cxa_thread_atexit.cpp) | Apache 2.0 with LLVM exception |
 

bench/fetch/response-headers-server.mjs

@@ -0,0 +1,44 @@
+// Serves a 2-byte body under one of three response-header profiles, keyed by
+// path: /small (2 headers), /nginx (14, API-gateway style), /cookies (long
+// Location/Set-Cookie values). Pair with response-headers.mjs; run it in its
+// own process (ideally pinned to other cores) so it doesn't share a CPU with
+// the client:  PORT=4001 bun bench/fetch/response-headers-server.mjs
+const sets = {
+  small: { "Content-Type": "text/plain", "X-Id": "1" },
+  nginx: {
+    "Server": "nginx/1.25.3",
+    "Content-Type": "application/json; charset=utf-8",
+    "Vary": "Accept-Encoding",
+    "Cache-Control": "private, max-age=0, no-cache, no-store, must-revalidate",
+    "ETag": 'W/"bc55-7Zci8Yc4Bq3vJk8pQ0h5nX0m9sE"',
+    "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
+    "X-Content-Type-Options": "nosniff",
+    "X-Frame-Options": "SAMEORIGIN",
+    "X-Request-Id": "3f1c9a7e-5b2d-4c8e-9f0a-1b2c3d4e5f60",
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Expose-Headers": "X-Request-Id, ETag, Link",
+    "X-RateLimit-Limit": "5000",
+    "X-RateLimit-Remaining": "4987",
+    "X-RateLimit-Reset": "1723025489",
+  },
+  cookies: {
+    "Location":
+      "https://accounts.example.com/signin/v2/identifier?continue=https%3A%2F%2Fmail.example.com%2Fmail%2F&service=mail&flowName=GlifWebSignIn&flowEntry=ServiceLogin",
+    "Set-Cookie":
+      "__Host-SESSION=CgQIARAB.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age=2592000",
+    "Content-Type": "text/html; charset=UTF-8",
+    "Alt-Svc": 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000',
+  },
+};
+const responses = Object.fromEntries(Object.entries(sets).map(([k, h]) => [k, { headers: new Headers(h) }]));
+const server = Bun.serve({
+  port: Number(process.env.PORT || 0),
+  reusePort: true,
+  fetch(req) {
+    const kind = new URL(req.url).pathname.slice(1) || "nginx";
+    const init = responses[kind];
+    if (!init) return new Response(`unknown profile "${kind}"; use one of ${Object.keys(responses)}`, { status: 404 });
+    return new Response("ok", init);
+  },
+});
+console.log(server.port);

bench/fetch/response-headers.mjs

@@ -0,0 +1,57 @@
+// Client CPU per fetch() against response-headers-server.mjs.
+//   bun bench/fetch/response-headers.mjs <port> [small|nginx|cookies] [body|get|iter] [requests] [concurrency]
+// mode: body = read the body only; get = also headers.get(); iter = also iterate all headers.
+// Reports requests/s and this process's CPU time per request (user+sys across
+// all threads), which is the number to compare between builds when the server
+// is the throughput bottleneck.
+const [port, kind = "nginx", mode = "body", total = 200000, conc = 64] = process.argv.slice(2);
+const url = `http://127.0.0.1:${port}/${kind}`;
+const N = Number(total),
+  C = Number(conc);
+const isCount = n => Number.isSafeInteger(n) && n >= 1;
+if (
+  !(isCount(Number(port)) && port <= 65535) ||
+  !["body", "get", "iter"].includes(mode) ||
+  !isCount(N) ||
+  !isCount(C)
+) {
+  console.error("usage: response-headers.mjs <port> [small|nginx|cookies] [body|get|iter] [requests] [concurrency]");
+  process.exit(1);
+}
+let sink = 0;
+async function one() {
+  const r = await fetch(url);
+  if (mode === "get") sink += r.headers.get("content-type").length;
+  else if (mode === "iter") for (const [k, v] of r.headers) sink += k.length + v.length;
+  const b = await r.arrayBuffer();
+  sink += b.byteLength;
+}
+// warmup
+await Promise.all(Array.from({ length: C }, one));
+const cpu0 = process.cpuUsage();
+const t0 = performance.now();
+let started = 0;
+async function worker() {
+  while (started < N) {
+    started++;
+    await one();
+  }
+}
+await Promise.all(Array.from({ length: C }, worker));
+const t = performance.now() - t0;
+const cpu = process.cpuUsage(cpu0);
+const cpuMs = (cpu.user + cpu.system) / 1000;
+console.log(
+  JSON.stringify({
+    kind,
+    mode,
+    N,
+    C,
+    ms: +t.toFixed(0),
+    rps: Math.round(N / (t / 1000)),
+    cpu_us_per_req: +((cpuMs * 1000) / N).toFixed(2),
+    user_ms: Math.round(cpu.user / 1000),
+    sys_ms: Math.round(cpu.system / 1000),
+    sink,
+  }),
+);

docs/project/license.mdx

@@ -30,7 +30,6 @@ Bun statically links these libraries:
 | [`libarchive`](https://github.com/libarchive/libarchive)                                                                                         | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING)             |
 | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api)                                                                           | BSD 3-Clause                                                                                 |
 | [`mimalloc`](https://github.com/microsoft/mimalloc)                                                                                              | MIT                                                                                          |
-| [`picohttp`](https://github.com/h2o/picohttpparser)                                                                                              | dual-licensed under the Perl License or the MIT License                                      |
 | [`zstd`](https://github.com/facebook/zstd)                                                                                                       | dual-licensed under the BSD License or GPLv2 license                                         |
 | [`simdutf`](https://github.com/simdutf/simdutf)                                                                                                  | Apache 2.0                                                                                   |
 | [`tinycc`](https://github.com/tinycc/tinycc)                                                                                                     | LGPL v2.1                                                                                    |
@@ -42,6 +41,7 @@ Bun statically links these libraries:
 | [`libuv`](https://github.com/libuv/libuv) (on Windows)                                                                                           | MIT                                                                                          |
 | [`libdeflate`](https://github.com/ebiggers/libdeflate)                                                                                           | MIT                                                                                          |
 | A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets)                                                                          | Apache 2.0 licensed                                                                          |
+| HTTP/1.1 chunked-encoding decoder derived from [`picohttpparser`](https://github.com/h2o/picohttpparser)                                         | MIT licensed                                                                                 |
 | Parts of [Tigerbeetle's IO code](https://github.com/tigerbeetle/tigerbeetle/blob/532c8b70b9142c17e07737ab6d3da68d7500cbca/src/io/windows.zig#L1) | Apache 2.0 licensed                                                                          |
 
 ## Polyfills

scripts/build/bun.ts

@@ -189,9 +189,7 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
     }
   }
 
-  // Collect all dep lib paths, include dirs, output stamps, and directly-
-  // compiled source files (deps like picohttpparser that provide .c files
-  // instead of a .a — we compile those alongside bun's own sources).
+  // Collect all dep lib paths, include dirs, and output stamps.
   const depLibs: string[] = [];
   const depObjects: string[] = [];
   const depIncludes: string[] = [];
@@ -325,9 +323,6 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
     noPchSources.add(rescleBinding);
   }
 
-  // Deps with provides.sources compiled in the loop below so each dep's
-  // phony can point at its own .o files.
-
   // Codegen .cpp files — compiled like regular sources.
   cxxSources.push(...codegen.cppSources);
   cxxSources.push(...codegen.bindgenV2Cpp);
@@ -387,16 +382,15 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
 
   // Compile all .c files. No PCH — dep signal applied directly.
   const cObjects: string[] = [];
-  const compileC = (src: string): string => {
-    const obj = cc(n, cfg, src, {
-      flags: cFlagsFull,
-      implicitInputs: depHeaderSignal,
-      orderOnlyInputs: codegenOrderOnly,
-    });
-    cObjects.push(obj);
-    return obj;
-  };
-  for (const src of cSources) compileC(src);
+  for (const src of cSources) {
+    cObjects.push(
+      cc(n, cfg, src, {
+        flags: cFlagsFull,
+        implicitInputs: depHeaderSignal,
+        orderOnlyInputs: codegenOrderOnly,
+      }),
+    );
+  }
 
   // InternalModuleRegistryConstants.S — `.incbin`s the bundled JS module sources
   // so InternalModuleRegistry.cpp sees a tiny {offset, length} table instead of
@@ -411,16 +405,6 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
     }),
   );
 
-  // Deps that contribute source files for bun to compile directly (via
-  // provides.sources) instead of building a lib. Compile them here with
-  // bun's full flag set and give each a phony so `--target <name>` builds
-  // its .o files. libs.length === 0 guard: deps with a build step already
-  // got a phony in resolveDep — don't emit a duplicate.
-  for (const d of deps) {
-    if (d.sources.length === 0 || d.libs.length > 0) continue;
-    n.phony(d.name, d.sources.map(compileC));
-  }
-
   // Dep objects (when !cfg.archiveDeps) are linked alongside bun's own
   // objects — same response file, same archive in cpp-only mode. With
   // cfg.archiveDeps they live in depLibs as .a files instead.

scripts/build/deps/index.ts

@@ -28,7 +28,6 @@ import { lsqpack } from "./lsqpack.ts";
 import { lsquic } from "./lsquic.ts";
 import { mimalloc } from "./mimalloc.ts";
 import { nodejsHeaders } from "./nodejs-headers.ts";
-import { picohttpparser } from "./picohttpparser.ts";
 import { sqlite } from "./sqlite.ts";
 import { tinycc } from "./tinycc.ts";
 import { webkit } from "./webkit.ts";
@@ -43,8 +42,7 @@ import { zstd } from "./zstd.ts";
  * boringssl near the end — many things depend on crypto/ssl symbols.
  */
 export const allDeps: readonly Dependency[] = [
-  // Header-only / source-only first — no link order concerns.
-  picohttpparser,
+  // Header-only first — no link order concerns.
   nodejsHeaders,
 
   zlib,
@@ -95,7 +93,6 @@ export {
   lsquic,
   mimalloc,
   nodejsHeaders,
-  picohttpparser,
   sqlite,
   tinycc,
   webkit,

scripts/build/flags.ts

@@ -1531,7 +1531,6 @@ export function bunIncludes(cfg: Config): string[] {
     join(cwd, "src/uws_sys"),
     codegenDir,
     vendorDir,
-    join(vendorDir, "picohttpparser"),
     join(vendorDir, "zlib"),
     // NODEJS_HEADERS_PATH comes from the nodejs dep; added separately
   ];