diff --git a/CLAUDE.md b/CLAUDE.md index b5d708bcf4ea..e5789442ef05 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -208,7 +208,7 @@ Third-party C/C++ libraries are vendored locally and can be read from disk (thes - `vendor/tinycc/` - TinyCC (FFI JIT compiler, fork: oven-sh/tinycc) - `vendor/WebKit/` - WebKit/JavaScriptCore (JS engine) - `vendor/zig/` - Zig compiler/stdlib -- `vendor/zlib/` - zlib (compression, cloudflare fork) +- `vendor/zlib/` - zlib-ng (compression, zlib-compat mode) - `vendor/zstd/` - Zstandard (compression) Build configuration for these is in `scripts/build/deps/*.ts`. diff --git a/LICENSE.md b/LICENSE.md index 170bc19e1aab..5ba23859bbf9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -30,7 +30,7 @@ Bun statically links these libraries: | [`simdutf`](https://github.com/simdutf/simdutf) | Apache 2.0 | | [`tinycc`](https://github.com/tinycc/tinycc) | LGPL v2.1 | | [`uSockets`](https://github.com/uNetworking/uSockets) | Apache 2.0 | -| [`zlib-cloudflare`](https://github.com/cloudflare/zlib) | zlib | +| [`zlib-ng`](https://github.com/zlib-ng/zlib-ng) | zlib | | [`c-ares`](https://github.com/c-ares/c-ares) | MIT licensed | | [`libicu`](https://github.com/unicode-org/icu) 72 | [license here](https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE) | | [`libbase64`](https://github.com/aklomp/base64/blob/master/LICENSE) | BSD 2-Clause | diff --git a/bench/snippets/fetch-gzip.mjs b/bench/snippets/fetch-gzip.mjs new file mode 100644 index 000000000000..af2068a104c9 --- /dev/null +++ b/bench/snippets/fetch-gzip.mjs @@ -0,0 +1,36 @@ +// Benchmark fetch() decoding gzip Content-Encoding — the path PR #16100 +// found regressed (Next.js standalone server). Server gzips responses, +// client inflates them via the streaming inflate path. +import { bench, group, run } from "../runner.mjs"; +import zlib from "node:zlib"; + +const html = Buffer.from("
Hello World
".repeat(500)); +const htmlGz = zlib.gzipSync(html, { level: 6 }); + +using server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/gz") { + return new Response(htmlGz, { + headers: { "Content-Encoding": "gzip", "Content-Type": "text/html" }, + }); + } + return new Response(html, { headers: { "Content-Type": "text/html" } }); + }, +}); + +const base = `http://localhost:${server.port}`; + +group("fetch + gzip decode", () => { + bench("11KB gzipped → text()", async () => { + const res = await fetch(`${base}/gz`); + await res.text(); + }); + bench("11KB plain → text()", async () => { + const res = await fetch(`${base}/plain`); + await res.text(); + }); +}); + +await run(); diff --git a/bench/snippets/zlib-comprehensive.mjs b/bench/snippets/zlib-comprehensive.mjs new file mode 100644 index 000000000000..e9192201fd7f --- /dev/null +++ b/bench/snippets/zlib-comprehensive.mjs @@ -0,0 +1,167 @@ +// Comprehensive zlib benchmark: sync, async, streaming, realistic payloads. +// Designed to compare cloudflare-zlib vs zlib-ng on the workloads Bun cares about: +// - HTTP gzip response encoding (streaming, level 1/6) +// - npm tarball gunzip (streaming inflate, ~10-500KB inputs) +// - fetch() Content-Encoding: gzip decoding +// - Bun.gzipSync / node:zlib one-shot + +import { bench, group, run } from "../runner.mjs"; +import zlib from "node:zlib"; +import { promisify } from "node:util"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import crypto from "node:crypto"; + +const gzip = promisify(zlib.gzip); +const gunzip = promisify(zlib.gunzip); +const deflate = promisify(zlib.deflate); +const inflate = promisify(zlib.inflate); + +// ─── Test corpora ─── +// "html": realistic web page — repeated structure with varying content (compresses well, ~85%) +// "json": API response — keys repeat, values vary (compresses ~70%) +// "binary": random bytes — incompressible, exercises the "give up fast" path +// "code": minified JS — what npm tarballs actually contain + +function makeHtml(kb) { + let s = "Bench"; + let i = 0; + while (s.length < kb * 1024) { + s += `

Item ${i}

Lorem ipsum dolor sit amet ${i % 17}.

`; + i++; + } + return Buffer.from(s); +} + +function makeJson(kb) { + const arr = []; + let i = 0; + while (JSON.stringify(arr).length < kb * 1024) { + arr.push({ id: i, name: `user_${i}`, email: `u${i}@example.com`, active: i % 3 === 0, score: (i * 7919) % 1000 }); + i++; + } + return Buffer.from(JSON.stringify(arr)); +} + +const corpora = { + "html-4K": makeHtml(4), + "html-128K": makeHtml(128), + "html-1M": makeHtml(1024), + "json-128K": makeJson(128), + "binary-128K": crypto.randomBytes(128 * 1024), +}; + +// Pre-compress for decompression benches +const gzipped = {}; +for (const [name, buf] of Object.entries(corpora)) { + gzipped[name] = zlib.gzipSync(buf, { level: 6 }); +} + +// ─── Sync one-shot ─── +group("gzipSync level=1", () => { + for (const [name, buf] of Object.entries(corpora)) { + bench(name, () => zlib.gzipSync(buf, { level: 1 })); + } +}); + +group("gzipSync level=6", () => { + for (const [name, buf] of Object.entries(corpora)) { + bench(name, () => zlib.gzipSync(buf, { level: 6 })); + } +}); + +group("gunzipSync", () => { + for (const [name, buf] of Object.entries(gzipped)) { + bench(name, () => zlib.gunzipSync(buf)); + } +}); + +// ─── Async one-shot (threadpool) ─── +group("gzip async level=6", () => { + for (const [name, buf] of Object.entries(corpora)) { + bench(name, async () => await gzip(buf, { level: 6 })); + } +}); + +group("gunzip async", () => { + for (const [name, buf] of Object.entries(gzipped)) { + bench(name, async () => await gunzip(buf)); + } +}); + +// ─── Streaming (HTTP server / npm install path) ─── +// Feed input in 16KB chunks like a real socket would. +function chunked(buf, size = 16 * 1024) { + const chunks = []; + for (let i = 0; i < buf.length; i += size) chunks.push(buf.subarray(i, i + size)); + return chunks; +} + +const streamInputs = { + "html-128K": chunked(corpora["html-128K"]), + "html-1M": chunked(corpora["html-1M"]), +}; +const streamGzInputs = { + "html-128K": chunked(gzipped["html-128K"]), + "html-1M": chunked(gzipped["html-1M"]), +}; + +async function drain(stream) { + for await (const _ of stream); +} + +group("createGzip stream level=1", () => { + for (const [name, chunks] of Object.entries(streamInputs)) { + bench(name, async () => { + const gz = zlib.createGzip({ level: 1 }); + const src = Readable.from(chunks); + await pipeline(src, gz, drain); + }); + } +}); + +group("createGzip stream level=6", () => { + for (const [name, chunks] of Object.entries(streamInputs)) { + bench(name, async () => { + const gz = zlib.createGzip({ level: 6 }); + const src = Readable.from(chunks); + await pipeline(src, gz, drain); + }); + } +}); + +group("createGunzip stream", () => { + for (const [name, chunks] of Object.entries(streamGzInputs)) { + bench(name, async () => { + const gz = zlib.createGunzip(); + const src = Readable.from(chunks); + await pipeline(src, gz, drain); + }); + } +}); + +// ─── deflateInit/inflateInit overhead (small payloads, many iterations) ─── +// zlib-ng has higher init cost due to larger state structs. This matters for +// per-request gzip on tiny responses. +const tiny = Buffer.from("Hello, World!"); +const tinyGz = zlib.gzipSync(tiny); + +group("init overhead (13B payload)", () => { + bench("gzipSync", () => zlib.gzipSync(tiny, { level: 6 })); + bench("gunzipSync", () => zlib.gunzipSync(tinyGz)); + bench("deflateSync", () => zlib.deflateSync(tiny, { level: 6 })); + bench("inflateSync", () => zlib.inflateSync(zlib.deflateSync(tiny))); +}); + +// ─── Compression ratio (printed, not benched) ─── +console.log("\n# Compression ratio (output bytes, level=6):"); +console.log("# corpus input output ratio"); +for (const [name, buf] of Object.entries(corpora)) { + const out = zlib.gzipSync(buf, { level: 6 }); + console.log( + `# ${name.padEnd(12)} ${String(buf.length).padStart(8)} ${String(out.length).padStart(8)} ${((out.length / buf.length) * 100).toFixed(1)}%`, + ); +} +console.log(`# zlib version: ${process.versions.zlib}\n`); + +await run(); diff --git a/docs/project/license.mdx b/docs/project/license.mdx index fa51203c263c..6d44dd63e891 100644 --- a/docs/project/license.mdx +++ b/docs/project/license.mdx @@ -35,7 +35,7 @@ Bun statically links these libraries: | [`simdutf`](https://github.com/simdutf/simdutf) | Apache 2.0 | | [`tinycc`](https://github.com/tinycc/tinycc) | LGPL v2.1 | | [`uSockets`](https://github.com/uNetworking/uSockets) | Apache 2.0 | -| [`zlib-cloudflare`](https://github.com/cloudflare/zlib) | zlib | +| [`zlib-ng`](https://github.com/zlib-ng/zlib-ng) | zlib | | [`c-ares`](https://github.com/c-ares/c-ares) | MIT licensed | | [`libicu`](https://github.com/unicode-org/icu) 72 | [license here](https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE) | | [`libbase64`](https://github.com/aklomp/base64/blob/master/LICENSE) | BSD 2-Clause | diff --git a/patches/zlib/CMakeLists.txt.patch b/patches/zlib/CMakeLists.txt.patch deleted file mode 100644 index 5b6261981472..000000000000 --- a/patches/zlib/CMakeLists.txt.patch +++ /dev/null @@ -1,8 +0,0 @@ ---- CMakeLists.txt -+++ CMakeLists.txt -@@ -1,4 +1,4 @@ --cmake_minimum_required(VERSION 2.8.12) -+cmake_minimum_required(VERSION 2.8.12...3.5) - - project(zlib C) - diff --git a/patches/zlib/clang-cl-arm64.patch b/patches/zlib/clang-cl-arm64.patch new file mode 100644 index 000000000000..7269696e158b --- /dev/null +++ b/patches/zlib/clang-cl-arm64.patch @@ -0,0 +1,55 @@ +clang-cl on Windows ARM64 defines _MSC_VER but is not real MSVC: it ships +clang's own /. MSVC's defines +vld1q_u16_x4/vld1q_u8_x4/vst1q_u16_x4 as macros, which macro-expand the +polyfill function definitions at neon_intrins.h:40-63 into garbage. MSVC's + lacks the ARM ACLE CRC intrinsics (__crc32b etc.) that clang's + provides. + +Gate the MSVC-specific includes on !defined(__clang__) so clang-cl takes the +standard ACLE/NEON path. Also fix the matching cmake feature-detection probes +so HAVE_ARMV8_INTRIN and NEON_HAS_LD4 succeed under clang-cl, which sets +ARM_CRC32_INTRIN/ARM_NEON_HASLD4 and skips the polyfills entirely. + +Upstream: not yet reported (zlib-ng CI does not cover clang-cl ARM64). +--- a/arch/arm/neon_intrins.h ++++ b/arch/arm/neon_intrins.h +@@ -1,7 +1,7 @@ + #ifndef ARM_NEON_INTRINS_H + #define ARM_NEON_INTRINS_H + +-#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC)) ++#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_ARM64) || defined(_M_ARM64EC)) + /* arm64_neon.h is MSVC specific */ + # include + #else +--- a/arch/arm/acle_intrins.h ++++ b/arch/arm/acle_intrins.h +@@ -2,7 +2,7 @@ + #define ARM_ACLE_INTRINS_H + + #include +-#ifdef _MSC_VER ++#if defined(_MSC_VER) && !defined(__clang__) + # include + #elif defined(HAVE_ARM_ACLE_H) + # include +--- a/cmake/detect-intrinsics.cmake ++++ b/cmake/detect-intrinsics.cmake +@@ -42,7 +42,7 @@ macro(check_armv8_compiler_flag) + ) + # Check whether compiler supports ARMv8 intrinsics + check_c_source_compiles( +- "#if defined(_MSC_VER) ++ "#if defined(_MSC_VER) && !defined(__clang__) + #include + #else + #include +@@ -285,7 +285,7 @@ macro(check_neon_ld4_intrinsics) + # Check whether compiler supports loading 4 neon vecs into a register range + set(CMAKE_REQUIRED_FLAGS "${NEONFLAG} ${NATIVEFLAG} ${ZNOLTOFLAG}") + check_c_source_compiles( +- "#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC)) ++ "#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_ARM64) || defined(_M_ARM64EC)) + # include + #else + # include diff --git a/patches/zlib/deflate.h.patch b/patches/zlib/deflate.h.patch deleted file mode 100644 index 91eca4c8bf79..000000000000 --- a/patches/zlib/deflate.h.patch +++ /dev/null @@ -1,26 +0,0 @@ ---- deflate.h -+++ deflate.h -@@ -326,23 +326,7 @@ extern const uint8_t ZLIB_INTERNAL _dist_code[]; - flush = (s->sym_next == s->sym_end); \ - } - --#ifdef _MSC_VER -- --/* MSC doesn't have __builtin_expect. Just ignore likely/unlikely and -- hope the compiler optimizes for the best. --*/ --#define likely(x) (x) --#define unlikely(x) (x) -- --int __inline __builtin_ctzl(unsigned long mask) --{ -- unsigned long index ; -- -- return _BitScanForward(&index, mask) == 0 ? 32 : ((int)index) ; --} --#else - #define likely(x) __builtin_expect((x),1) - #define unlikely(x) __builtin_expect((x),0) --#endif - - #endif /* DEFLATE_H */ diff --git a/patches/zlib/ucm.cmake.patch b/patches/zlib/ucm.cmake.patch deleted file mode 100644 index 325587f632d8..000000000000 --- a/patches/zlib/ucm.cmake.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ucm.cmake -+++ ucm.cmake -@@ -10,7 +10,7 @@ - # The documentation can be found at the library's page: - # https://github.com/onqtam/ucm - --cmake_minimum_required(VERSION 2.8.12) -+cmake_minimum_required(VERSION 2.8.12...3.5) - - include(CMakeParseArguments) - diff --git a/scripts/build/bun.ts b/scripts/build/bun.ts index 13c343caea59..f641f6c81c02 100644 --- a/scripts/build/bun.ts +++ b/scripts/build/bun.ts @@ -158,9 +158,13 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput { n.comment("─── Dependencies ───"); n.blank(); const deps: ResolvedDep[] = []; + const depsByName = new Map(); for (const dep of allDeps) { - const resolved = resolveDep(n, cfg, dep); - if (resolved !== null) deps.push(resolved); + const resolved = resolveDep(n, cfg, dep, depsByName); + if (resolved !== null) { + deps.push(resolved); + depsByName.set(dep.name, resolved); + } } // Collect all dep lib paths, include dirs, output stamps, and directly- @@ -449,7 +453,7 @@ function emitZigOnly(n: Ninja, cfg: Config, sources: Sources): BunOutput { // Only dep: zstd, for @cImport headers. resolveDep emits its // fetch/configure/build; emitZig only depends on the fetch stamp. - const zstdDep = resolveDep(n, cfg, zstd); + const zstdDep = resolveDep(n, cfg, zstd, new Map()); assert(zstdDep !== null, "zstd resolveDep returned null — should never be skipped"); // Codegen: emitted fully, but only zigInputs/zigOrderOnly are pulled. diff --git a/scripts/build/deps/libarchive.ts b/scripts/build/deps/libarchive.ts index 3377f23b0f0c..7b366cd12638 100644 --- a/scripts/build/deps/libarchive.ts +++ b/scripts/build/deps/libarchive.ts @@ -10,7 +10,7 @@ */ import type { Dependency } from "../source.ts"; -import { depSourceDir } from "../source.ts"; +import { depBuildDir } from "../source.ts"; const LIBARCHIVE_COMMIT = "ded82291ab41d5e355831b96b0e1ff49e24d8939"; @@ -35,9 +35,11 @@ export const libarchive: Dependency = { ], // libarchive's configure-time check_include_file("zlib.h") needs zlib's - // headers on disk. We don't LINK zlib into libarchive (ENABLE_ZLIB=OFF) — - // we just need the compile-time knowledge that deflate exists so - // libarchive compiles its gzip filter instead of fork/exec'ing gzip(1). + // headers on disk. zlib-ng generates zlib.h during its own cmake configure, + // so we need zlib BUILT (not just fetched) before libarchive configures. + // We don't LINK zlib into libarchive (ENABLE_ZLIB=OFF) — we just need the + // compile-time knowledge that deflate exists so libarchive compiles its + // gzip filter instead of fork/exec'ing gzip(1). fetchDeps: ["zlib"], build: cfg => ({ @@ -46,9 +48,9 @@ export const libarchive: Dependency = { pic: true, libSubdir: "libarchive", - // -I into zlib's SOURCE dir (vendor/zlib/). This is why fetchDeps exists: - // the zlib source must be on disk before libarchive's configure runs. - extraCFlags: [`-I${depSourceDir(cfg, "zlib")}`], + // -I into zlib's BUILD dir (zlib-ng generates zlib.h there at configure + // time). zlib must be built before libarchive's configure runs. + extraCFlags: [`-I${depBuildDir(cfg, "zlib")}`], args: { ENABLE_INSTALL: "OFF", diff --git a/scripts/build/deps/zlib.ts b/scripts/build/deps/zlib.ts index ad58a077b4a5..deb3630c0e3c 100644 --- a/scripts/build/deps/zlib.ts +++ b/scripts/build/deps/zlib.ts @@ -1,12 +1,23 @@ /** - * zlib — the classic deflate library. Cloudflare's fork has SIMD-accelerated - * CRC and hash (2-4x faster compression on modern CPUs). Backs node:zlib - * and the gzip Content-Encoding. + * zlib-ng — next-generation zlib with SIMD-accelerated CRC, adler32, hash, + * and chunk operations across SSE2/AVX2/AVX512/NEON/VSX. Runtime CPU + * detection picks the fastest path. Backs node:zlib and gzip Content-Encoding. + * + * Built in ZLIB_COMPAT mode so the ABI matches stock zlib (z_stream layout, + * deflateInit_/inflateInit_ symbols). zlib.h is GENERATED into the build dir + * by cmake configure — it does not exist in the source tree. */ -import type { Dependency, NestedCmakeBuild } from "../source.ts"; +import type { Dependency } from "../source.ts"; +import { depBuildDir } from "../source.ts"; -const ZLIB_COMMIT = "886098f3f339617b4243b286f5ed364b9989e245"; +// Pin to a release tag, not develop. Two regressions landed on develop AFTER +// 2.3.3 that are NOT present at this commit — re-audit before bumping past it: +// - 172b8544 (Apr 2026): inverted COPY guard disables Chorba CRC32 fast-path +// on PCLMULQDQ-only x64 (Westmere–Comet Lake, Zen1–Zen3) +// - e5129cfe (Jan 2026): deflateBound() hits __builtin_unreachable() after +// Z_FINISH (s->wrap negated to -1/-2, switch falls to default) +const ZLIB_COMMIT = "12731092979c6d07f42da27da673a9f6c7b13586"; // 2.3.3 export const zlib: Dependency = { name: "zlib", @@ -14,54 +25,51 @@ export const zlib: Dependency = { source: () => ({ kind: "github-archive", - repo: "cloudflare/zlib", + repo: "zlib-ng/zlib-ng", commit: ZLIB_COMMIT, }), - // All patches are unconditional. The remove-machine-x64 patch deletes - // an old workaround for a cmake bug fixed in 2011 — we require cmake - // 3.24+ so the workaround is pure dead weight. patches: [ - "patches/zlib/CMakeLists.txt.patch", - "patches/zlib/deflate.h.patch", - "patches/zlib/ucm.cmake.patch", - "scripts/build/patches/zlib/remove-machine-x64.patch", + // clang-cl defines _MSC_VER but needs clang's /, + // not MSVC's /. Upstream gates on _MSC_VER alone. + "patches/zlib/clang-cl-arm64.patch", ], - build: cfg => { - const spec: NestedCmakeBuild = { - kind: "nested-cmake", - targets: ["zlib"], - args: { - BUILD_EXAMPLES: "OFF", - }, - }; - - // Apple clang defines TARGET_OS_* macros that conflict with zlib's own - // platform detection (it has `#ifdef TARGET_OS_MAC` gates that predate - // apple's convention). The flag makes clang stop auto-defining them. - // See https://gitlab.kitware.com/cmake/cmake/-/issues/25755. - if (cfg.darwin) { - spec.extraCFlags = ["-fno-define-target-os-macros"]; - spec.extraCxxFlags = ["-fno-define-target-os-macros"]; - } - - return spec; - }, + build: () => ({ + kind: "nested-cmake", + targets: ["zlib-ng"], + args: { + ZLIB_COMPAT: "ON", + ZLIB_ENABLE_TESTS: "OFF", + WITH_GTEST: "OFF", + WITH_BENCHMARKS: "OFF", + // Runtime CPU detection: ship one binary that picks AVX512/AVX2/SSE2 + // at startup. WITH_NATIVE_INSTRUCTIONS would hard-code -march=native. + WITH_NATIVE_INSTRUCTIONS: "OFF", + WITH_RUNTIME_CPU_DETECTION: "ON", + // zlib-ng 340f2f6e moved infback.c's distance-too-far-back check behind + // INFLATE_STRICT (default OFF). Upstream zlib has it unconditional. Bun + // doesn't call inflateBack(), but anything in-process linking the same + // lib could read adjacent heap on a malicious raw-deflate stream with + // windowBits<15. Hardens at zero perf cost to inflate() proper. + WITH_INFLATE_STRICT: "ON", + BUILD_SHARED_LIBS: "OFF", + }, + }), provides: cfg => { - // zlib's OUTPUT_NAME logic: on unix/mingw → "z", on MSVC → "zlib" - // (or "zlibd" for CMAKE_BUILD_TYPE=Debug — cmake's Windows debug - // suffix convention). + // zlib-ng OUTPUT_NAME on unix → "z"; on MSVC → "zlibstatic" with + // CMAKE_DEBUG_POSTFIX "d" (set inside if(MSVC), which clang-cl satisfies). let lib: string; if (cfg.windows) { - lib = cfg.debug ? "zlibd" : "zlib"; + lib = cfg.debug ? "zlibstaticd" : "zlibstatic"; } else { lib = "z"; } return { libs: [lib], - includes: ["."], + // zlib.h + zconf.h are generated by cmake configure into the build dir. + includes: [depBuildDir(cfg, "zlib")], }; }, }; diff --git a/scripts/build/patches/zlib/remove-machine-x64.patch b/scripts/build/patches/zlib/remove-machine-x64.patch deleted file mode 100644 index 91058d8f8d6e..000000000000 --- a/scripts/build/patches/zlib/remove-machine-x64.patch +++ /dev/null @@ -1,25 +0,0 @@ -Remove workaround for CMake bug #11240 (2010, fixed in cmake 2.8.4). - -The hardcoded /machine:x64 breaks arm64 windows builds. The workaround is -long obsolete — lib.exe has always set /MACHINE automatically from the object -file architecture; the original bug was in cmake's static lib link command -line construction, not in lib.exe itself. - ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -231,13 +231,6 @@ - if(UNIX OR MINGW) - set_target_properties(zlib PROPERTIES OUTPUT_NAME z) - endif() -- #============================================================================ -- # work around to CMake bug which affects 64-bit Windows -- # see http://public.kitware.com/Bug/view.php?id=11240 -- #============================================================================ -- if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND MSVC) -- set_target_properties(zlib PROPERTIES STATIC_LIBRARY_FLAGS "/machine:x64") -- endif() - endif() - - if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL) - - diff --git a/scripts/build/source.ts b/scripts/build/source.ts index 22b6a3e7db92..bee06834a845 100644 --- a/scripts/build/source.ts +++ b/scripts/build/source.ts @@ -374,11 +374,15 @@ export interface Dependency { patches?: string[] | ((cfg: Config) => string[]); /** - * Other deps whose SOURCE must be ready before this dep's build runs. + * Other deps that must be BUILT before this dep's configure runs. * Used for header-level dependencies — e.g. libarchive needs zlib's - * headers at compile time (`-I${vendorDir}/zlib`), so zlib must be - * fetched first. This adds an order-only dep on the other dep's source - * stamp — it does NOT link the other dep's libs (that's `provides.libs`). + * headers at configure time (`check_include_file("zlib.h")`). zlib-ng + * generates `zlib.h` during its own cmake configure, so libarchive must + * wait for zlib's full build, not just its source fetch. + * + * Resolves to the named dep's build outputs (lib files for nested-cmake, + * source stamp for header-only). Order-only on configure, implicit on + * build. Does NOT link the other dep's libs (that's `provides.libs`). */ fetchDeps?: string[]; @@ -575,8 +579,8 @@ export function depSourceDir(cfg: Config, name: string): string { } /** - * Path to a dep's fetch stamp. Used by fetchDeps to add cross-dep - * ordering (e.g. libarchive's build waits for zlib's .ref). + * Path to a dep's fetch stamp. Used by zig-only mode to depend on zstd's + * source being on disk without resolving the full dep graph. */ export function depSourceStamp(cfg: Config, name: string): string { return resolve(depSourceDir(cfg, name), ".ref"); @@ -597,7 +601,12 @@ export function depBuildDir(cfg: Config, name: string): string { * If the dep is disabled (enabled() returns false), returns null. Caller * should skip. */ -export function resolveDep(n: Ninja, cfg: Config, dep: Dependency): ResolvedDep | null { +export function resolveDep( + n: Ninja, + cfg: Config, + dep: Dependency, + resolved: ReadonlyMap, +): ResolvedDep | null { if (dep.enabled && !dep.enabled(cfg)) { return null; } @@ -686,19 +695,23 @@ export function resolveDep(n: Ninja, cfg: Config, dep: Dependency): ResolvedDep } // ─── Resolve fetchDeps → extra inputs on configure + build ─── - // These are deps whose SOURCE must be ready before we build (not link). - // E.g. libarchive compiles with -I${vendorDir}/zlib so zlib must be fetched. + // These are deps that must be BUILT before we configure (not link). + // E.g. libarchive's configure runs check_include_file("zlib.h"), and + // zlib-ng generates zlib.h during its own cmake configure — so we depend + // on zlib's lib output (which implies its configure ran). // - // On CONFIGURE: order-only. Configure needs the headers to exist (for - // check_include_file), but doesn't track their content — feature - // detection results are cached in CMakeCache.txt regardless. + // On CONFIGURE: order-only. Configure needs the headers to exist, but + // doesn't track their content — feature detection is cached in + // CMakeCache.txt regardless. // - // On BUILD: implicit. If the cross-dep source is re-fetched (commit bump), - // its headers may have changed; our .o files track them via the inner - // ninja's .d files. We need to re-invoke `cmake --build` so the inner - // ninja can detect staleness. Restat on the build rule ensures that if - // the headers DIDN'T actually change, the inner no-op prunes downstream. - const fetchDepStamps = (dep.fetchDeps ?? []).map(d => depSourceStamp(cfg, d)); + // On BUILD: implicit. If the cross-dep rebuilds (commit bump), its + // headers may have changed; our .o files track them via the inner + // ninja's .d files. Restat prunes downstream when nothing changed. + const fetchDepStamps = (dep.fetchDeps ?? []).flatMap(d => { + const r = resolved.get(d); + assert(r, `${dep.name}: fetchDeps references '${d}' but it wasn't resolved first — fix allDeps ordering`); + return r.outputs; + }); // ─── Step 2+3: build ─── let libs: string[]; diff --git a/scripts/verify-baseline-static/allowlist-x64-windows.txt b/scripts/verify-baseline-static/allowlist-x64-windows.txt index fca82a5b4c3c..8e9460b96d70 100644 --- a/scripts/verify-baseline-static/allowlist-x64-windows.txt +++ b/scripts/verify-baseline-static/allowlist-x64-windows.txt @@ -656,3 +656,48 @@ wcsnlen [AVX, AVX2] # ---------------------------------------------------------------------------- _RINvYNtNt9hashbrown6hasher18DefaultHashBuilderNtNt4core4hash11BuildHasher8hash_oneRNtNtNt8lol_html4html10local_name9LocalNameEB1R_ [AVX, AVX2] [AVX, AVX2] + + +# ---------------------------------------------------------------------------- +# zlib-ng. Gate: init_functable() — cpu_check_features() CPUID at first call, +# results cached in `functable` (vendor/zlib/functable.c, arch/x86/x86_features.c). +# chunkcopy_safe is a static-inline in inflate_p.h that may outline from any of +# the per-ISA inffast_tpl.h instantiations; the outlined copy is local to that +# TU and only reachable from the gated inflate_fast_* of the same tier. +# (31 symbols) +# ---------------------------------------------------------------------------- +# has_pclmulqdq +crc32_pclmulqdq [PCLMULQDQ] +crc32_fold_pclmulqdq [PCLMULQDQ] +crc32_fold_pclmulqdq_copy [PCLMULQDQ] +crc32_fold_pclmulqdq_final [PCLMULQDQ] +crc32_fold_pclmulqdq_reset [PCLMULQDQ] +# has_avx2 && has_bmi2 +adler32_avx2 [AVX, AVX2, BMI2] +adler32_fold_copy_avx2 [AVX, AVX2, BMI2] +chunkmemset_avx2 [AVX, AVX2, BMI2] +chunkmemset_safe_avx2 [AVX, AVX2, BMI2] +compare256_avx2 [AVX, AVX2, BMI2] +inflate_fast_avx2 [AVX, AVX2, BMI2] +longest_match_avx2 [AVX, AVX2, BMI2] +longest_match_slow_avx2 [AVX, AVX2, BMI2] +slide_hash_avx2 [AVX, AVX2, BMI2] +# has_avx512_common (F+DQ+BW+VL+BMI2) +adler32_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +adler32_fold_copy_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +chunkmemset_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +chunkmemset_safe_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +compare256_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +inflate_fast_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +longest_match_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +longest_match_slow_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +chunkcopy_safe [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +# has_avx512vnni +adler32_avx512_vnni [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, AVX512_VNNI, BMI2] +adler32_fold_copy_avx512_vnni [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, AVX512_VNNI, BMI2] +# has_pclmulqdq && has_avx512_common && has_vpclmulqdq +crc32_vpclmulqdq [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq_copy [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq_final [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq_reset [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] diff --git a/scripts/verify-baseline-static/allowlist-x64.txt b/scripts/verify-baseline-static/allowlist-x64.txt index 9cd64dde57cb..535c2dcfcba7 100644 --- a/scripts/verify-baseline-static/allowlist-x64.txt +++ b/scripts/verify-baseline-static/allowlist-x64.txt @@ -642,10 +642,48 @@ deflate_decompress_bmi2 [BMI2] # ---------------------------------------------------------------------------- -# zlib crc32 PCLMULQDQ. -# (1 symbols) -# ---------------------------------------------------------------------------- -crc32 [PCLMULQDQ] +# zlib-ng. Gate: init_functable() — cpu_check_features() CPUID at first call, +# results cached in `functable` (vendor/zlib/functable.c, arch/x86/x86_features.c). +# chunkcopy_safe is a static-inline in inflate_p.h that may outline from any of +# the per-ISA inffast_tpl.h instantiations; the outlined copy is local to that +# TU and only reachable from the gated inflate_fast_* of the same tier. +# (31 symbols) +# ---------------------------------------------------------------------------- +# has_pclmulqdq +crc32_pclmulqdq [PCLMULQDQ] +crc32_fold_pclmulqdq [PCLMULQDQ] +crc32_fold_pclmulqdq_copy [PCLMULQDQ] +crc32_fold_pclmulqdq_final [PCLMULQDQ] +crc32_fold_pclmulqdq_reset [PCLMULQDQ] +# has_avx2 && has_bmi2 +adler32_avx2 [AVX, AVX2, BMI2] +adler32_fold_copy_avx2 [AVX, AVX2, BMI2] +chunkmemset_avx2 [AVX, AVX2, BMI2] +chunkmemset_safe_avx2 [AVX, AVX2, BMI2] +compare256_avx2 [AVX, AVX2, BMI2] +inflate_fast_avx2 [AVX, AVX2, BMI2] +longest_match_avx2 [AVX, AVX2, BMI2] +longest_match_slow_avx2 [AVX, AVX2, BMI2] +slide_hash_avx2 [AVX, AVX2, BMI2] +# has_avx512_common (F+DQ+BW+VL+BMI2) +adler32_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +adler32_fold_copy_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +chunkmemset_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +chunkmemset_safe_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +compare256_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +inflate_fast_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +longest_match_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +longest_match_slow_avx512 [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +chunkcopy_safe [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2] +# has_avx512vnni +adler32_avx512_vnni [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, AVX512_VNNI, BMI2] +adler32_fold_copy_avx512_vnni [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, AVX512_VNNI, BMI2] +# has_pclmulqdq && has_avx512_common && has_vpclmulqdq +crc32_vpclmulqdq [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq_copy [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq_final [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] +crc32_fold_vpclmulqdq_reset [AVX, AVX2, AVX512BW, AVX512DQ, AVX512F, AVX512VL, BMI2, PCLMULQDQ, VPCLMULQDQ] # ---------------------------------------------------------------------------- diff --git a/src/bun.js/api/HashObject.zig b/src/bun.js/api/HashObject.zig index 92b3f6da4e55..2319dbe19b60 100644 --- a/src/bun.js/api/HashObject.zig +++ b/src/bun.js/api/HashObject.zig @@ -6,7 +6,7 @@ pub const adler32 = hashWrap(std.hash.Adler32); pub const crc32 = hashWrap(struct { pub fn hash(seed: u32, bytes: []const u8) u32 { // zlib takes a 32-bit length, so chunk large inputs to avoid truncation. - var crc: u64 = seed; + var crc: bun.zlib.uLong = seed; var offset: usize = 0; while (offset < bytes.len) { const remaining = bytes.len - offset; diff --git a/src/crash_handler.zig b/src/crash_handler.zig index cb88ae42f56b..58a6a191bdcf 100644 --- a/src/crash_handler.zig +++ b/src/crash_handler.zig @@ -1340,10 +1340,10 @@ fn encodeTraceString(opts: TraceString, writer: anytype) !void { try writer.writeByte('0'); var compressed_bytes: [2048]u8 = undefined; - var len: usize = compressed_bytes.len; - const ret: bun.zlib.ReturnCode = @enumFromInt(bun.zlib.compress2(&compressed_bytes, &len, message.ptr, message.len, 9)); + var len: bun.zlib.uLong = compressed_bytes.len; + const ret: bun.zlib.ReturnCode = @enumFromInt(bun.zlib.compress2(&compressed_bytes, &len, message.ptr, @intCast(message.len), 9)); const compressed = switch (ret) { - .Ok => compressed_bytes[0..len], + .Ok => compressed_bytes[0..@intCast(len)], // Insufficient memory. .MemError => return error.OutOfMemory, // The buffer dest was not large enough to hold the compressed data. diff --git a/src/deps/zlib.win32.zig b/src/deps/zlib.win32.zig index 458a97ce53f3..b1cfbd91c154 100644 --- a/src/deps/zlib.win32.zig +++ b/src/deps/zlib.win32.zig @@ -9,7 +9,11 @@ pub const off_t = _off_t; const voidpf = *anyopaque; const Bytef = [*]u8; const uInt = c_uint; -const uLong = u64; +// zlib-ng compat (and stock zlib) typedef uLong as `unsigned long` — 4 bytes on +// Windows LLP64. cloudflare/zlib used uint64_t, which is why this was u64. With +// the wrong width, sizeof(z_stream) mismatches and inflateInit_/deflateInit_ +// return Z_VERSION_ERROR. +const uLong = c_ulong; const z_size_t = usize; const uLongf = uLong; const voidpc = ?*const anyopaque; @@ -144,12 +148,6 @@ pub extern fn inflateResetKeep(z_streamp) ReturnCode; pub extern fn deflateResetKeep(z_streamp) ReturnCode; pub const z_off_t = c_long; -pub const ZLIB_VERSION = "1.2.13"; -pub const ZLIB_VERNUM = @as(c_int, 0x12d0); -pub const ZLIB_VER_MAJOR = @as(c_int, 1); -pub const ZLIB_VER_MINOR = @as(c_int, 2); -pub const ZLIB_VER_REVISION = @as(c_int, 13); -pub const ZLIB_VER_SUBREVISION = @as(c_int, 0); pub const Z_NO_FLUSH = @as(c_int, 0); pub const Z_PARTIAL_FLUSH = @as(c_int, 1); pub const Z_SYNC_FLUSH = @as(c_int, 2); diff --git a/src/zlib.zig b/src/zlib.zig index 458c20fb6d0e..a5d21dcd6ada 100644 --- a/src/zlib.zig +++ b/src/zlib.zig @@ -19,9 +19,13 @@ pub const gzFile = [*c]struct_gzFile_s; // https://zlib.net/manual.html#Stream const Byte = u8; const uInt = u32; -const uLong = u64; +// zlib-ng compat (and stock zlib) use `unsigned long` — 4 bytes on Windows +// LLP64, 8 on LP64 unix. cloudflare/zlib hard-coded uint64_t, which is why +// this was u64. Must match the C side or compress()/compressBound()/adler32() +// have an ABI mismatch on Windows. +pub const uLong = c_ulong; const Bytef = Byte; -const uLongf = uLong; +pub const uLongf = uLong; const voidpf = ?*anyopaque; // typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); @@ -677,7 +681,7 @@ pub extern fn deflateReset(stream: z_streamp) ReturnCode; // to return Z_STREAM_END. Note that it is possible for the compressed size to // be larger than the value returned by deflateBound() if flush options other // than Z_FINISH or Z_NO_FLUSH are used. -pub extern fn deflateBound(strm: z_streamp, sourceLen: u64) u64; +pub extern fn deflateBound(strm: z_streamp, sourceLen: uLong) uLong; /// /// This is another version of deflateInit with more compression options. The @@ -840,7 +844,7 @@ pub const ZlibCompressorArrayList = struct { @sizeOf(zStream_struct), )) { ReturnCode.Ok => { - zlib_reader.list.ensureTotalCapacityPrecise(list_allocator, deflateBound(&zlib_reader.zlib, input.len)) catch { + zlib_reader.list.ensureTotalCapacityPrecise(list_allocator, deflateBound(&zlib_reader.zlib, @intCast(input.len))) catch { zlib_reader.deinit(); return error.OutOfMemory; }; diff --git a/test/js/node/process/process.test.js b/test/js/node/process/process.test.js index 74745394411f..597aafe3b6d0 100644 --- a/test/js/node/process/process.test.js +++ b/test/js/node/process/process.test.js @@ -277,7 +277,7 @@ it("process.versions", () => { libarchive: "ded82291ab41d5e355831b96b0e1ff49e24d8939", mimalloc: "d56d5b27f8f9bfcd2d2f5f1020ff5e100781b062", picohttpparser: "066d2b1e9ab820703db0837a7255d92d30f0c9f5", - zlib: "886098f3f339617b4243b286f5ed364b9989e245", + zlib: "12731092979c6d07f42da27da673a9f6c7b13586", tinycc: "12882eee073cfe5c7621bcfadf679e1372d4537b", lolhtml: "77127cd2b8545998756e8d64e36ee2313c4bb312", ares: "3ac47ee46edd8ea40370222f91613fc16c434853",