ICU: per-item zstd decompression hook for compressed libicudata

scripts/build.ts

@@ -373,6 +373,7 @@ function parseArgs(argv: string[]): CliArgs {
     "timeTrace",
     "ci",
     "buildkite",
+    "allowPreviewWebkit",
   ]);
   // PartialConfig fields that are STRINGS.
   const stringFields = new Set([

scripts/build/config.ts

@@ -273,6 +273,8 @@ export interface Config {
   nodejsAbiVersion: string;
   /** WebKit commit. Default in versions.ts; override to test a WebKit branch. */
   webkitVersion: string;
+  /** Permit `webkitVersion` to be an `autobuild-preview-*` PR tag. */
+  allowPreviewWebkit: boolean;
 }
 
 /**
@@ -318,6 +320,7 @@ export interface PartialConfig {
   nodejsVersion?: string;
   nodejsAbiVersion?: string;
   webkitVersion?: string;
+  allowPreviewWebkit?: boolean;
 }
 
 /**
@@ -821,6 +824,7 @@ export function resolveConfig(partial: PartialConfig, toolchain: Toolchain): Con
   const nodejsVersion = partial.nodejsVersion ?? versionDefaults.nodejsVersion;
   const nodejsAbiVersion = partial.nodejsAbiVersion ?? versionDefaults.nodejsAbiVersion;
   const webkitVersion = partial.webkitVersion ?? versionDefaults.webkitVersion;
+  const allowPreviewWebkit = partial.allowPreviewWebkit ?? false;
 
   // ─── macOS SDK ───
   // Must be passed to nested cmake builds or they'll pick the wrong SDK.
@@ -917,6 +921,7 @@ export function resolveConfig(partial: PartialConfig, toolchain: Toolchain): Con
     nodejsAbiVersion,
     canaryRevision,
     webkitVersion,
+    allowPreviewWebkit,
   };
 }
 

scripts/build/deps/webkit.ts

@@ -3,7 +3,12 @@
  * for local mode. Override via `--webkit-version=<hash>` to test a branch.
  * From https://github.com/oven-sh/WebKit releases.
  */
-export const WEBKIT_VERSION = "3167a44fb92c268c83f09b232b38a9f3e7f9655a";
+// TEMPORARY: preview build of oven-sh/WebKit#237 (per-item zstd compression
+// of ICU display-name trees). Pinned to the pre-#236 preview commit because
+// Bun's bindings haven't adapted to #236's JSC API changes yet (bun#31169).
+// Replace with the real autobuild-<sha> once both #237 and bun#31169 land.
+// DO NOT MERGE with this value.
+export const WEBKIT_VERSION = "autobuild-preview-pr-237-72c07745";
 
 /**
  * WebKit (JavaScriptCore) — the JS engine.
@@ -42,6 +47,7 @@ export const WEBKIT_VERSION = "3167a44fb92c268c83f09b232b38a9f3e7f9655a";
 import { homedir } from "node:os";
 import { join, resolve } from "node:path";
 import type { Config } from "../config.ts";
+import { BuildError } from "../error.ts";
 import { computeCpuTargetFlags } from "../flags.ts";
 import { slash } from "../shell.ts";
 import { type Dependency, type NestedCmakeBuild, type Source, depBuildDir, depSourceDir } from "../source.ts";
@@ -74,6 +80,13 @@ function prebuiltUrl(cfg: Config): string {
   const arch = cfg.arm64 ? "arm64" : "amd64";
   const name = `bun-webkit-${os}-${arch}${prebuiltSuffix(cfg)}`;
   const version = cfg.webkitVersion;
+  if (version.startsWith("autobuild-preview-") && !cfg.allowPreviewWebkit) {
+    throw new BuildError(
+      `WEBKIT_VERSION is pinned to a PR preview tag (${version}). ` +
+        `Preview prebuilts are deleted when the PR closes. ` +
+        `Pass --allow-preview-webkit=on to build anyway, or revert to a stable hash before merging.`
+    );
+  }
   const tag = version.startsWith("autobuild-") ? version : `autobuild-${version}`;
   return `https://github.com/oven-sh/WebKit/releases/download/${tag}/${name}.tar.gz`;
 }
@@ -83,7 +96,9 @@ function prebuiltUrl(cfg: Config): string {
  * doesn't reuse a wrong-ABI extraction.
  */
 function prebuiltDestDir(cfg: Config): string {
-  const version16 = cfg.webkitVersion.slice(0, 16);
+  // Strip the autobuild- prefix (matches prebuiltUrl) so two different
+  // preview/tag pins don't collapse to the same "autobuild-previe" cache key.
+  const version16 = cfg.webkitVersion.replace(/^autobuild-/, "").slice(0, 16);
   // Cross-compiled targets share a host (and cache dir) with native builds,
   // so include os+arch in the key — otherwise a FreeBSD/arm64 extraction
   // collides with a Linux/x64 one at the same WebKit version.

src/jsc/bindings/bun_icu_decompress.cpp

@@ -0,0 +1,118 @@
+// Per-item zstd decompression hook for ICU common data.
+//
+// oven-sh/WebKit's ICU build (icu/udata-decompress-hook.patch) inserts a weak
+// call to bun_icu_maybe_decompress between TOC lookup and checkDataItem.
+// Display-name items (curr/ lang/ region/ unit/ zone/, non-en) are stored as
+// raw zstd frames; everything else keeps its 0xda27 header and passes through
+// after one u32 compare. Decompressed buffers are cached for the process
+// lifetime, keyed by their .rodata address.
+//
+// The dict symbols are emitted by the repacked libicudata.a; declaring them
+// weak here lets this file link against a prebuilt that predates the repack
+// (the hook is then never called, since no item is compressed).
+
+#include "root.h"
+
+// The repacked libicudata.a (and the patched udata.cpp that calls this hook)
+// are produced by oven-sh/WebKit's Dockerfile / Dockerfile.musl only. On every
+// other platform ICU is unmodified, so there is nothing to decompress and the
+// weak externs below have no definer — gate the whole implementation to keep
+// non-ELF weak-symbol semantics out of the picture.
+#if OS(LINUX)
+
+#include "MimallocWTFMalloc.h"
+
+#include <wtf/HashMap.h>
+#include <wtf/Lock.h>
+#include <wtf/NeverDestroyed.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+
+static_assert(ZSTD_MAGICNUMBER == 0xFD2FB528);
+// Raw ICU items have bytes[2..3] == {0xda, 0x27} (ucmndata.h MAGIC1/MAGIC2),
+// so their first u32 is 0x27da'hhhh — cannot collide with zstd's magic.
+
+extern "C" __attribute__((weak)) const unsigned char bun_icu_zstd_dict[];
+extern "C" __attribute__((weak)) const unsigned int bun_icu_zstd_dict_size;
+
+namespace Bun {
+
+class ICUDecompressor {
+public:
+    static ICUDecompressor& get()
+    {
+        static LazyNeverDestroyed<ICUDecompressor> instance;
+        static std::once_flag once;
+        std::call_once(once, [] { instance.construct(); });
+        return instance.get();
+    }
+
+    const void* decompress(const void* p, int32_t* length)
+    {
+        Locker locker { m_lock };
+
+        if (auto it = m_cache.find(p); it != m_cache.end()) {
+            *length = static_cast<int32_t>(ZSTD_getFrameContentSize(p, frameBound(*length)));
+            return it->value;
+        }
+
+        size_t clen = ZSTD_findFrameCompressedSize(p, frameBound(*length));
+        if (ZSTD_isError(clen))
+            return p;
+        auto dlen = ZSTD_getFrameContentSize(p, clen);
+        if (dlen == ZSTD_CONTENTSIZE_UNKNOWN || dlen == ZSTD_CONTENTSIZE_ERROR)
+            return p;
+
+        // tryAlignedMalloc asserts size is a multiple of alignment in debug
+        // builds; ICU item sizes are only 4-aligned, so round up.
+        size_t alloc = WTF::roundUpToMultipleOf<16>(static_cast<size_t>(dlen));
+        void* buf = MimallocMalloc::tryAlignedMalloc(alloc, 16);
+        if (!buf)
+            return p;
+        size_t r = m_ddict
+            ? ZSTD_decompress_usingDDict(m_dctx, buf, static_cast<size_t>(dlen), p, clen, m_ddict)
+            : ZSTD_decompressDCtx(m_dctx, buf, static_cast<size_t>(dlen), p, clen);
+        if (ZSTD_isError(r)) {
+            MimallocMalloc::free(buf);
+            return p;
+        }
+
+        m_cache.add(p, buf);
+        *length = static_cast<int32_t>(dlen);
+        return buf;
+    }
+
+private:
+    ICUDecompressor()
+        : m_dctx(ZSTD_createDCtx())
+        , m_ddict(&bun_icu_zstd_dict_size && bun_icu_zstd_dict_size
+                  ? ZSTD_createDDict_byReference(bun_icu_zstd_dict, bun_icu_zstd_dict_size)
+                  : nullptr)
+    {
+    }
+
+    static size_t frameBound(int32_t tocLength) { return tocLength > 0 ? static_cast<size_t>(tocLength) : (1u << 20); }
+
+    friend class WTF::LazyNeverDestroyed<ICUDecompressor>;
+
+    WTF::Lock m_lock;
+    WTF::HashMap<const void*, void*> m_cache WTF_GUARDED_BY_LOCK(m_lock);
+    ZSTD_DCtx* const m_dctx;
+    ZSTD_DDict* const m_ddict;
+};
+
+} // namespace Bun
+
+extern "C" const void* bun_icu_maybe_decompress(const void* p, int32_t* length)
+{
+    if (!p)
+        return p;
+    uint32_t magic;
+    std::memcpy(&magic, p, sizeof(magic));
+    if (magic != ZSTD_MAGICNUMBER) [[likely]]
+        return p;
+    return Bun::ICUDecompressor::get().decompress(p, length);
+}
+
+#endif // OS(LINUX)