Rewrite the JSON parser around a SIMD structural index

bench/json-corpus/.gitignore

@@ -0,0 +1 @@
+*.json

bench/json-corpus/fetch.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Fetch the JSON benchmark fixture corpus: real package.json files plus npm registry responses
+# (abbreviated install-v1 manifests and full packuments). Gitignored; only compared within a checkout.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+ACCEPT_ABBREV='Accept: application/vnd.npm.install-v1+json'
+
+pkg() { curl -fsSL "https://unpkg.com/$1@$2/package.json" -o "pkgjson-$1.json"; echo "pkgjson-$1.json"; }
+pkg express 4.19.2
+pkg react 18.3.1
+pkg typescript 5.5.4
+pkg webpack 5.93.0
+pkg eslint 9.8.0
+pkg axios 1.7.3
+pkg next 14.2.5
+pkg vite 5.3.5
+
+abbrev() { curl -fsSL -H "$ACCEPT_ABBREV" "https://registry.npmjs.org/$1" -o "manifest-abbrev-$2.json"; echo "manifest-abbrev-$2.json"; }
+abbrev is-number is-number
+abbrev express express
+abbrev react react
+abbrev lodash lodash
+abbrev typescript typescript
+abbrev next next
+abbrev @types/node types-node
+abbrev @babel/core babel-core
+
+full() { curl -fsSL "https://registry.npmjs.org/$1" -o "packument-full-$2.json"; echo "packument-full-$2.json"; }
+full express express
+full axios axios
+
+abbrev drizzle-orm drizzle-orm
+abbrev drizzle-kit drizzle-kit
+
+ls -la

bench/json-corpus/jsonc-parse.mjs

@@ -0,0 +1,47 @@
+// End-to-end benchmark of `Bun.JSONC.parse` over the corpus in this directory, with `JSON.parse`
+// as a reference. Usage: `bun bench/json-corpus/jsonc-parse.mjs [name-filter]`.
+import { readdirSync, readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+
+const dir = dirname(Bun.fileURLToPath(import.meta.url));
+const filter = process.argv[2] ?? "";
+const files = readdirSync(dir)
+  .filter(f => f.endsWith(".json") && f.includes(filter))
+  .sort();
+
+const runs = +(process.env.RUNS ?? 30);
+function bench(label, fn) {
+  fn();
+  fn();
+  const times = [];
+  for (let i = 0; i < runs; i++) {
+    const t0 = Bun.nanoseconds();
+    fn();
+    times.push(Bun.nanoseconds() - t0);
+  }
+  times.sort((a, b) => a - b);
+  return { label, best: times[0], median: times[(times.length / 2) | 0] };
+}
+
+console.log("bun", Bun.version, Bun.revision.slice(0, 9));
+console.log(
+  "file".padEnd(40),
+  "bytes".padStart(10),
+  "JSONC med".padStart(12),
+  "JSON.parse".padStart(12),
+  "JSONC MiB/s".padStart(11),
+);
+for (const f of files) {
+  const text = readFileSync(join(dir, f), "utf8");
+  const bytes = Buffer.byteLength(text);
+  const jsonc = bench("jsonc", () => Bun.JSONC.parse(text));
+  const jsonp = bench("json", () => JSON.parse(text));
+  const mbs = bytes / (jsonc.median / 1e9) / (1 << 20);
+  console.log(
+    f.padEnd(40),
+    String(bytes).padStart(10),
+    (jsonc.median / 1e6).toFixed(3).padStart(9) + " ms",
+    (jsonp.median / 1e6).toFixed(3).padStart(9) + " ms",
+    mbs.toFixed(0).padStart(11),
+  );
+}

scripts/bench-json-rust.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Build + run the JSON parser criterion bench (src/parsers/benches/json_parse.rs): compiles the
+# native pieces the parser reaches into one archive and points RUSTFLAGS at it. Needs `bun bd` once.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ ! -d vendor/mimalloc ] || [ ! -d vendor/highway ]; then
+  echo "error: vendor/ not populated — run \`bun bd\` (or \`bun run build\`) once first" >&2
+  exit 1
+fi
+
+SUP=target/bench-json-cdeps
+mkdir -p "$SUP/wtf"
+export BUN_CODEGEN_DIR=${BUN_CODEGEN_DIR:-$PWD/build/debug/codegen}
+if [ ! -f "$BUN_CODEGEN_DIR/json_byte_class.h" ]; then
+  echo "error: $BUN_CODEGEN_DIR/json_byte_class.h not found — run \`bun bd\` once first" >&2
+  exit 1
+fi
+CC=${CC:-cc}
+CXX=${CXX:-c++}
+SIMDUTF_VERSION=v7.3.6
+SIMDUTF_STAMP="$SUP/.simdutf-$SIMDUTF_VERSION"
+if [ ! -f "$SIMDUTF_STAMP" ] || [ ! -f "$SUP/simdutf.cpp" ]; then
+  curl -fsSL -o "$SUP/simdutf.cpp" "https://github.com/simdutf/simdutf/releases/download/$SIMDUTF_VERSION/simdutf.cpp"
+  curl -fsSL -o "$SUP/simdutf.h" "https://github.com/simdutf/simdutf/releases/download/$SIMDUTF_VERSION/simdutf.h"
+  rm -f "$SUP"/simdutf*.o "$SUP"/.simdutf-*
+  touch "$SIMDUTF_STAMP"
+fi
+printf '#pragma once\n#include "simdutf.h"\n' > "$SUP/wtf/SIMDUTF.h"
+
+build() {
+  local out=$1
+  shift
+  if [ ! -f "$out" ] || [ "${*: -1}" -nt "$out" ]; then "$@" -o "$out"; fi
+}
+MI_FLAGS=(-O2 -fPIC -ftls-model=initial-exec -DNDEBUG -D_GNU_SOURCE -DMI_STATIC_LIB
+  -DMI_SKIP_COLLECT_ON_EXIT=1 -DMI_DEFAULT_ALLOW_THP=0 -DMI_NO_SET_VMA_NAME=1)
+build "$SUP/mimalloc.o" $CC "${MI_FLAGS[@]}" -Ivendor/mimalloc/include -c vendor/mimalloc/src/static.c
+build "$SUP/simdutf.o" $CXX -O3 -fPIC -std=c++20 -I"$SUP" -c "$SUP/simdutf.cpp"
+build "$SUP/simdutf_shim.o" $CXX -O3 -fPIC -std=c++20 -I"$SUP" -c src/parsers/benches/support/simdutf_shim.cpp
+for f in abort targets per_target print timer nanobenchmark aligned_allocator; do
+  build "$SUP/hwy_$f.o" $CXX -O3 -fPIC -std=c++17 -Ivendor/highway -c "vendor/highway/hwy/$f.cc"
+done
+if [ -f src/jsc/bindings/highway_json.cpp ]; then
+  $CXX -O3 -fPIC -std=c++17 -Ivendor/highway -Isrc/jsc/bindings -I"$BUN_CODEGEN_DIR" -c src/jsc/bindings/highway_json.cpp -o "$SUP/highway_json.o"
+fi
+rm -f "$SUP/libbun_bench_cdeps.a"
+ar rcs "$SUP/libbun_bench_cdeps.a" "$SUP"/*.o
+ranlib "$SUP/libbun_bench_cdeps.a"
+
+export MIMALLOC_PURGE_DELAY=${MIMALLOC_PURGE_DELAY:-2000}
+export BUN_JSON_BENCH_FIXTURES=${BUN_JSON_BENCH_FIXTURES:-$PWD/bench/json-corpus}
+CXXLIB=stdc++
+[ "$(uname -s)" = Darwin ] && CXXLIB=c++
+export RUSTFLAGS="${RUSTFLAGS:-} -Clink-arg=$PWD/$SUP/libbun_bench_cdeps.a -Clink-arg=-l$CXXLIB -Clink-arg=-lm -Clink-arg=-ldl -Clink-arg=-lpthread -Clink-arg=-lc"
+
+if [ "${1:-}" = "--test" ]; then
+  shift
+  exec cargo test -p bun_parsers --lib --release "$@"
+fi
+exec cargo bench -p bun_parsers --bench json_parse "$@"

scripts/build/CLAUDE.md

@@ -173,42 +173,43 @@ Split CI modes: `rust-only` (lolhtml+codegen+cargo → libbun_rust.a), `cpp-only
 
 ## Module inventory
 
-| File                           | Owns                                                                                                  |
-| ------------------------------ | ----------------------------------------------------------------------------------------------------- |
-| `build.ts` (parent dir)        | CLI entry — parse args, call configure, spawn ninja, optionally exec                                  |
-| `configure.ts`                 | `configure()` — toolchain → config → `build.ninja`                                                    |
-| `config.ts`                    | `Config`/`PartialConfig`/`Toolchain`/`Host` types, `resolveConfig()`                                  |
-| `profiles.ts`                  | Named `PartialConfig` presets + `getProfile()`                                                        |
-| `tools.ts`                     | Tool discovery: `findTool()`, `resolveLlvmToolchain()`, version parsing                               |
-| `flags.ts`                     | Flat flag tables, `computeFlags()`, `computeDepFlags()`, `computeCpuTargetFlags()`                    |
-| `ninja.ts`                     | `Ninja` class — the build-file writer                                                                 |
-| `rules.ts`                     | `registerAllRules()` — calls each module's `registerXxxRules()`                                       |
-| `compile.ts`                   | `cc`/`cxx`/`pch`/`link`/`ar` + `registerCompileRules()`                                               |
-| `unified.ts`                   | WebKit-style unified-source bundling, `generateUnifiedSources()`                                      |
-| `source.ts`                    | `Dependency` types, `resolveDep()`, fetch/configure/build emission                                    |
-| `codegen.ts`                   | Code generation steps, `emitCodegen()`, `CodegenOutputs`                                              |
-| `rust.ts`                      | `cargo build` step, `emitRust()`, `rustLibPath()`, cross-compile matrix                               |
-| `cargo-config.ts`              | Generates the git-ignored `.cargo/config.toml` (per-target `linker` from `cfg.hostCxx`)               |
-| `bun.ts`                       | `emitBun()` — assembles deps+codegen+rust+compile+link                                                |
-| `shims.ts`                     | Platform/toolchain workaround dylibs, `emitShims()`                                                   |
-| `workarounds.ts`               | Self-obsoleting workaround registry, `checkWorkarounds()`                                             |
-| `macos-sdk.ts`                 | macOS SDK resolution/download for darwin cross-compiles — `resolveMacosSdkPath()`, `ensureMacosSdk()` |
-| `features-json.ts`             | Host-side `features.json` for cross lanes — `parsePackedFeaturesList()`, `crossFeaturesJson()`        |
-| `depVersionsHeader.ts`         | Generates `bun_dependency_versions.h` for `process.versions`                                          |
-| `buildOptionsRs.ts`            | Generates `build_options.rs` (`bun_core::build_options`) from `Config`                                |
-| `stream.ts`                    | Subprocess output wrapper — FD-3 sideband, prefixed line streaming                                    |
-| `shell.ts`                     | `quote()`/`slash()` — shell escaping for ninja commands                                               |
-| `fs.ts`                        | `writeIfChanged()`, `mkdirAll()`                                                                      |
-| `error.ts`                     | `BuildError` with hint/file/cause, `assert()`                                                         |
-| `download.ts`                  | `downloadWithRetry()`, archive extraction                                                             |
-| `winsysroot.ts`                | Windows MSVC CRT + SDK sysroot (xwin): validates, adds case aliases, CI fetch                         |
-| `fetch-cli.ts`                 | Build-time CLI ninja invokes for downloads                                                            |
-| `ci.ts`                        | CI integration — annotations, artifacts, log groups                                                   |
-| `clean.ts`                     | `bun run clean` preset-based cleanup                                                                  |
-| `glob-sources.ts` (parent dir) | Source glob patterns + CLI to print them                                                              |
-| `deps/*.ts`                    | One `Dependency` object per vendored dep                                                              |
-| `deps/index.ts`                | `allDeps` array — fetch order + link order                                                            |
-| `shims/*.c`                    | Platform workaround sources                                                                           |
+| File                           | Owns                                                                                                                    |
+| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------- |
+| `build.ts` (parent dir)        | CLI entry — parse args, call configure, spawn ninja, optionally exec                                                    |
+| `configure.ts`                 | `configure()` — toolchain → config → `build.ninja`                                                                      |
+| `config.ts`                    | `Config`/`PartialConfig`/`Toolchain`/`Host` types, `resolveConfig()`                                                    |
+| `profiles.ts`                  | Named `PartialConfig` presets + `getProfile()`                                                                          |
+| `tools.ts`                     | Tool discovery: `findTool()`, `resolveLlvmToolchain()`, version parsing                                                 |
+| `flags.ts`                     | Flat flag tables, `computeFlags()`, `computeDepFlags()`, `computeCpuTargetFlags()`                                      |
+| `ninja.ts`                     | `Ninja` class — the build-file writer                                                                                   |
+| `rules.ts`                     | `registerAllRules()` — calls each module's `registerXxxRules()`                                                         |
+| `compile.ts`                   | `cc`/`cxx`/`pch`/`link`/`ar` + `registerCompileRules()`                                                                 |
+| `unified.ts`                   | WebKit-style unified-source bundling, `generateUnifiedSources()`                                                        |
+| `source.ts`                    | `Dependency` types, `resolveDep()`, fetch/configure/build emission                                                      |
+| `codegen.ts`                   | Code generation steps, `emitCodegen()`, `CodegenOutputs`                                                                |
+| `rust.ts`                      | `cargo build` step, `emitRust()`, `rustLibPath()`, cross-compile matrix                                                 |
+| `cargo-config.ts`              | Generates the git-ignored `.cargo/config.toml` (per-target `linker` from `cfg.hostCxx`)                                 |
+| `bun.ts`                       | `emitBun()` — assembles deps+codegen+rust+compile+link                                                                  |
+| `shims.ts`                     | Platform/toolchain workaround dylibs, `emitShims()`                                                                     |
+| `workarounds.ts`               | Self-obsoleting workaround registry, `checkWorkarounds()`                                                               |
+| `macos-sdk.ts`                 | macOS SDK resolution/download for darwin cross-compiles — `resolveMacosSdkPath()`, `ensureMacosSdk()`                   |
+| `features-json.ts`             | Host-side `features.json` for cross lanes — `parsePackedFeaturesList()`, `crossFeaturesJson()`                          |
+| `depVersionsHeader.ts`         | Generates `bun_dependency_versions.h` for `process.versions`                                                            |
+| `buildOptionsRs.ts`            | Generates `build_options.rs` (`bun_core::build_options`) from `Config`                                                  |
+| `jsonByteClass.ts`             | Generates `json_byte_class.{h,rs}` — the JSON byte classification shared by the SIMD kernel and the Rust scalar indexer |
+| `stream.ts`                    | Subprocess output wrapper — FD-3 sideband, prefixed line streaming                                                      |
+| `shell.ts`                     | `quote()`/`slash()` — shell escaping for ninja commands                                                                 |
+| `fs.ts`                        | `writeIfChanged()`, `mkdirAll()`                                                                                        |
+| `error.ts`                     | `BuildError` with hint/file/cause, `assert()`                                                                           |
+| `download.ts`                  | `downloadWithRetry()`, archive extraction                                                                               |
+| `winsysroot.ts`                | Windows MSVC CRT + SDK sysroot (xwin): validates, adds case aliases, CI fetch                                           |
+| `fetch-cli.ts`                 | Build-time CLI ninja invokes for downloads                                                                              |
+| `ci.ts`                        | CI integration — annotations, artifacts, log groups                                                                     |
+| `clean.ts`                     | `bun run clean` preset-based cleanup                                                                                    |
+| `glob-sources.ts` (parent dir) | Source glob patterns + CLI to print them                                                                                |
+| `deps/*.ts`                    | One `Dependency` object per vendored dep                                                                                |
+| `deps/index.ts`                | `allDeps` array — fetch order + link order                                                                              |
+| `shims/*.c`                    | Platform workaround sources                                                                                             |
 
 ## Key types
 

scripts/build/bun.ts

@@ -300,6 +300,14 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
   // force-include would lock those in before the source can speak.
   const noPchSources = new Set<string>();
 
+  // highway_json.cpp is compiled -O2 even in debug profiles (see its
+  // fileOverrides entry in flags.ts); a TU at a different -O level than the
+  // PCH cannot use the PCH ("__OPTIMIZE__ ... was disabled in precompiled
+  // file"). It only includes highway + libc headers anyway.
+  if (cfg.debug) {
+    noPchSources.add(resolve(cfg.cwd, "src/jsc/bindings/highway_json.cpp"));
+  }
+
   // Windows-only cpp sources (rescle — PE resource editor for --compile).
   if (cfg.windows) {
     // rescle.h does `#define UNICODE` before including ATL; with PCH the

scripts/build/codegen.ts

@@ -41,6 +41,7 @@ import { generateBuildOptionsRs } from "./buildOptionsRs.ts";
 import type { Config } from "./config.ts";
 import { BuildError, assert } from "./error.ts";
 import { writeIfChanged } from "./fs.ts";
+import { generateJsonByteClass } from "./jsonByteClass.ts";
 import type { Ninja } from "./ninja.ts";
 import { quote, quoteArgs } from "./shell.ts";
 
@@ -272,6 +273,13 @@ export function emitCodegen(n: Ninja, cfg: Config, sources: Sources): CodegenOut
   o.all.push(buildOptionsRs);
   o.rustInputs.push(buildOptionsRs);
 
+  // Same shape: the JSON byte-classification tables, consumed by both the
+  // Highway kernel (.h) and the Rust scalar indexer (.rs).
+  const jsonByteClass = generateJsonByteClass(cfg);
+  o.all.push(jsonByteClass.h, jsonByteClass.rs);
+  o.rustInputs.push(jsonByteClass.rs);
+  o.cppHeaders.push(jsonByteClass.h);
+
   emitBunError(ctx);
   emitStringMaps(ctx);
   emitFallbackDecoder(ctx);

scripts/build/flags.ts

@@ -1536,6 +1536,17 @@ export const fileOverrides: FileOverride[] = [
     when: c => c.windows,
     desc: "Vendored electron/rcedit; VersionInfo ctor throws std::system_error caught in OnEnumResourceLanguage. Self-contained throw/catch — already excluded from PCH",
   },
+  {
+    file: "src/jsc/bindings/highway_json.cpp",
+    extraFlags: ["-O2"],
+    when: c => c.debug,
+    desc:
+      "Always optimize the JSON structural-index kernel (debug builds use -O0 globally). At -O0 every " +
+      "highway op is an outlined call taking 64-byte vectors by value through the stack, and clang's " +
+      "unoptimized codegen for that pattern raises #GP on an aligned zmm stack store under the debug " +
+      "sanitizer set; it would also make every JSON parse in debug builds pathologically slow. " +
+      "Covered by test/js/bun/jsonc/jsonc.test.ts and `scripts/bench-json-rust.sh --test`.",
+  },
 ];
 
 // ═══════════════════════════════════════════════════════════════════════════