CI: two Linux x64 lanes with the stack protector, _FORTIFY_SOURCE, stack clash protection and zeroed locals

.github/scripts/lanes.mjs

@@ -49,6 +49,11 @@ const LTO_WINDOWS = "/clang:-flto=thin /clang:-fno-split-lto-unit";
 
 const SANITIZERS = "address,undefined";
 
+// All of it is decided when a function is compiled to IR (a function attribute, a store, a call to a checked libc
+// function), so it means the same with and without LTO. _FORTIFY_SOURCE is 2 because glibc 2.31 has nothing above it.
+const HARDENING =
+  "-fstack-protector-strong -D_FORTIFY_SOURCE=2 -fstack-clash-protection -ftrivial-auto-var-init=zero -ftrivial-auto-var-init-max-size=4096";
+
 // Which ICU every platform but macOS (which uses the system's) builds and bundles: icu/source.json, and nowhere else.
 const icu = JSON.parse(readFileSync(join(root, "icu/source.json"), "utf8"));
 const ICU = { ICU_VERSION: icu.version, ICU_SHA256: icu.sha256 };
@@ -79,10 +84,14 @@ const variants = {
   "assertions": { buildType: "Release", mimalloc: true, assertions: true },
   "coverage": { buildType: "Release", mimalloc: true, coverage: true },
   "assertions-coverage": { buildType: "Release", mimalloc: true, assertions: true, coverage: true },
+  // "release" and "lto" with ICU and WebKit compiled with HARDENING, and nothing else changed. Same ABI as those two.
+  "hardened": { buildType: "Release", mimalloc: true, hardened: true },
+  "lto-hardened": { buildType: "Release", mimalloc: true, lto: true, hardened: true },
 };
 const NO_ASAN = ["release", "lto", "debug"];
 const ALL = [...NO_ASAN, "asan", "debug-asan"];
 const FUZZING = ["assertions", "coverage", "assertions-coverage"];
+const HARDENED = ["hardened", "lto-hardened"];
 
 // Each platform:
 //   label(arch)     the label of the release variant; the other variants append -<variant>
@@ -95,19 +104,19 @@ const FUZZING = ["assertions", "coverage", "assertions-coverage"];
 //   imageInputs     files and directories the `base` stage copies in, besides the Dockerfile
 //   tested          per architecture, the variants whose jsc shell the `test` job runs the JavaScriptCore tests with, and
 //                   the runner it does that on: a machine of the lane's own platform and architecture. "lto" ("release"
-//                   where there is no lto lane) is what bun ships. The asan lanes are not tested: the tests take 65 to
-//                   100 minutes there on Linux and macOS and do not finish in 275 on Windows. `quick` is --quick, for
-//                   a runner with few cores.
+//                   where there is no lto lane) is what bun ships, and nothing else runs "lto-hardened". The asan lanes
+//                   are not tested: the tests take 65 to 100 minutes there on Linux and macOS and do not finish in 275
+//                   on Windows. `quick` is --quick, for a runner with few cores.
 const platforms = [
   {
     label: arch => `bun-webkit-linux-${arch}`,
     // The container is ubuntu 20.04 x86_64, for its glibc (2.31); arm64 is cross-compiled against an ubuntu 20.04
     // arm64 sysroot in the same container.
     dockerfile: "Dockerfile",
     packageOS: "linux",
-    lanes: { amd64: [...ALL, ...FUZZING], arm64: ALL },
+    lanes: { amd64: [...ALL, ...FUZZING, ...HARDENED], arm64: ALL },
     tested: {
-      amd64: { on: "linux-x64-gh", variants: ["lto"] },
+      amd64: { on: "linux-x64-gh", variants: ["lto", "lto-hardened"] },
       arm64: { on: "linux-arm64-gh", variants: ["lto"] },
     },
     image: () => "linux-glibc",
@@ -118,6 +127,7 @@ const platforms = [
       ENABLE_SANITIZERS: v.sanitizers ?? "",
       ENABLE_ASSERTS: v.assertions ? "ON" : "AUTO",
       ENABLE_FUZZILLI: v.coverage ? "ON" : "OFF",
+      HARDENING_FLAGS: v.hardened ? HARDENING : "",
       // Explicit --target: Ubuntu's clang defaults to x86_64-pc-linux-gnu, while bun's own objects and its Rust code
       // are <arch>-unknown-linux-gnu; with LTO, lld warns about the vendor mismatch once per JavaScriptCore object.
       MARCH_FLAG:

CLAUDE.md

@@ -188,15 +188,15 @@ Custom event loop implementation for Bun's runtime requirements
 
 ## CI/CD
 
-`.github/workflows/ci.yml` is the one build workflow, and a push to `main` and a pull request from a branch of this repository run exactly the same jobs. (A pull request from a fork runs none of them: its token cannot publish, so `plan` is skipped and everything else with it. Push the branch to this repository to build it.) They differ only in what the result is called: `main` publishes the release `autobuild-{sha}`, a pull request publishes the prerelease `autobuild-preview-pr-{n}-{sha8}` (built from the pull request's head) and links it from a comment on the pull request. Running the workflow by hand on a branch builds that branch's head as a prerelease `autobuild-{sha}`. Either way it builds every lane and publishes them as one GitHub release. A lane is one `<label>.tar.gz`, e.g. `bun-webkit-linux-amd64-lto`: Linux glibc/musl, macOS, Windows, FreeBSD and Android, x64/arm64, release/LTO/debug/ASAN, and for Linux glibc x64 three more for fuzzing (`-assertions`, `-coverage`, `-assertions-coverage`: the release lane with `ENABLE_ASSERTS=ON`, `ENABLE_FUZZILLI=ON` or both, see `variants` in `lanes.mjs`). Every lane builds on a Linux x64 runner, in a linux/amd64 Docker container, and everything that is not Linux x64 is cross-compiled there with `clang --target` and a sysroot: macOS, Windows, FreeBSD, Android, and Linux arm64 (glibc against an ubuntu 20.04 arm64 sysroot, so glibc 2.31 like the x64 container itself; musl against an alpine aarch64 sysroot). The glibc and musl versions a lane is built against are checked when the toolchain image is built, and every glibc `jsc` that is linked is checked not to need symbols newer than `GLIBC_2.31`.
+`.github/workflows/ci.yml` is the one build workflow, and a push to `main` and a pull request from a branch of this repository run exactly the same jobs. (A pull request from a fork runs none of them: its token cannot publish, so `plan` is skipped and everything else with it. Push the branch to this repository to build it.) They differ only in what the result is called: `main` publishes the release `autobuild-{sha}`, a pull request publishes the prerelease `autobuild-preview-pr-{n}-{sha8}` (built from the pull request's head) and links it from a comment on the pull request. Running the workflow by hand on a branch builds that branch's head as a prerelease `autobuild-{sha}`. Either way it builds every lane and publishes them as one GitHub release. A lane is one `<label>.tar.gz`, e.g. `bun-webkit-linux-amd64-lto`: Linux glibc/musl, macOS, Windows, FreeBSD and Android, x64/arm64, release/LTO/debug/ASAN, and for Linux glibc x64 three more for fuzzing (`-assertions`, `-coverage`, `-assertions-coverage`: the release lane with `ENABLE_ASSERTS=ON`, `ENABLE_FUZZILLI=ON` or both, see `variants` in `lanes.mjs`) and `-hardened` and `-lto-hardened` (the release and LTO lanes with ICU and WebKit compiled with `HARDENING` in `lanes.mjs`: stack protector, `_FORTIFY_SOURCE`, stack clash protection, zeroed locals). Every lane builds on a Linux x64 runner, in a linux/amd64 Docker container, and everything that is not Linux x64 is cross-compiled there with `clang --target` and a sysroot: macOS, Windows, FreeBSD, Android, and Linux arm64 (glibc against an ubuntu 20.04 arm64 sysroot, so glibc 2.31 like the x64 container itself; musl against an alpine aarch64 sysroot). The glibc and musl versions a lane is built against are checked when the toolchain image is built, and every glibc `jsc` that is linked is checked not to need symbols newer than `GLIBC_2.31`.
 
 The lanes are defined in `.github/scripts/lanes.mjs` and nowhere else: label, runner, Dockerfile, every build argument passed to it, and whether the lane is tested. `node .github/scripts/lanes.mjs` lists them (`--json` for the full settings), and `lanes.mjs build <label> --output <dir>` builds one; that command is also what CI runs, so a lane can be reproduced exactly. To add, remove or change a lane, edit `platforms` there. The workflow's `plan` job names the release and turns that table into the matrices of `image`, `build` (lanes that are only built), `build-tested` (lanes that are also tested; same steps) and `test`, and into the list of assets the `release` job requires.
 
 Each lane is built by a Dockerfile (`Dockerfile`, `Dockerfile.musl`, `Dockerfile.macos`, `Dockerfile.windows`, `Dockerfile.freebsd`, `Dockerfile.android`). In every one of them the `base` stage is the toolchain and nothing else (compilers, SDKs, sysroots): it must not take any lane setting, so that it is identical for every lane of a platform and architecture; lane settings go in the stages on top (`lane`, `build_icu`, `build_webkit`). Everything a lane needs from the network is in `base` too (packages, SDKs, sysroots, ICU's sources and host tools, `mig`, zstd, node): the stages on top must not download or install anything, so a lane's only network use is pulling its image. CI keeps those `base` stages as images in `ghcr.io/oven-sh/bun-webkit-build-env`, tagged `<toolchain>-<hash>`, the hash being of the Dockerfile up to the end of `base` (less comments and lane-only ARGs), the files `base` copies in and the build arguments it takes. The `image` job builds and pushes the ones whose tag is missing (so only after a change to a `base` stage), failing if it cannot, and each lane starts from its image (`lanes.mjs build --base-image`, which is `--build-context base=docker-image://...`). Without `--base-image` the toolchain is built as part of the build. Old images are not kept around: a build of `main` runs `prune`, which deletes every image that `main`'s lanes do not use, that is more than 14 days old and that is not one of the two newest of its toolchain: images a pull request pushed recently survive while a toolchain change is being worked on, and so does the image `main` used until the last change.
 
 The release starts as a draft, each lane uploads its tarball onto it, and the `release` job publishes it once every lane succeeded (or deletes the draft when one failed).
 
-The `test` job runs `Tools/Scripts/run-javascriptcore-tests` (JSTests, LayoutTests/js, the PerformanceTests collections) and `testFFI` against the `bin/jsc` built by the lanes marked `tested` in `lanes.mjs`, as soon as those lanes are built, each on a runner of its own platform and architecture: the Linux x64 and arm64 lto lanes, the builds Bun ships there (about 11 minutes each). Nothing else is tested. The asan lanes are built and published but not tested: the tests take 65 to 100 minutes against them on Linux and macOS and do not finish in 275 on Windows; no tested lane has assertions on. The macOS and Windows lanes are not tested either. The job can run them, on GitHub's standard runners (3 or 4 cores, so with `--quick`, which also skips the `//@ slow!` tests; on Windows with Windows' own perl and `--test-writer=ruby --ruby-runner`, as upstream's Windows bots did): the comment at the top of `lanes.mjs` has the `tested` entries. macOS took 16 minutes; Windows x64 took 145, because `--ruby-runner` runs the tests one at a time. It runs every test before failing (`--no-fail-fast`) and turns the run red when any fail, but does not gate the release, which is still published; the step's own log shows each `FAIL:` as it happens, a progress line and the runner's results (the full output is over 100,000 lines, more than the log viewer shows); the failing tests are listed in the job summary (whether a failing test is one to change is what `JSTests/BUN-TEST-DIFFERENCES.md` is for), and the workflow artifact has `jsc-failures.log` (what each failing run printed, with its repro command, grouped under its name), the full `jsc-tests.log` and the results JSON. To run them against an asan build locally (`--no-slow`: the tests marked `//@ slow!` pass the 300 second hard timeout under ASan; `--asan` sets `$asan` in `run-jsc-stress-tests`, for `//@ skip if $asan`): extract `bun-webkit/bin` from the asan tarball and run `ASAN_OPTIONS=detect_leaks=0:allocator_may_return_null=1:allow_user_segv_handler=1 JSCTEST_memoryLimit=4294967296 Tools/Scripts/run-javascriptcore-tests --no-build --root=<path>/bun-webkit --release --jsc-only --no-testmasm --no-testair --no-testb3 --no-testdfg --no-testapi --no-testwasmdebugger --no-testlibjsctools --no-fail-fast --memory-limited --no-slow --asan` (`--quick` instead of `--no-slow` for what the macOS and Windows legs run; add `--filter <regex>` for a subset).
+The `test` job runs `Tools/Scripts/run-javascriptcore-tests` (JSTests, LayoutTests/js, the PerformanceTests collections) and `testFFI` against the `bin/jsc` built by the lanes marked `tested` in `lanes.mjs`, as soon as those lanes are built, each on a runner of its own platform and architecture: the Linux x64 and arm64 lto lanes, the builds Bun ships there (about 11 minutes each), and the x64 `-lto-hardened` lane. Nothing else is tested. The asan lanes are built and published but not tested: the tests take 65 to 100 minutes against them on Linux and macOS and do not finish in 275 on Windows; no tested lane has assertions on. The macOS and Windows lanes are not tested either. The job can run them, on GitHub's standard runners (3 or 4 cores, so with `--quick`, which also skips the `//@ slow!` tests; on Windows with Windows' own perl and `--test-writer=ruby --ruby-runner`, as upstream's Windows bots did): the comment at the top of `lanes.mjs` has the `tested` entries. macOS took 16 minutes; Windows x64 took 145, because `--ruby-runner` runs the tests one at a time. It runs every test before failing (`--no-fail-fast`) and turns the run red when any fail, but does not gate the release, which is still published; the step's own log shows each `FAIL:` as it happens, a progress line and the runner's results (the full output is over 100,000 lines, more than the log viewer shows); the failing tests are listed in the job summary (whether a failing test is one to change is what `JSTests/BUN-TEST-DIFFERENCES.md` is for), and the workflow artifact has `jsc-failures.log` (what each failing run printed, with its repro command, grouped under its name), the full `jsc-tests.log` and the results JSON. To run them against an asan build locally (`--no-slow`: the tests marked `//@ slow!` pass the 300 second hard timeout under ASan; `--asan` sets `$asan` in `run-jsc-stress-tests`, for `//@ skip if $asan`): extract `bun-webkit/bin` from the asan tarball and run `ASAN_OPTIONS=detect_leaks=0:allocator_may_return_null=1:allow_user_segv_handler=1 JSCTEST_memoryLimit=4294967296 Tools/Scripts/run-javascriptcore-tests --no-build --root=<path>/bun-webkit --release --jsc-only --no-testmasm --no-testair --no-testb3 --no-testdfg --no-testapi --no-testwasmdebugger --no-testlibjsctools --no-fail-fast --memory-limited --no-slow --asan` (`--quick` instead of `--no-slow` for what the macOS and Windows legs run; add `--filter <regex>` for a subset).
 
 ## Architecture Notes
 

Dockerfile

@@ -11,6 +11,8 @@ ARG ENABLE_ASSERTS="AUTO"
 # JavaScriptCore compiled with -fsanitize-coverage=trace-pc-guard, and a jsc that Fuzzilli can drive. A CMake option and
 # not a flag in CFLAGS: those also build ICU, whose tools run during the build and have no coverage callbacks.
 ARG ENABLE_FUZZILLI="OFF"
+# Added to the flags ICU and WebKit are compiled with.
+ARG HARDENING_FLAGS=""
 ARG USE_MIMALLOC="OFF"
 ARG USE_EXTERNAL_MIMALLOC="OFF"
 # What the lane is built for: x86_64, which is what this container is, or aarch64, cross-compiled against the sysroot
@@ -296,12 +298,15 @@ ARG DEFAULT_CFLAGS
 ARG ENABLE_SANITIZERS
 ARG ENABLE_ASSERTS
 ARG ENABLE_FUZZILLI
+ARG HARDENING_FLAGS
 ARG USE_MIMALLOC
 ARG USE_EXTERNAL_MIMALLOC
 ARG LINUX_ARCH
 ARG ICU_VERSION
 
 ENV LTO_FLAG="${LTO_FLAG}"
+ENV CFLAGS="$CFLAGS ${HARDENING_FLAGS}"
+ENV CXXFLAGS="$CXXFLAGS ${HARDENING_FLAGS}"
 
 # Download and build ICU.
 #
@@ -382,6 +387,9 @@ ENV RELEASE_FLAGS=${RELEASE_FLAGS}
 #
 # With ENABLE_FUZZILLI the libraries must call the coverage callbacks and leave them undefined: Bun defines its own.
 #
+# With HARDENING_FLAGS jsc must import what the stack protector and _FORTIFY_SOURCE call. It is read off jsc because the
+# archives of an LTO lane are bitcode, which does not name __stack_chk_fail yet.
+#
 # clang searches C_INCLUDE_PATH (gcc-13's builtin-header dir) before its own
 # resource dir, so C TUs including <immintrin.h> (mimalloc static.c, -march=nehalem)
 # pick up gcc's incompatible copy. clang ships its own; drop it for this step.
@@ -444,6 +452,11 @@ RUN --mount=type=tmpfs,target=/webkitbuild \
             ! llvm-nm --defined-only $WEBKIT_OUT_DIR/lib/*.a | grep -w $callback || { echo "error: a library defines $callback, which is for whoever links it to do" >&2; exit 1; }; \
         done; \
     fi && \
+    if [ -n "$HARDENING_FLAGS" ]; then \
+        llvm-readelf --dyn-syms $WEBKIT_OUT_DIR/bin/jsc | grep -o '__[a-z_]*_chk[a-z_]*' | sort -u > /tmp/chk && cat /tmp/chk && \
+        grep -qx __stack_chk_fail /tmp/chk || { echo "error: jsc has no stack protector" >&2; exit 1; }; \
+        grep -qv __stack_chk /tmp/chk || { echo "error: jsc calls no checked libc function, _FORTIFY_SOURCE did nothing" >&2; exit 1; }; \
+    fi && \
     cp -r $WEBKIT_OUT_DIR/lib/*.a /output/lib && \
     cp $WEBKIT_OUT_DIR/*.h /output/include && \
     cp -r $WEBKIT_OUT_DIR/bin /output/bin && \