src: disable calls to mimalloc when asan is enabled [v2]

.buildkite/ci.mjs

@@ -186,11 +186,9 @@ function getImageKey(platform) {
   if (features?.length) {
     key += `-with-${features.join("-")}`;
   }
-
   if (abi) {
     key += `-${abi}`;
   }
-
   return key;
 }
 
@@ -581,7 +579,8 @@ function getTestBunStep(platform, options, testOptions = {}) {
 
   const depends = [];
   if (!buildId) {
-    depends.push(`${getTargetKey(platform)}-build-bun`);
+    const build_platform = profile === "lsan" ? { ...platform, profile: "asan" } : platform;
+    depends.push(`${getTargetKey(build_platform)}-build-bun`);
   }
 
   return {
@@ -592,7 +591,7 @@ function getTestBunStep(platform, options, testOptions = {}) {
     retry: getRetry(),
     cancel_on_build_failing: isMergeQueue(),
     parallelism: os === "darwin" ? 2 : 10,
-    timeout_in_minutes: profile === "asan" || os === "windows" ? 45 : 30,
+    timeout_in_minutes: profile === "asan" || profile === "lsan" || os === "windows" ? 60 : 30,
     env: {
       ASAN_OPTIONS: "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=0",
     },
@@ -1140,8 +1139,9 @@ async function getPipeline(options = {}) {
     );
   }
 
+  const { skipTests, forceTests, testFiles } = options;
+
   if (!isMainBranch()) {
-    const { skipTests, forceTests, testFiles } = options;
     if (!skipTests || forceTests) {
       steps.push(
         ...testPlatforms.map(target => ({
@@ -1152,6 +1152,17 @@ async function getPipeline(options = {}) {
       );
     }
   }
+  if (includeASAN && (!skipTests || forceTests)) {
+    const asan_targets = testPlatforms.filter(v => v.profile === "asan");
+    const lsan_targets = asan_targets.map(v => ({ ...v, profile: "lsan" }));
+    for (let i = 0; i < lsan_targets.length; i++) {
+      steps.push({
+        key: getTargetKey(lsan_targets[i]),
+        group: getTargetLabel(asan_targets[i]),
+        steps: [getTestBunStep(lsan_targets[i], options, { testFiles, buildId })],
+      });
+    }
+  }
 
   if (isMainBranch()) {
     steps.push(getReleaseStep(buildPlatforms, options));

cmake/Options.cmake

@@ -199,9 +199,11 @@ optionx(USE_WEBKIT_ICU BOOL "Use the ICU libraries from WebKit" DEFAULT ${DEFAUL
 
 optionx(ERROR_LIMIT STRING "Maximum number of errors to show when compiling C++ code" DEFAULT "100")
 
-# This is not an `option` because setting this variable to OFF is experimental
-# and unsupported. This replaces the `use_mimalloc` variable previously in
-# bun.zig, and enables C++ code to also be aware of the option.
-set(USE_MIMALLOC_AS_DEFAULT_ALLOCATOR ON)
+set(USE_MIMALLOC_AS_DEFAULT_ALLOCATOR_DEFAULT ON)
+if(ENABLE_ASAN)
+  set(USE_MIMALLOC_AS_DEFAULT_ALLOCATOR_DEFAULT OFF)
+endif()
+
+optionx(USE_MIMALLOC_AS_DEFAULT_ALLOCATOR BOOL "Use mimalloc as the default allocator" DEFAULT ${USE_MIMALLOC_AS_DEFAULT_ALLOCATOR_DEFAULT})
 
 list(APPEND CMAKE_ARGS -DCMAKE_EXPORT_COMPILE_COMMANDS=ON)

package.json

@@ -60,8 +60,8 @@
     "lint": "bunx oxlint --config=oxlint.json --format=github src/js",
     "lint:fix": "oxlint --config oxlint.json --fix",
     "test": "node scripts/runner.node.mjs --exec-path ./build/debug/bun-debug",
-    "testleak": "BUN_DESTRUCT_VM_ON_EXIT=1 ASAN_OPTIONS=detect_leaks=1 LSAN_OPTIONS=malloc_context_size=100:print_suppressions=1:suppressions=$npm_config_local_prefix/test/leaksan.supp ./build/debug/bun-debug",
     "test:release": "node scripts/runner.node.mjs --exec-path ./build/release/bun",
+    "testleak": "BUN_DESTRUCT_VM_ON_EXIT=1 ASAN_OPTIONS=allow_user_segv_handler=1:disable_coredump=0:detect_leaks=1 LSAN_OPTIONS=malloc_context_size=200:print_suppressions=0:suppressions=$npm_config_local_prefix/test/leaksan.supp ./build/debug/bun-debug",
     "banned": "bun test test/internal/ban-words.test.ts",
     "glob-sources": "bun scripts/glob-sources.mjs",
     "zig": "vendor/zig/zig.exe",

scripts/runner.node.mjs

@@ -1,4 +1,4 @@
-#! /usr/bin/env node
+#!/usr/bin/env node
 
 // This is a script that runs `bun test` to test Bun itself.
 // It is not intended to be used as a test runner for other projects.
@@ -81,7 +81,6 @@ function getNodeParallelTestTimeout(testPath) {
     return 90_000;
   }
   if (!isCI) return 60_000; // everything slower in debug mode
-  if (options["step"]?.includes("-asan-")) return 60_000;
   return 20_000;
 }
 
@@ -169,8 +168,13 @@ const { values: options, positionals: filters } = parseArgs({
     },
   },
 });
+startGroup("CLI Options", () => {
+  console.log(options);
+});
 
 const cliOptions = options;
+const isASAN = options["step"]?.includes("-asan-");
+const isLSAN = options["step"]?.includes("-lsan-");
 
 if (cliOptions.junit) {
   try {
@@ -329,7 +333,7 @@ const skipsForLeaksan = (() => {
  * @returns {boolean}
  */
 const shouldValidateExceptions = test => {
-  return !(skipsForExceptionValidation.includes(test) || skipsForExceptionValidation.includes("test/" + test));
+  return !skipsForExceptionValidation.includes(test);
 };
 
 /**
@@ -338,7 +342,7 @@ const shouldValidateExceptions = test => {
  * @returns {boolean}
  */
 const shouldValidateLeakSan = test => {
-  return !(skipsForLeaksan.includes(test) || skipsForLeaksan.includes("test/" + test));
+  return !skipsForLeaksan.includes(test);
 };
 
 /**
@@ -594,17 +598,16 @@ async function runTests() {
               NO_COLOR: "1",
               BUN_DEBUG_QUIET_LOGS: "1",
             };
-            if ((basename(execPath).includes("asan") || !isCI) && shouldValidateExceptions(testPath)) {
+            if ((isASAN || !isCI) && shouldValidateExceptions(title)) {
               env.BUN_JSC_validateExceptionChecks = "1";
               env.BUN_JSC_dumpSimulatedThrows = "1";
             }
-            if ((basename(execPath).includes("asan") || !isCI) && shouldValidateLeakSan(testPath)) {
+            if ((isLSAN || !isCI) && shouldValidateLeakSan(title)) {
               env.BUN_DESTRUCT_VM_ON_EXIT = "1";
               env.ASAN_OPTIONS = "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=1:abort_on_error=1";
-              // prettier-ignore
-              env.LSAN_OPTIONS = `malloc_context_size=100:print_suppressions=0:suppressions=${process.cwd()}/test/leaksan.supp`;
+              env.LSAN_OPTIONS = `malloc_context_size=200:print_suppressions=0:suppressions=${process.cwd()}/test/leaksan.supp`;
             }
-            return runTest(title, async () => {
+            return runTest(title, async index => {
               const { ok, error, stdout, crashes } = await spawnBun(execPath, {
                 cwd: cwd,
                 args: [
@@ -613,7 +616,7 @@ async function runTests() {
                   absoluteTestPath,
                 ],
                 timeout: getNodeParallelTestTimeout(title),
-                env,
+                env: { ...env, TEST_SERIAL_ID: index },
                 stdout: parallelism > 1 ? () => {} : chunk => pipeTestStdout(process.stdout, chunk),
                 stderr: parallelism > 1 ? () => {} : chunk => pipeTestStdout(process.stderr, chunk),
               });
@@ -1340,15 +1343,14 @@ async function spawnBunTest(execPath, testPath, opts = { cwd }) {
     GITHUB_ACTIONS: "true", // always true so annotations are parsed
     ...opts["env"],
   };
-  if ((basename(execPath).includes("asan") || !isCI) && shouldValidateExceptions(relative(cwd, absPath))) {
+  if ((isASAN || !isCI) && shouldValidateExceptions(relative(cwd, absPath))) {
     env.BUN_JSC_validateExceptionChecks = "1";
     env.BUN_JSC_dumpSimulatedThrows = "1";
   }
-  if ((basename(execPath).includes("asan") || !isCI) && shouldValidateLeakSan(relative(cwd, absPath))) {
+  if ((isLSAN || !isCI) && shouldValidateLeakSan(relative(cwd, absPath))) {
     env.BUN_DESTRUCT_VM_ON_EXIT = "1";
     env.ASAN_OPTIONS = "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=1:abort_on_error=1";
-    // prettier-ignore
-    env.LSAN_OPTIONS = `malloc_context_size=100:print_suppressions=0:suppressions=${process.cwd()}/test/leaksan.supp`;
+    env.LSAN_OPTIONS = `malloc_context_size=200:print_suppressions=0:suppressions=${process.cwd()}/test/leaksan.supp`;
   }
 
   const { ok, error, stdout, crashes } = await spawnBun(execPath, {
@@ -1963,8 +1965,9 @@ async function getExecPathFromBuildKite(target, buildId) {
   mkdirSync(releasePath, { recursive: true });
 
   let zipPath;
+  const build_target = target.includes("-lsan-") ? target.replace("-lsan-", "-asan-") : target;
   downloadLoop: for (let i = 0; i < 10; i++) {
-    const args = ["artifact", "download", "**", releasePath, "--step", target];
+    const args = ["artifact", "download", "**", releasePath, "--step", build_target];
     if (buildId) {
       args.push("--build", buildId);
     }
@@ -1985,12 +1988,12 @@ async function getExecPathFromBuildKite(target, buildId) {
       break downloadLoop;
     }
 
-    console.warn(`Waiting for ${target}.zip to be available...`);
-    await new Promise(resolve => setTimeout(resolve, i * 1000));
+    console.warn(`Waiting for ${build_target}.zip to be available...`);
+    await new Promise(resolve => setTimeout(resolve, (i + 1) * 1000));
   }
 
   if (!zipPath) {
-    throw new Error(`Could not find ${target}.zip from Buildkite: ${releasePath}`);
+    throw new Error(`Could not find ${build_target}.zip from Buildkite: ${releasePath}`);
   }
 
   await unzip(zipPath, releasePath);
@@ -2081,6 +2084,7 @@ function formatTestToMarkdown(result, concise, retries) {
 
     const testTitle = testPath.replace(/\\/g, "/");
     const testUrl = getFileUrl(testPath, errorLine);
+    const stripped = stripAnsi(stdout);
 
     if (concise) {
       markdown += "<li>";
@@ -2105,16 +2109,26 @@ function formatTestToMarkdown(result, concise, retries) {
     if (newFiles.includes(testTitle)) {
       markdown += ` (new)`;
     }
+    if (stripped.length > 1024 * 32) {
+      markdown += ` (truncated)`;
+    }
 
     if (concise) {
       markdown += "</li>\n";
     } else {
       markdown += "</summary>\n\n";
+      let inner = stripped;
+      // https://buildkite.com/docs/agent/v3/cli-annotate
+      // > The annotation body can be supplied as a command line argument, or by piping content into the command. The maximum size of each annotation body is 1MiB.
+      if (inner.length > 1024 * 32) {
+        inner = inner.slice(inner.length - 1024 * 32); // trim to the last 32kb of the message
+        inner = inner.slice(inner.indexOf("\n")); // don't cutoff in the middle of a line
+      }
       if (isBuildkite) {
-        const preview = escapeCodeBlock(stdout);
+        const preview = escapeCodeBlock(inner);
         markdown += `\`\`\`terminal\n${preview}\n\`\`\`\n`;
       } else {
-        const preview = escapeHtml(stripAnsi(stdout));
+        const preview = escapeHtml(stripAnsi(inner));
         markdown += `<pre><code>${preview}</code></pre>\n`;
       }
       markdown += "\n\n</details>\n\n";

scripts/utils.mjs

@@ -2695,7 +2695,7 @@ export function reportAnnotationToBuildKite({ context, label, content, style = "
     return;
   }
   if (attempt > 0) {
-    const cause = error ?? signal ?? `code ${status}`;
+    const cause = error ?? [`signal ${signal}`, `code ${status}`, stderr];
     throw new Error(`Failed to create annotation: ${label}`, { cause });
   }
   const errorContent = formatAnnotationToHtml({

src/allocators.zig

@@ -1,9 +1,14 @@
 pub const c_allocator = basic.c_allocator;
 pub const z_allocator = basic.z_allocator;
-pub const freeWithoutSize = basic.freeWithoutSize;
 pub const mimalloc = @import("./allocators/mimalloc.zig");
 pub const MimallocArena = @import("./allocators/MimallocArena.zig");
 
+pub const malloc = basic.malloc;
+pub const realloc = basic.realloc;
+pub const calloc = basic.calloc;
+pub const free = basic.free;
+pub const usable_size = basic.usable_size;
+
 pub const allocation_scope = @import("./allocators/allocation_scope.zig");
 pub const AllocationScope = allocation_scope.AllocationScope;
 pub const AllocationScopeIn = allocation_scope.AllocationScopeIn;

src/allocators/NullableAllocator.zig

@@ -7,22 +7,22 @@ ptr: *anyopaque = undefined,
 // the regular `ptr` because `ptr` may be undefined.
 vtable: ?*const std.mem.Allocator.VTable = null,
 
-pub inline fn init(allocator: ?std.mem.Allocator) NullableAllocator {
+pub fn init(allocator: ?std.mem.Allocator) NullableAllocator {
     return if (allocator) |a| .{
         .ptr = a.ptr,
         .vtable = a.vtable,
     } else .{};
 }
 
-pub inline fn isNull(this: NullableAllocator) bool {
+pub fn isNull(this: NullableAllocator) bool {
     return this.vtable == null;
 }
 
-pub inline fn isWTFAllocator(this: NullableAllocator) bool {
+pub fn isWTFAllocator(this: NullableAllocator) bool {
     return bun.String.isWTFAllocator(this.get() orelse return false);
 }
 
-pub inline fn get(this: NullableAllocator) ?std.mem.Allocator {
+pub fn get(this: NullableAllocator) ?std.mem.Allocator {
     return if (this.vtable) |vt| std.mem.Allocator{ .ptr = this.ptr, .vtable = vt } else null;
 }
 

src/allocators/basic.zig

@@ -139,10 +139,11 @@ const z_allocator_vtable = Allocator.VTable{
     .free = &ZAllocator.free_with_z_allocator,
 };
 
-/// mimalloc can free allocations without being given their size.
-pub fn freeWithoutSize(ptr: ?*anyopaque) void {
-    mimalloc.mi_free(ptr);
-}
+pub const malloc = mimalloc.mi_malloc;
+pub const free = mimalloc.mi_free;
+pub const realloc = mimalloc.mi_realloc;
+pub const calloc = mimalloc.mi_calloc;
+pub const usable_size = mimalloc.mi_usable_size;
 
 const Environment = @import("../env.zig");
 const std = @import("std");

src/allocators/fallback.zig

@@ -1,9 +1,16 @@
 pub const c_allocator = std.heap.c_allocator;
 pub const z_allocator = @import("./fallback/z.zig").allocator;
 
-/// libc can free allocations without being given their size.
-pub fn freeWithoutSize(ptr: ?*anyopaque) void {
-    std.c.free(ptr);
-}
+pub const malloc = std.c.malloc;
+pub const free = std.c.free;
+pub const realloc = std.c.realloc;
+pub const calloc = std.c.calloc;
+pub const usable_size = switch (Environment.os) {
+    .mac => std.c.malloc_size,
+    .linux => std.c.malloc_usable_size,
+    .windows => std.c._msize,
+    .wasm => @compileError("unreachable"),
+};
 
+const Environment = @import("../env.zig");
 const std = @import("std");

src/bake/production.zig

@@ -30,7 +30,7 @@ pub fn buildCommand(ctx: bun.cli.Command.Context) !void {
     defer arena.deinit();
 
     const vm = try VirtualMachine.initBake(.{
-        .allocator = arena.allocator(),
+        .allocator = bun.default_allocator,
         .log = ctx.log,
         .args = ctx.args,
         .smol = ctx.runtime_options.smol,

src/boringssl.zig

@@ -59,18 +59,18 @@ pub fn initClient() *boring.SSL {
 // may result in deadlocks, crashes, or memory corruption.
 
 export fn OPENSSL_memory_alloc(size: usize) ?*anyopaque {
-    return bun.mimalloc.mi_malloc(size);
+    return bun.allocators.malloc(size);
 }
 
 // BoringSSL always expects memory to be zero'd
 export fn OPENSSL_memory_free(ptr: *anyopaque) void {
-    const len = bun.mimalloc.mi_usable_size(ptr);
+    const len = bun.allocators.usable_size(ptr);
     @memset(@as([*]u8, @ptrCast(ptr))[0..len], 0);
-    bun.mimalloc.mi_free(ptr);
+    bun.allocators.free(ptr);
 }
 
 export fn OPENSSL_memory_get_size(ptr: ?*const anyopaque) usize {
-    return bun.mimalloc.mi_usable_size(ptr);
+    return bun.allocators.usable_size(ptr);
 }
 
 const INET6_ADDRSTRLEN = if (bun.Environment.isWindows) 65 else 46;

src/brotli.zig

@@ -9,7 +9,7 @@ pub const BrotliAllocator = struct {
             return zone.malloc_zone_malloc(len) orelse bun.outOfMemory();
         }
 
-        return mimalloc.mi_malloc(len) orelse bun.outOfMemory();
+        return bun.allocators.malloc(len) orelse bun.outOfMemory();
     }
 
     pub fn free(_: ?*anyopaque, data: ?*anyopaque) callconv(.c) void {
@@ -19,7 +19,7 @@ pub const BrotliAllocator = struct {
             return;
         }
 
-        mimalloc.mi_free(data);
+        bun.allocators.free(data);
     }
 };
 
@@ -281,7 +281,5 @@ pub const BrotliCompressionStream = struct {
     }
 };
 
-const std = @import("std");
-
 const bun = @import("bun");
-const mimalloc = bun.mimalloc;
+const std = @import("std");