bmalloc, libpas: stop fragmenting the heap into VMAs, and bound the EAGAIN retry loop

Source/bmalloc/bmalloc/BSyscall.h

@@ -25,8 +25,25 @@
 
 #pragma once
 
+#include "BPlatform.h"
 #include <errno.h>
 
+#if !BOS(WINDOWS)
+#include <time.h>
+#endif
+
+// Linux reports a failed VMA split from madvise() as EAGAIN (mm/madvise.c turns
+// -ENOMEM into -EAGAIN). Once a process is at vm.max_map_count that failure is
+// permanent, so an uncapped retry loop spins on the syscall until it is killed.
+#define BSYSCALL_MAX_EAGAIN_RETRIES 100u
+#define BSYSCALL_EAGAIN_BACKOFF_NSEC 1000000L
+
 #define SYSCALL(x) do { \
-    while ((x) == -1 && errno == EAGAIN) { } \
+    unsigned bmallocSyscallRetries = 0; \
+    while ((x) == -1 && errno == EAGAIN) { \
+        struct timespec bmallocSyscallBackoff = { 0, BSYSCALL_EAGAIN_BACKOFF_NSEC }; \
+        if (++bmallocSyscallRetries > BSYSCALL_MAX_EAGAIN_RETRIES) \
+            break; \
+        nanosleep(&bmallocSyscallBackoff, nullptr); \
+    } \
 } while (0);

Source/bmalloc/bmalloc/VMAllocate.h

@@ -324,10 +324,11 @@ inline void vmDeallocatePhysicalPages(void* p, size_t vmSize)
 #elif BOS(FREEBSD)
     SYSCALL(madvise(p, vmSize, MADV_FREE));
 #else
+    // MADV_DONTDUMP is deliberately not paired with MADV_DONTNEED here: toggling
+    // VM_DONTDUMP over sub-ranges splits the VMA, and the resulting fragmentation
+    // drives the process into vm.max_map_count. Pages dropped by MADV_DONTNEED are
+    // no longer resident, so the core dump skips them either way.
     SYSCALL(madvise(p, vmSize, MADV_DONTNEED));
-#if BOS(LINUX)
-    SYSCALL(madvise(p, vmSize, MADV_DONTDUMP));
-#endif
 #endif
 }
 
@@ -342,9 +343,6 @@ inline void vmAllocatePhysicalPages(void* p, size_t vmSize)
     // Instead the kernel will commit pages as they are touched.
 #else
     SYSCALL(madvise(p, vmSize, MADV_NORMAL));
-#if BOS(LINUX)
-    SYSCALL(madvise(p, vmSize, MADV_DODUMP));
-#endif
 #endif
 }
 #else

Source/bmalloc/libpas/src/libpas/pas_page_malloc.c

@@ -384,8 +384,8 @@ static void commit_impl(void* ptr, size_t size, bool do_mprotect, bool is_symmet
     }
 
 #if PAS_OS(LINUX)
+    /* No MADV_DODUMP: see the note in decommit_impl. */
     PAS_ASSERT(!is_symmetric);
-    PAS_SYSCALL(madvise(ptr, size, MADV_DODUMP));
 #elif PAS_OS(WINDOWS)
     if (is_symmetric) {
         /*
@@ -472,9 +472,14 @@ static void decommit_impl(void* ptr, size_t size,
     PAS_ASSERT(!is_symmetric);
     PAS_SYSCALL(madvise(ptr, size, MADV_FREE));
 #elif PAS_OS(LINUX)
+    /*
+     * MADV_DONTDUMP is deliberately not paired with MADV_DONTNEED here: toggling
+     * VM_DONTDUMP over sub-ranges splits the VMA, and the resulting fragmentation
+     * drives the process into vm.max_map_count. Pages dropped by MADV_DONTNEED are
+     * no longer resident, so the core dump skips them either way.
+     */
     PAS_ASSERT(!is_symmetric);
     PAS_SYSCALL(madvise(ptr, size, MADV_DONTNEED));
-    PAS_SYSCALL(madvise(ptr, size, MADV_DONTDUMP));
 #elif PAS_OS(WINDOWS)
     if (is_symmetric) {
         /*

Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c

@@ -558,7 +558,7 @@ void pas_thread_local_cache_ensure_committed(pas_thread_local_cache* thread_loca
         pas_lock_assert_held(&thread_local_cache->node->scavenger_lock);
 
         /* Asymmetric, paired with decommit_allocator_range. The page never lost its commit
-           charge; this is a no-op on Windows and a MADV_DODUMP on Linux. */
+           charge, so this is a no-op on Windows and on Linux. */
         pas_page_malloc_commit_without_mprotect(
             (char*)thread_local_cache + (page_index << pas_page_malloc_alignment_shift()),
             pas_page_malloc_alignment(),

Source/bmalloc/libpas/src/libpas/pas_utils.h

@@ -49,6 +49,8 @@
 
 #if PAS_OS(WINDOWS)
 #include <intrin.h>
+#else
+#include <time.h>
 #endif
 
 PAS_IGNORE_CLANG_WARNINGS_BEGIN("qualifier-requires-header")
@@ -1324,8 +1326,20 @@ static inline bool pas_is_divisible_by(unsigned value, uint64_t magic_constant)
 enum cpp_initialization_t { cpp_initialization };
 #endif
 
+/* Linux reports a failed VMA split from madvise() as EAGAIN (mm/madvise.c turns
+   -ENOMEM into -EAGAIN). Once a process is at vm.max_map_count that failure is
+   permanent, so an uncapped retry loop spins on the syscall until it is killed. */
+#define PAS_MAX_EAGAIN_SYSCALL_RETRIES 100u
+#define PAS_EAGAIN_SYSCALL_BACKOFF_NSEC 1000000L
+
 #define PAS_SYSCALL(x) do { \
-    while ((x) == -1 && errno == EAGAIN) { } \
+    unsigned pas_syscall_retries = 0; \
+    while ((x) == -1 && errno == EAGAIN) { \
+        struct timespec pas_syscall_backoff = { 0, PAS_EAGAIN_SYSCALL_BACKOFF_NSEC }; \
+        if (++pas_syscall_retries > PAS_MAX_EAGAIN_SYSCALL_RETRIES) \
+            break; \
+        nanosleep(&pas_syscall_backoff, NULL); \
+    } \
 } while (0)
 
 PAS_END_EXTERN_C;