Fix std::nothrow aborting when exceptions are disabled

ChangeLog.md

@@ -24,6 +24,9 @@ See docs/process.md for more on how version tagging works.
   developers and helps take a care of post-checkout tasks such as `npm install`.
   If this script needs to be run (e.g. becuase package.json was changed, emcc
   will exit with an error. (#19736)
+- If exceptions are disabled, using `new` together with `std::nothrow` no
+  longer aborts if the allocation fails. Instead `nullptr` is returned now.
+  This does not change the behavior of regular usage of `new`.
 
 3.1.47 - 10/09/23
 -----------------

src/settings.js

@@ -140,6 +140,10 @@ var MALLOC = "dlmalloc";
 //      which it did not previously. If you don't want that, just stop passing
 //      it in at link time.
 //
+// Note that this setting does not affect the behavior of operator new in C++.
+// This function will always abort on allocation failure if exceptions are disabled.
+// If you want new to return 0 on failure, use it with std::nothrow.
+//
 // [link]
 var ABORTING_MALLOC = true;
 

system/lib/libcxx/src/new.cpp

@@ -90,10 +90,34 @@ operator new(std::size_t size) _THROW_BAD_ALLOC
     return p;
 }
 
+#if defined(__EMSCRIPTEN__) && defined(_LIBCPP_NO_EXCEPTIONS)
+void* _new_nothrow(size_t size) noexcept
+{
+    /// We cannot call ::operator new(size) here because it would abort
+    /// when malloc returns 0 and exceptions are disabled.
+    /// Expected behaviour of std::nothrow is not return 0 in that case.
+    void* p = nullptr;
+    if (size == 0)
+        size = 1;
+    while ((p = ::malloc(size)) == nullptr)
+    {
+        std::new_handler nh = std::get_new_handler();
+        if (nh)
+            nh();
+        else
+            break;
+    }
+    return p;
+}
+#endif
+
 _LIBCPP_WEAK
 void*
 operator new(size_t size, const std::nothrow_t&) noexcept
 {
+#if defined(__EMSCRIPTEN__) && defined(_LIBCPP_NO_EXCEPTIONS)
+    return _new_nothrow(size);
+#else
     void* p = nullptr;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     try
@@ -107,6 +131,7 @@ operator new(size_t size, const std::nothrow_t&) noexcept
     }
 #endif // _LIBCPP_NO_EXCEPTIONS
     return p;
+#endif // __EMSCRIPTEN__ && _LIBCPP_NO_EXCEPTIONS
 }
 
 _LIBCPP_WEAK
@@ -120,6 +145,9 @@ _LIBCPP_WEAK
 void*
 operator new[](size_t size, const std::nothrow_t&) noexcept
 {
+#if defined(__EMSCRIPTEN__) && defined(_LIBCPP_NO_EXCEPTIONS)
+    return _new_nothrow(size);
+#else
     void* p = nullptr;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     try
@@ -133,6 +161,7 @@ operator new[](size_t size, const std::nothrow_t&) noexcept
     }
 #endif // _LIBCPP_NO_EXCEPTIONS
     return p;
+#endif // __EMSCRIPTEN__ && _LIBCPP_NO_EXCEPTIONS
 }
 
 _LIBCPP_WEAK

test/core/test_nothrow_new.cpp

@@ -0,0 +1,20 @@
+// Copyright 2023 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+#include <iostream>
+#include <new>
+
+char* data;
+
+int main() {
+  data = new (std::nothrow) char[20 * 1024 * 1024];
+  if (data == nullptr) {
+    std::cout << "success" << std::endl;
+    return 0;
+  } else {
+    std::cout << "failure" << std::endl;
+    return 1;
+  }
+}

test/core/test_nothrow_new.out

@@ -0,0 +1 @@
+success

test/test_core.py

@@ -2440,6 +2440,18 @@ def test_aborting_new(self, args):
     self.emcc_args += args
     self.do_core_test('test_aborting_new.cpp')
 
+  @parameterized({
+    'nogrow': (['-sABORTING_MALLOC=0'],),
+    'grow': (['-sABORTING_MALLOC=0', '-sALLOW_MEMORY_GROWTH', '-sMAXIMUM_MEMORY=18MB'],)
+  })
+  @no_asan('requires more memory when growing')
+  @no_lsan('requires more memory when growing')
+  @no_4gb('depends on MAXIMUM_MEMORY')
+  @no_2gb('depends on MAXIMUM_MEMORY')
+  def test_nothrow_new(self, args):
+    self.emcc_args += args
+    self.do_core_test('test_nothrow_new.cpp')
+
   @no_wasm2js('no WebAssembly.Memory()')
   @no_asan('ASan alters the memory size')
   @no_lsan('LSan alters the memory size')