Honor virtual memory limit

eng/native/tryrun.cmake

@@ -72,6 +72,7 @@ if(DARWIN)
     set_cache_value(HAVE_MMAP_DEV_ZERO_EXITCODE 1)
     set_cache_value(HAVE_PROCFS_CTL_EXITCODE 1)
     set_cache_value(HAVE_PROCFS_STAT_EXITCODE 1)
+    set_cache_value(HAVE_PROCFS_STATM_EXITCODE 1)
     set_cache_value(HAVE_SCHED_GETCPU_EXITCODE 1)
     set_cache_value(HAVE_SCHED_GET_PRIORITY_EXITCODE 0)
     set_cache_value(HAVE_VALID_NEGATIVE_INF_POW_EXITCODE 0)
@@ -108,6 +109,7 @@ elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s39
   set_cache_value(HAVE_MMAP_DEV_ZERO_EXITCODE 0)
   set_cache_value(HAVE_PROCFS_CTL_EXITCODE 1)
   set_cache_value(HAVE_PROCFS_STAT_EXITCODE 0)
+  set_cache_value(HAVE_PROCFS_STATM_EXITCODE 0)
   set_cache_value(HAVE_SCHED_GETCPU_EXITCODE 0)
   set_cache_value(HAVE_SCHED_GET_PRIORITY_EXITCODE 0)
   set_cache_value(HAVE_VALID_NEGATIVE_INF_POW_EXITCODE 0)
@@ -137,6 +139,7 @@ elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s39
     set_cache_value(HAVE_CLOCK_REALTIME 1)
     set_cache_value(HAVE_BROKEN_FIFO_KEVENT_EXITCODE 1)
     set_cache_value(HAVE_PROCFS_STAT 0)
+    set_cache_value(HAVE_PROCFS_STATM 0)
     set_cache_value(UNGETC_NOT_RETURN_EOF 0)
     set_cache_value(HAVE_COMPATIBLE_ILOGBNAN 1)
     set_cache_value(HAVE_FUNCTIONAL_PTHREAD_ROBUST_MUTEXES_EXITCODE 0)
@@ -165,6 +168,7 @@ elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s39
     set_cache_value(HAVE_COMPATIBLE_EXP_EXITCODE 0)
     set_cache_value(HAVE_COMPATIBLE_ILOGBNAN_EXITCODE 0)
     set_cache_value(HAVE_PROCFS_STAT_EXITCODE 1)
+    set_cache_value(HAVE_PROCFS_STATM_EXITCODE 1)
   endif()
 else()
   message(FATAL_ERROR "Unsupported platform. OS: ${CMAKE_SYSTEM_NAME}, arch: ${TARGET_ARCH_NAME}")

src/coreclr/gc/env/gcenv.os.h

@@ -410,6 +410,11 @@ class GCToOSInterface
     //  non zero if it has succeeded, 0 if it has failed
     static size_t GetVirtualMemoryLimit();
 
+    // Return the maximum address of the of the virtual address space of this process.
+    // Return:
+    //  non zero if it has succeeded, 0 if it has failed
+    static size_t GetVirtualMemoryMaxAddress();
+
     // Get the physical memory that this process can use.
     // Return:
     //  non zero if it has succeeded, 0 if it has failed

src/coreclr/gc/gc.cpp

@@ -9556,7 +9556,7 @@ int gc_heap::grow_brick_card_tables (uint8_t* start,
         {
             //modify the highest address so the span covered
             //is twice the previous one.
-            uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
+            uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryMaxAddress());
             // On non-Windows systems, we get only an approximate value that can possibly be
             // slightly lower than the saved_g_highest_address.
             // In such case, we set the top to the saved_g_highest_address so that the
@@ -48232,8 +48232,9 @@ HRESULT GCHeap::Initialize()
         else
         {
             // If no hard_limit is configured the reservation size is min of 1/2 GetVirtualMemoryLimit() or max of 256Gb or 2x physical limit.
-            gc_heap::regions_range = min(GCToOSInterface::GetVirtualMemoryLimit()/2, max((size_t)256 * 1024 * 1024 * 1024, (size_t)(2 * gc_heap::total_physical_mem)));
+            gc_heap::regions_range = max((size_t)256 * 1024 * 1024 * 1024, (size_t)(2 * gc_heap::total_physical_mem));
         }
+        gc_heap::regions_range = min(gc_heap::regions_range, GCToOSInterface::GetVirtualMemoryLimit()/2);
         gc_heap::regions_range = align_on_page(gc_heap::regions_range);
     }
     GCConfig::SetGCRegionRange(gc_heap::regions_range);

src/coreclr/gc/unix/config.gc.h.in

@@ -32,5 +32,6 @@
 #cmakedefine01 HAVE_XSW_USAGE
 #cmakedefine01 HAVE_XSWDEV
 #cmakedefine01 HAVE_NON_LEGACY_STATFS
+#cmakedefine01 HAVE_PROCFS_STATM
 
 #endif // __CONFIG_H__

src/coreclr/gc/unix/configure.cmake

@@ -184,4 +184,21 @@ check_prototype_definition(
     ${STATFS_INCLUDES}
     HAVE_NON_LEGACY_STATFS)
 
+set(CMAKE_REQUIRED_LIBRARIES)
+check_cxx_source_runs("
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+int main(void) {
+  int fd;
+
+  fd = open(\"/proc/self/statm\", O_RDONLY);
+  if (fd == -1) {
+    exit(1);
+  }
+  exit(0);
+}" HAVE_PROCFS_STATM)
+
 configure_file(${CMAKE_CURRENT_LIST_DIR}/config.gc.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.gc.h)

src/coreclr/gc/unix/gcenv.unix.cpp

@@ -26,6 +26,8 @@
 #include <sys/swap.h>
 #endif
 
+#include <sys/resource.h>
+
 #undef min
 #undef max
 
@@ -1085,10 +1087,66 @@ const AffinitySet* GCToOSInterface::SetGCThreadsAffinitySet(uintptr_t configAffi
     return &g_processAffinitySet;
 }
 
+#if HAVE_PROCFS_STATM
 // Return the size of the user-mode portion of the virtual address space of this process.
+size_t GetCurrentVirtualMemorySize()
+{
+    size_t result = (size_t)-1;
+    size_t linelen;
+    char* line = nullptr;
+
+    // process virtual memory size is reported in the first column of the /proc/self/statm
+    FILE* file = fopen("/proc/self/statm", "r");
+    if (file != nullptr && getline(&line, &linelen, file) != -1)
+    {
+        // The first column of the /proc/self/statm contains the virtual memory size
+        char* context = nullptr;
+        char* strTok = strtok_r(line, " ", &context);
+
+        errno = 0;
+        result = strtoull(strTok, nullptr, 0);
+        if (errno == 0)
+        {
+            long pageSize = sysconf(_SC_PAGE_SIZE);
+            if (pageSize != -1)
+            {
+                result = result * pageSize;
+            }
+        }
+        else
+        {
+            assert(!"Failed to parse statm file contents.");
+            result = (size_t)-1;
+        }
+    }
+
+    if (file)
+        fclose(file);
+    free(line);
+
+    return result;
+}
+#endif // HAVE_PROCFS_STATM
+
+// Return the size of the available user-mode portion of the virtual address space of this process.
 // Return:
 //  non zero if it has succeeded, (size_t)-1 if not available
 size_t GCToOSInterface::GetVirtualMemoryLimit()
+{
+    rlimit addressSpaceLimit;
+    if ((getrlimit(RLIMIT_AS, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+    {
+        return addressSpaceLimit.rlim_cur;
+    }
+
+    // No virtual memory limit
+    return (size_t)-1;
+}
+
+// Return the maximum address of the of the virtual address space of this process.
+// Return:
+//  non zero if it has succeeded, 0 if it has failed
+size_t GCToOSInterface::GetVirtualMemoryMaxAddress()
 {
 #ifdef HOST_64BIT
 #ifndef TARGET_RISCV64
@@ -1302,34 +1360,49 @@ void GCToOSInterface::GetMemoryStatus(uint64_t restricted_limit, uint32_t* memor
     uint64_t available = 0;
     uint32_t load = 0;
 
-    if (memory_load != nullptr || available_physical != nullptr)
+    size_t used;
+    if (restricted_limit != 0)
     {
-        size_t used;
-        if (restricted_limit != 0)
+        // Get the physical memory in use - from it, we can get the physical memory available.
+        // We do this only when we have the total physical memory available.
+        if (GetPhysicalMemoryUsed(&used))
         {
-            // Get the physical memory in use - from it, we can get the physical memory available.
-            // We do this only when we have the total physical memory available.
-            if (GetPhysicalMemoryUsed(&used))
-            {
-                available = restricted_limit > used ? restricted_limit - used : 0;
-                load = (uint32_t)(((float)used * 100) / (float)restricted_limit);
-            }
+            available = restricted_limit > used ? restricted_limit - used : 0;
+            load = (uint32_t)(((float)used * 100) / (float)restricted_limit);
         }
-        else
+    }
+    else
+    {
+        available = GetAvailablePhysicalMemory();
+
+        if (memory_load != NULL)
         {
-            available = GetAvailablePhysicalMemory();
+            bool isRestricted;
+            uint64_t total = GetPhysicalMemoryLimit(&isRestricted);
 
-            if (memory_load != NULL)
+            if (total > available)
             {
-                bool isRestricted;
-                uint64_t total = GetPhysicalMemoryLimit(&isRestricted);
+                used = total - available;
+                load = (uint32_t)(((float)used * 100) / (float)total);
+            }
 
-                if (total > available)
+#if HAVE_PROCFS_STATM
+            rlimit addressSpaceLimit;
+            if ((getrlimit(RLIMIT_AS, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+            {
+                // If there is virtual address space limit set, compute virtual memory load and change
+                // the load to this one in case it is higher than the physical memory load
+                size_t used_virtual = GetCurrentVirtualMemorySize();
+                if (used_virtual != (size_t)-1)
                 {
-                    used = total - available;
-                    load = (uint32_t)(((float)used * 100) / (float)total);
+                    uint32_t load_virtual = (uint32_t)(((float)used_virtual * 100) / (float)addressSpaceLimit.rlim_cur);
+                    if (load_virtual > load)
+                    {
+                        load = load_virtual;
+                    }
                 }
             }
+#endif // HAVE_PROCFS_STATM
         }
     }
 

src/coreclr/gc/windows/gcenv.windows.cpp

@@ -943,7 +943,16 @@ const AffinitySet* GCToOSInterface::SetGCThreadsAffinitySet(uintptr_t configAffi
     return &g_processAffinitySet;
 }
 
-// Return the size of the user-mode portion of the virtual address space of this process.
+// Return the maximum address of the of the virtual address space of this process.
+// Return:
+//  non zero if it has succeeded, 0 if it has failed
+size_t GCToOSInterface::GetVirtualMemoryMaxAddress()
+{
+    // On Windows, the maximum address is the same as the virtual memory limit, unlike Unix
+    return GCToOSInterface::GetVirtualMemoryLimit();
+}
+
+// Return the size of the available user-mode portion of the virtual address space of this process.
 // Return:
 //  non zero if it has succeeded, (size_t)-1 if not available
 size_t GCToOSInterface::GetVirtualMemoryLimit()

src/coreclr/pal/src/map/virtual.cpp

@@ -31,7 +31,9 @@ SET_DEFAULT_DEBUG_CHANNEL(VIRTUAL); // some headers have code with asserts, so d
 #include "pal/init.h"
 #include "pal/utils.h"
 #include "common.h"
+#include <clrconfignocache.h>
 
+#include <sys/resource.h>
 #include <sys/types.h>
 #include <sys/mman.h>
 #include <errno.h>
@@ -1617,11 +1619,33 @@ void ExecutableMemoryAllocator::Initialize()
 void ExecutableMemoryAllocator::TryReserveInitialMemory()
 {
     CPalThread* pthrCurrent = InternalGetCurrentThread();
-    int32_t sizeOfAllocation = MaxExecutableMemorySizeNearCoreClr;
     int32_t preferredStartAddressIncrement;
     UINT_PTR preferredStartAddress;
     UINT_PTR coreclrLoadAddress;
 
+    int32_t sizeOfAllocation = MaxExecutableMemorySizeNearCoreClr;
+    int32_t initialReserveLimit = -1;
+    rlimit addressSpaceLimit;
+    if ((getrlimit(RLIMIT_AS, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+    {
+        // By default reserve max 20% of the available virtual address space
+        rlim_t initialExecMemoryPerc = 20;
+        CLRConfigNoCache defInitialExecMemoryPerc = CLRConfigNoCache::Get("InitialExecMemoryPercent", /*noprefix*/ false, &getenv);
+        if (defInitialExecMemoryPerc.IsSet())
+        {
+            DWORD perc;
+            if (defInitialExecMemoryPerc.TryAsInteger(16, perc))
+            {
+                initialExecMemoryPerc = perc;
+            }
+        }
+
+        initialReserveLimit = addressSpaceLimit.rlim_cur * initialExecMemoryPerc / 100;
+        if (initialReserveLimit < sizeOfAllocation)
+        {
+            sizeOfAllocation = initialReserveLimit;
+        }
+    }
 #if defined(TARGET_ARM) || defined(TARGET_ARM64)
     // Smaller steps on ARM because we try hard finding a spare memory in a 128Mb
     // distance from coreclr so e.g. all calls from corelib to coreclr could use relocs
@@ -1697,7 +1721,7 @@ void ExecutableMemoryAllocator::TryReserveInitialMemory()
         //     does not exceed approximately 2 GB.
         //   - The code heap allocator for the JIT can allocate from this address space. Beyond this reservation, one can use
         //     the DOTNET_CodeHeapReserveForJumpStubs environment variable to reserve space for jump stubs.
-        sizeOfAllocation = MaxExecutableMemorySize;
+        sizeOfAllocation = (initialReserveLimit != -1) ? initialReserveLimit : MaxExecutableMemorySize;
         m_startAddress = ReserveVirtualMemory(pthrCurrent, nullptr, sizeOfAllocation, MEM_RESERVE_EXECUTABLE);
         if (m_startAddress == nullptr)
         {