Support hardware with more than 1024 CPUs

src/coreclr/gc/allocation.cpp

@@ -4118,8 +4118,8 @@ void gc_heap::balance_heaps (alloc_context* acontext)
         if (set_home_heap)
         {
             /*
-                        // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
-                        if (n_heaps > MAX_SUPPORTED_CPUS)
+                        // Since we are balancing up to MAX_SUPPORTED_HEAPS, no need for this.
+                        if (n_heaps > MAX_SUPPORTED_HEAPS)
                         {
                             // on machines with many processors cache affinity is really king, so don't even try
                             // to balance on these.

src/coreclr/gc/env/gcenv.os.h

@@ -6,6 +6,9 @@
 #ifndef __GCENV_OS_H__
 #define __GCENV_OS_H__
 
+#include <new>
+using std::nothrow;
+
 #include <minipal/mutex.h>
 
 #define NUMA_NODE_UNDEFINED UINT16_MAX
@@ -138,12 +141,12 @@ class GCEvent {
 typedef void (*GCThreadFunction)(void* param);
 
 #ifdef HOST_64BIT
-// Right now we support maximum 1024 procs - meaning that we will create at most
+// Right now we support maximum 1024 heaps - meaning that we will create at most
 // that many GC threads and GC heaps.
-#define MAX_SUPPORTED_CPUS 1024
+#define MAX_SUPPORTED_HEAPS 1024
 #define MAX_SUPPORTED_NODES 64
 #else
-#define MAX_SUPPORTED_CPUS 64
+#define MAX_SUPPORTED_HEAPS 64
 #define MAX_SUPPORTED_NODES 16
 #endif // HOST_64BIT
 
@@ -152,7 +155,8 @@ class AffinitySet
 {
     static const size_t BitsPerBitsetEntry = 8 * sizeof(uintptr_t);
 
-    uintptr_t m_bitset[MAX_SUPPORTED_CPUS / BitsPerBitsetEntry];
+    uintptr_t *m_bitset = nullptr;
+    size_t m_bitsetDataSize = 0;
 
     static uintptr_t GetBitsetEntryMask(size_t cpuIndex)
     {
@@ -166,11 +170,22 @@ class AffinitySet
 
 public:
 
-    static const size_t BitsetDataSize = MAX_SUPPORTED_CPUS / BitsPerBitsetEntry;
+    bool Initialize(int cpuCount)
+    {
+        m_bitsetDataSize = (cpuCount + BitsPerBitsetEntry - 1) / BitsPerBitsetEntry;
+        m_bitset = new (nothrow) uintptr_t[m_bitsetDataSize];
+        if (m_bitset == nullptr)
+        {
+            return false;
+        }
+
+        memset(m_bitset, 0, sizeof(uintptr_t) * m_bitsetDataSize);
+        return true;
+    }
 
-    AffinitySet()
+    ~AffinitySet()
     {
-        memset(m_bitset, 0, sizeof(m_bitset));
+        delete[] m_bitset;
     }
 
     uintptr_t* GetBitsetData()
@@ -181,25 +196,28 @@ class AffinitySet
     // Check if the set contains a processor
     bool Contains(size_t cpuIndex) const
     {
+        assert(GetBitsetEntryIndex(cpuIndex) < m_bitsetDataSize);
         return (m_bitset[GetBitsetEntryIndex(cpuIndex)] & GetBitsetEntryMask(cpuIndex)) != 0;
     }
 
     // Add a processor to the set
     void Add(size_t cpuIndex)
     {
+        assert(GetBitsetEntryIndex(cpuIndex) < m_bitsetDataSize);
         m_bitset[GetBitsetEntryIndex(cpuIndex)] |= GetBitsetEntryMask(cpuIndex);
     }
 
     // Remove a processor from the set
     void Remove(size_t cpuIndex)
     {
+        assert(GetBitsetEntryIndex(cpuIndex) < m_bitsetDataSize);
         m_bitset[GetBitsetEntryIndex(cpuIndex)] &= ~GetBitsetEntryMask(cpuIndex);
     }
 
     // Check if the set is empty
     bool IsEmpty() const
     {
-        for (size_t i = 0; i < MAX_SUPPORTED_CPUS / BitsPerBitsetEntry; i++)
+        for (size_t i = 0; i < m_bitsetDataSize; i++)
         {
             if (m_bitset[i] != 0)
             {
@@ -210,11 +228,17 @@ class AffinitySet
         return true;
     }
 
+    // Return the capacity of the affinity set (maximum number of processor indices it can hold)
+    size_t MaxCpuCount() const
+    {
+        return m_bitsetDataSize * BitsPerBitsetEntry;
+    }
+
     // Return number of processors in the affinity set
     size_t Count() const
     {
         size_t count = 0;
-        for (size_t i = 0; i < MAX_SUPPORTED_CPUS; i++)
+        for (size_t i = 0; i < m_bitsetDataSize * BitsPerBitsetEntry; i++)
         {
             if (Contains(i))
             {
@@ -479,6 +503,13 @@ class GCToOSInterface
     //  Number of processors on the machine
     static uint32_t GetTotalProcessorCount();
 
+    // Gets the maximum number of processors that could potentially exist on
+    // the machine (including offlined ones). Processor indices returned by
+    // GetCurrentProcessorNumber are guaranteed to be less than this value.
+    // Return:
+    //  Maximum number of processors
+    static uint32_t GetMaxProcessorCount();
+
     // Is NUMA support available
     static bool CanEnableGCNumaAware();
 

src/coreclr/gc/gc.cpp

@@ -650,7 +650,7 @@ class t_join
     gc_join_flavor flavor;
 
 #ifdef JOIN_STATS
-    uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
+    uint64_t start[MAX_SUPPORTED_HEAPS], end[MAX_SUPPORTED_HEAPS], start_seq;
     // remember join id and last thread to arrive so restart can use these
     int thd;
     // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
@@ -4502,10 +4502,10 @@ class heap_select
     static unsigned n_sniff_buffers;
     static unsigned cur_sniff_index;
 
-    static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
-    static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
-    static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
-    static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
+    static uint16_t *proc_no_to_heap_no;
+    static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_HEAPS];
+    static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_HEAPS];
+    static uint16_t *numa_node_to_heap_map;
 
 #ifdef HEAP_BALANCE_INSTRUMENTATION
     // Note this is the total numa nodes GC heaps are on. There might be
@@ -4529,6 +4529,16 @@ class heap_select
     static BOOL init(int n_heaps)
     {
         assert (sniff_buffer == NULL && n_sniff_buffers == 0);
+
+        uint32_t maxCpuCount = GCToOSInterface::GetMaxProcessorCount();
+        proc_no_to_heap_no = new (nothrow) uint16_t[maxCpuCount];
+        if (proc_no_to_heap_no == NULL)
+        {
+            return FALSE;
+        }
+
+        memset(proc_no_to_heap_no, 0, maxCpuCount*sizeof(uint16_t));
+
         if (!GCToOSInterface::CanGetCurrentProcessorNumber())
         {
             n_sniff_buffers = n_heaps*2+1;
@@ -4555,15 +4565,15 @@ class heap_select
         // 2. assign heap numbers for each numa node
 
         // Pass 1: gather processor numbers and numa node numbers
-        uint16_t proc_no[MAX_SUPPORTED_CPUS];
-        uint16_t node_no[MAX_SUPPORTED_CPUS];
+        uint16_t proc_no[MAX_SUPPORTED_HEAPS];
+        uint16_t node_no[MAX_SUPPORTED_HEAPS];
         uint16_t max_node_no = 0;
         uint16_t heap_num;
         for (heap_num = 0; heap_num < n_heaps; heap_num++)
         {
             if (!GCToOSInterface::GetProcessorForHeap (heap_num, &proc_no[heap_num], &node_no[heap_num]))
                 break;
-            assert(proc_no[heap_num] < MAX_SUPPORTED_CPUS);
+            assert(proc_no[heap_num] < GCToOSInterface::GetMaxProcessorCount());
             if (!do_numa || node_no[heap_num] == NUMA_NODE_UNDEFINED)
                 node_no[heap_num] = 0;
             max_node_no = max(max_node_no, node_no[heap_num]);
@@ -4594,11 +4604,7 @@ class heap_select
         if (GCToOSInterface::CanGetCurrentProcessorNumber())
         {
             uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber();
-            // For a 32-bit process running on a machine with > 64 procs,
-            // even though the process can only use up to 32 procs, the processor
-            // index can be >= 64; or in the cpu group case, if the process is not running in cpu group #0,
-            // the GetCurrentProcessorNumber will return a number that's >= 64.
-            proc_no_to_heap_no[proc_no % MAX_SUPPORTED_CPUS] = (uint16_t)heap_number;
+            proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
         }
     }
 
@@ -4620,11 +4626,7 @@ class heap_select
         if (GCToOSInterface::CanGetCurrentProcessorNumber())
         {
             uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber();
-            // For a 32-bit process running on a machine with > 64 procs,
-            // even though the process can only use up to 32 procs, the processor
-            // index can be >= 64; or in the cpu group case, if the process is not running in cpu group #0,
-            // the GetCurrentProcessorNumber will return a number that's >= 64.
-            int adjusted_heap = proc_no_to_heap_no[proc_no % MAX_SUPPORTED_CPUS];
+            int adjusted_heap = proc_no_to_heap_no[proc_no];
             // with dynamic heap count, need to make sure the value is in range.
             if (adjusted_heap >= gc_heap::n_heaps)
             {
@@ -4686,9 +4688,21 @@ class heap_select
         return heap_no_to_numa_node[heap_number];
     }
 
-    static void init_numa_node_to_heap_map(int nheaps)
+    static bool init_numa_node_to_heap_map(int nheaps)
     {
         // Called right after GCHeap::Init() for each heap
+
+        uint32_t maxCpuCount = GCToOSInterface::GetMaxProcessorCount();
+        // The upper limit of the numa node numbers is maxCpuCount - 1 since in the worst case each processor could be on a different NUMA node. 
+        // We add +1 here to make it easier to calculate the heap number range for the last NUMA node.
+        numa_node_to_heap_map = new (nothrow) uint16_t[maxCpuCount + 1];
+        if (numa_node_to_heap_map == nullptr)
+        {
+            return false;
+        }
+
+        memset(numa_node_to_heap_map, 0, (maxCpuCount + 1)*sizeof(uint16_t));
+
         // For each NUMA node used by the heaps, the
         // numa_node_to_heap_map[numa_node] is set to the first heap number on that node and
         // numa_node_to_heap_map[numa_node + 1] is set to the first heap number not on that node
@@ -4709,7 +4723,8 @@ class heap_select
                 total_numa_nodes++;
                 heaps_on_node[total_numa_nodes].node_no = heap_no_to_numa_node[i];
 #endif
-
+                assert(heap_no_to_numa_node[i-1] < maxCpuCount);
+                assert(heap_no_to_numa_node[i] < maxCpuCount);
                 // Set the end of the heap number range for the previous NUMA node
                 numa_node_to_heap_map[heap_no_to_numa_node[i-1] + 1] =
                 // Set the start of the heap number range for the current NUMA node
@@ -4720,12 +4735,14 @@ class heap_select
 #endif
         }
 
+        assert(heap_no_to_numa_node[nheaps-1] < maxCpuCount);
         // Set the end of the heap range for the last NUMA node
         numa_node_to_heap_map[heap_no_to_numa_node[nheaps-1] + 1] = (uint16_t)nheaps; //mark the end with nheaps
 
 #ifdef HEAP_BALANCE_INSTRUMENTATION
         total_numa_nodes++;
 #endif
+        return true;
     }
 
     static bool get_info_proc (int index, uint16_t* proc_no, uint16_t* node_no, int* start_heap, int* end_heap)
@@ -4749,7 +4766,7 @@ class heap_select
 
         if (distribute_all_p)
         {
-            uint16_t current_heap_no_on_node[MAX_SUPPORTED_CPUS];
+            uint16_t current_heap_no_on_node[MAX_SUPPORTED_HEAPS];
             memset (current_heap_no_on_node, 0, sizeof (current_heap_no_on_node));
             uint16_t current_heap_no = 0;
 
@@ -4831,10 +4848,10 @@ class heap_select
 uint8_t* heap_select::sniff_buffer;
 unsigned heap_select::n_sniff_buffers;
 unsigned heap_select::cur_sniff_index;
-uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
-uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
-uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
-uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
+uint16_t* heap_select::proc_no_to_heap_no;
+uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_HEAPS];
+uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_HEAPS];
+uint16_t* heap_select::numa_node_to_heap_map;
 #ifdef HEAP_BALANCE_INSTRUMENTATION
 uint16_t  heap_select::total_numa_nodes;
 node_heap_count heap_select::heaps_on_node[MAX_SUPPORTED_NODES];

src/coreclr/gc/gcconfig.cpp

@@ -175,7 +175,8 @@ bool ParseGCHeapAffinitizeRanges(const char* cpu_index_ranges, AffinitySet* conf
                     break;
                 }
 
-                if ((start_index >= MAX_SUPPORTED_CPUS) || (end_index >= MAX_SUPPORTED_CPUS) || (end_index < start_index))
+                size_t maxCpuCount = GCToOSInterface::GetMaxProcessorCount();
+                if ((start_index >= maxCpuCount) || (end_index >= maxCpuCount) || (end_index < start_index))
                 {
                     // Invalid CPU index values or range
                     break;

src/coreclr/gc/gcpriv.h

@@ -1755,7 +1755,7 @@ class gc_heap
     PER_HEAP_ISOLATED_METHOD void move_aged_regions(region_free_list dest[count_free_region_kinds], region_free_list& src, free_region_kind kind, bool joined_last_gc_before_oom);
     PER_HEAP_ISOLATED_METHOD bool aged_region_p(heap_segment* region, free_region_kind kind);
     PER_HEAP_ISOLATED_METHOD void move_regions_to_decommit(region_free_list oregions[count_free_region_kinds]);
-    PER_HEAP_ISOLATED_METHOD size_t compute_basic_region_budgets(size_t heap_basic_budget_in_region_units[MAX_SUPPORTED_CPUS], size_t min_heap_basic_budget_in_region_units[MAX_SUPPORTED_CPUS], size_t total_basic_free_regions);
+    PER_HEAP_ISOLATED_METHOD size_t compute_basic_region_budgets(size_t heap_basic_budget_in_region_units[MAX_SUPPORTED_HEAPS], size_t min_heap_basic_budget_in_region_units[MAX_SUPPORTED_HEAPS], size_t total_basic_free_regions);
     PER_HEAP_ISOLATED_METHOD bool near_heap_hard_limit_p();
     PER_HEAP_ISOLATED_METHOD bool distribute_surplus_p(ptrdiff_t balance, int kind, bool aggressive_decommit_large_p);
     PER_HEAP_ISOLATED_METHOD void decide_on_decommit_strategy(bool joined_last_gc_before_oom);

src/coreclr/gc/interface.cpp

@@ -198,6 +198,12 @@ HRESULT GCHeap::Initialize()
 #else //!MULTIPLE_HEAPS
     GCConfig::SetServerGC(true);
     AffinitySet config_affinity_set;
+    if (!config_affinity_set.Initialize(GCToOSInterface::GetMaxProcessorCount()))
+    {
+        log_init_error_to_host ("Failed to initialize affinity set for GC heap affinity configuration");
+        return E_OUTOFMEMORY;
+    }
+
     GCConfigStringHolder cpu_index_ranges_holder(GCConfig::GetGCHeapAffinitizeRanges());
 
     uintptr_t config_affinity_mask = static_cast<uintptr_t>(GCConfig::GetGCHeapAffinitizeMask());
@@ -240,7 +246,7 @@ HRESULT GCHeap::Initialize()
 
     nhp = ((nhp_from_config == 0) ? g_num_active_processors : nhp_from_config);
 
-    nhp = min (nhp, (uint32_t)MAX_SUPPORTED_CPUS);
+    nhp = min (nhp, (uint32_t)MAX_SUPPORTED_HEAPS);
 
     gc_heap::gc_thread_no_affinitize_p = (gc_heap::heap_hard_limit ?
         !affinity_config_specified_p : (GCConfig::GetNoAffinitize() != 0));
@@ -532,7 +538,11 @@ HRESULT GCHeap::Initialize()
         }
     }
 
-    heap_select::init_numa_node_to_heap_map (nhp);
+    if (!heap_select::init_numa_node_to_heap_map (nhp))
+    {
+        log_init_error_to_host ("Initialization of NUMA node to heap map failed");
+        return E_OUTOFMEMORY;
+    }
 
     // If we have more active processors than heaps we still want to initialize some of the
     // mapping for the rest of the active processors because user threads can still run on

src/coreclr/gc/mark_phase.cpp

@@ -273,7 +273,7 @@ static size_t target_mark_count_for_heap (size_t total_mark_count, int heap_coun
 NOINLINE
 uint8_t** gc_heap::equalize_mark_lists (size_t total_mark_list_size)
 {
-    size_t local_mark_count[MAX_SUPPORTED_CPUS];
+    size_t local_mark_count[MAX_SUPPORTED_HEAPS];
     size_t total_mark_count = 0;
 
     // compute mark count per heap into a local array
@@ -674,9 +674,9 @@ void gc_heap::merge_mark_lists (size_t total_mark_list_size)
     int source_number = (size_t)heap_number;
 #endif //USE_REGIONS
 
-    uint8_t** source[MAX_SUPPORTED_CPUS];
-    uint8_t** source_end[MAX_SUPPORTED_CPUS];
-    int source_heap[MAX_SUPPORTED_CPUS];
+    uint8_t** source[MAX_SUPPORTED_HEAPS];
+    uint8_t** source_end[MAX_SUPPORTED_HEAPS];
+    int source_heap[MAX_SUPPORTED_HEAPS];
     int source_count = 0;
 
     for (int i = 0; i < n_heaps; i++)
@@ -687,7 +687,7 @@ void gc_heap::merge_mark_lists (size_t total_mark_list_size)
             source[source_count] = heap->mark_list_piece_start[source_number];
             source_end[source_count] = heap->mark_list_piece_end[source_number];
             source_heap[source_count] = i;
-            if (source_count < MAX_SUPPORTED_CPUS)
+            if (source_count < MAX_SUPPORTED_HEAPS)
                 source_count++;
         }
     }
@@ -1141,7 +1141,7 @@ void gc_heap::equalize_promoted_bytes(int condemned_gen_number)
         //  compute total promoted bytes per gen
         size_t total_surv = 0;
         size_t max_surv_per_heap = 0;
-        size_t surv_per_heap[MAX_SUPPORTED_CPUS];
+        size_t surv_per_heap[MAX_SUPPORTED_HEAPS];
         for (int i = 0; i < n_heaps; i++)
         {
             surv_per_heap[i] = 0;
@@ -1282,7 +1282,7 @@ void gc_heap::equalize_promoted_bytes(int condemned_gen_number)
             surplus_regions_by_size_class[size_class] = region;
         }
 
-        int next_heap_in_size_class[MAX_SUPPORTED_CPUS];
+        int next_heap_in_size_class[MAX_SUPPORTED_HEAPS];
         int heaps_by_deficit_size_class[NUM_SIZE_CLASSES];
         for (int i = 0; i < NUM_SIZE_CLASSES; i++)
         {