Distinguish OOM and failure to reprotect in thunks APIs

src/coreclr/nativeaot/Runtime/PalRedhawk.h

@@ -68,6 +68,10 @@
 // we have to (in which case these definitions will move to CommonTypes.h).
 typedef int32_t             HRESULT;
 
+#define S_OK  0x0
+#define E_FAIL 0x80004005
+#define E_OUTOFMEMORY 0x8007000E
+
 typedef WCHAR *             LPWSTR;
 typedef const WCHAR *       LPCWSTR;
 typedef char *              LPSTR;

src/coreclr/nativeaot/Runtime/ThunksMapping.cpp

@@ -100,13 +100,15 @@ FCIMPL0(int, RhpGetThunkBlockSize)
 }
 FCIMPLEND
 
-EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
+EXTERN_C HRESULT QCALLTYPE RhAllocateThunksMapping(void** ppThunksSection)
 {
 #ifdef WIN32
 
     void * pNewMapping = PalVirtualAlloc(THUNKS_MAP_SIZE * 2, PAGE_READWRITE);
     if (pNewMapping == NULL)
-        return NULL;
+    {
+        return E_OUTOFMEMORY;
+    }
 
     void * pThunksSection = pNewMapping;
     void * pDataSection = (uint8_t*)pNewMapping + THUNKS_MAP_SIZE;
@@ -120,7 +122,9 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
     // changed anymore.
     void * pNewMapping = PalVirtualAlloc(THUNKS_MAP_SIZE * 2, PAGE_EXECUTE_READ);
     if (pNewMapping == NULL)
-        return NULL;
+    {
+        return E_OUTOFMEMORY;
+    }
 
     void * pThunksSection = pNewMapping;
     void * pDataSection = (uint8_t*)pNewMapping + THUNKS_MAP_SIZE;
@@ -129,7 +133,7 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
         !PalVirtualProtect(pThunksSection, THUNKS_MAP_SIZE, PAGE_EXECUTE_READWRITE))
     {
         PalVirtualFree(pNewMapping, THUNKS_MAP_SIZE * 2);
-        return NULL;
+        return E_FAIL;
     }
 
 #if defined(HOST_APPLE) && defined(HOST_ARM64)
@@ -306,13 +310,14 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
     if (!PalVirtualProtect(pThunksSection, THUNKS_MAP_SIZE, PAGE_EXECUTE_READ))
     {
         PalVirtualFree(pNewMapping, THUNKS_MAP_SIZE * 2);
-        return NULL;
+        return E_FAIL;
     }
 #endif
 
     PalFlushInstructionCache(pThunksSection, THUNKS_MAP_SIZE);
 
-    return pThunksSection;
+    *ppThunksSection = pThunksSection;
+    return S_OK;
 }
 
 // FEATURE_RX_THUNKS
@@ -327,7 +332,7 @@ FCDECL0(int, RhpGetThunkBlockSize);
 FCDECL1(void*, RhpGetThunkDataBlockAddress, void* addr);
 FCDECL1(void*, RhpGetThunkStubsBlockAddress, void* addr);
 
-EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
+EXTERN_C HRESULT QCALLTYPE RhAllocateThunksMapping(void** ppThunksSection)
 {
     static int nextThunkDataMapping = 0;
 
@@ -342,7 +347,7 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
 
     if (nextThunkDataMapping == thunkDataMappingCount)
     {
-        return NULL;
+        return E_FAIL;
     }
 
     if (g_pThunkStubData == NULL)
@@ -353,23 +358,24 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
 
         if (g_pThunkStubData == NULL)
         {
-            return NULL;
+            return E_OUTOFMEMORY;
         }
     }
 
     void* pThunkDataBlock = (int8_t*)g_pThunkStubData + nextThunkDataMapping * thunkDataMappingSize;
 
     if (VirtualAlloc(pThunkDataBlock, thunkDataMappingSize, MEM_COMMIT, PAGE_READWRITE) == NULL)
     {
-        return NULL;
+        return E_OUTOFMEMORY;
     }
 
     nextThunkDataMapping++;
 
     void* pThunks = RhpGetThunkStubsBlockAddress(pThunkDataBlock);
     ASSERT(RhpGetThunkDataBlockAddress(pThunks) == pThunkDataBlock);
 
-    return pThunks;
+    *ppThunksSection = pThunks;
+    return S_OK;
 }
 
 #else // FEATURE_FIXED_POOL_THUNKS
@@ -380,7 +386,7 @@ FCDECL0(int, RhpGetNumThunksPerBlock);
 FCDECL0(int, RhpGetThunkSize);
 FCDECL0(int, RhpGetThunkBlockSize);
 
-EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
+EXTERN_C HRESULT QCALLTYPE RhAllocateThunksMapping(void** ppThunksSection)
 {
     static void* pThunksTemplateAddress = NULL;
 
@@ -409,7 +415,7 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
         int templateRva = (int)((uint8_t*)RhpGetThunksBase() - pModuleBase);
 
         if (!PalAllocateThunksFromTemplate((HANDLE)pModuleBase, templateRva, templateSize, &pThunkMap))
-            return NULL;
+            return E_OUTOFMEMORY;
     }
 
     if (!PalMarkThunksAsValidCallTargets(
@@ -422,10 +428,11 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping()
         if (pThunkMap != pThunksTemplateAddress)
             PalFreeThunksFromTemplate(pThunkMap, templateSize);
 
-        return NULL;
+        return E_FAIL;
     }
 
-    return pThunkMap;
+    *ppThunksSection = pThunkMap;
+    return S_OK;
 }
 
 #endif // FEATURE_RX_THUNKS

src/coreclr/nativeaot/Runtime/portable.cpp

@@ -365,9 +365,9 @@ FCIMPL2(Object *, RhpCheckedXchg, Object ** location, Object * value)
 }
 FCIMPLEND
 
-FCIMPL0(void*, RhAllocateThunksMapping)
+FCIMPL1(HRESULT, RhAllocateThunksMapping, void ** ppThunksSection)
 {
-    return NULL;
+    return E_FAIL;
 }
 FCIMPLEND
 

src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeAugments.cs

@@ -711,18 +711,12 @@ public static string TryGetMethodDisplayStringFromIp(IntPtr ip)
 
         public static object CreateThunksHeap(IntPtr commonStubAddress)
         {
-            object? newHeap = ThunksHeap.CreateThunksHeap(commonStubAddress);
-            if (newHeap == null)
-                throw new OutOfMemoryException();
-            return newHeap;
+            return ThunksHeap.CreateThunksHeap(commonStubAddress);
         }
 
         public static IntPtr AllocateThunk(object thunksHeap)
         {
-            IntPtr newThunk = ((ThunksHeap)thunksHeap).AllocateThunk();
-            if (newThunk == IntPtr.Zero)
-                throw new OutOfMemoryException();
-            return newThunk;
+            return ((ThunksHeap)thunksHeap).AllocateThunk();
         }
 
         public static void FreeThunk(object thunksHeap, IntPtr thunkAddress)

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs

@@ -358,7 +358,7 @@ internal static IntPtr RhHandleAllocDependent(object primary, object secondary)
         internal static extern int RhpGetThunkBlockSize();
 
         [LibraryImport(RuntimeLibrary, EntryPoint = "RhAllocateThunksMapping")]
-        internal static partial IntPtr RhAllocateThunksMapping();
+        internal static unsafe partial int RhAllocateThunksMapping(IntPtr* ppMapping);
 
         //
         // calls to runtime for type equality checks

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/ThunkPool.cs

@@ -87,43 +87,26 @@ private unsafe ThunksHeap(IntPtr commonStubAddress)
 
             _allocatedBlocks = new AllocatedBlock();
 
-            IntPtr thunkStubsBlock;
-            lock (this)
-            {
-                thunkStubsBlock = ThunkBlocks.GetNewThunksBlock();
-            }
-
-            if (thunkStubsBlock != IntPtr.Zero)
-            {
-                IntPtr thunkDataBlock = RuntimeImports.RhpGetThunkDataBlockAddress(thunkStubsBlock);
+            IntPtr thunkStubsBlock = ThunkBlocks.GetNewThunksBlock();
+            IntPtr thunkDataBlock = RuntimeImports.RhpGetThunkDataBlockAddress(thunkStubsBlock);
 
-                // Address of the first thunk data cell should be at the beginning of the thunks data block (page-aligned)
-                Debug.Assert(((nuint)(nint)thunkDataBlock % Constants.PageSize) == 0);
+            // Address of the first thunk data cell should be at the beginning of the thunks data block (page-aligned)
+            Debug.Assert(((nuint)(nint)thunkDataBlock % Constants.PageSize) == 0);
 
-                // Update the last pointer value in the thunks data section with the value of the common stub address
-                *(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) = commonStubAddress;
-                Debug.Assert(*(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) == commonStubAddress);
+            // Update the last pointer value in the thunks data section with the value of the common stub address
+            *(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) = commonStubAddress;
+            Debug.Assert(*(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) == commonStubAddress);
 
-                // Set the head and end of the linked list
-                _nextAvailableThunkPtr = thunkDataBlock;
-                _lastThunkPtr = _nextAvailableThunkPtr + Constants.ThunkDataSize * (Constants.NumThunksPerBlock - 1);
+            // Set the head and end of the linked list
+            _nextAvailableThunkPtr = thunkDataBlock;
+            _lastThunkPtr = _nextAvailableThunkPtr + Constants.ThunkDataSize * (Constants.NumThunksPerBlock - 1);
 
-                _allocatedBlocks._blockBaseAddress = thunkStubsBlock;
-            }
+            _allocatedBlocks._blockBaseAddress = thunkStubsBlock;
         }
 
-        public static unsafe ThunksHeap? CreateThunksHeap(IntPtr commonStubAddress)
+        public static unsafe ThunksHeap CreateThunksHeap(IntPtr commonStubAddress)
         {
-            try
-            {
-                ThunksHeap newHeap = new ThunksHeap(commonStubAddress);
-
-                if (newHeap._nextAvailableThunkPtr != IntPtr.Zero)
-                    return newHeap;
-            }
-            catch (Exception) { }
-
-            return null;
+            return new ThunksHeap(commonStubAddress);
         }
 
         // TODO: Feature
@@ -134,47 +117,30 @@ private unsafe ThunksHeap(IntPtr commonStubAddress)
         //
         // Note: Expected to be called under lock
         //
-        private unsafe bool ExpandHeap()
+        private unsafe void ExpandHeap()
         {
-            AllocatedBlock newBlockInfo;
-
-            try
-            {
-                newBlockInfo = new AllocatedBlock();
-            }
-            catch (Exception)
-            {
-                return false;
-            }
+            AllocatedBlock newBlockInfo = new AllocatedBlock();
 
             IntPtr thunkStubsBlock = ThunkBlocks.GetNewThunksBlock();
+            IntPtr thunkDataBlock = RuntimeImports.RhpGetThunkDataBlockAddress(thunkStubsBlock);
 
-            if (thunkStubsBlock != IntPtr.Zero)
-            {
-                IntPtr thunkDataBlock = RuntimeImports.RhpGetThunkDataBlockAddress(thunkStubsBlock);
-
-                // Address of the first thunk data cell should be at the beginning of the thunks data block (page-aligned)
-                Debug.Assert(((nuint)(nint)thunkDataBlock % Constants.PageSize) == 0);
-
-                // Update the last pointer value in the thunks data section with the value of the common stub address
-                *(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) = _commonStubAddress;
-                Debug.Assert(*(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) == _commonStubAddress);
-
-                // Link the last entry in the old list to the first entry in the new list
-                *((IntPtr*)_lastThunkPtr) = thunkDataBlock;
+            // Address of the first thunk data cell should be at the beginning of the thunks data block (page-aligned)
+            Debug.Assert(((nuint)(nint)thunkDataBlock % Constants.PageSize) == 0);
 
-                // Update the pointer to the last entry in the list
-                _lastThunkPtr = *((IntPtr*)_lastThunkPtr) + Constants.ThunkDataSize * (Constants.NumThunksPerBlock - 1);
+            // Update the last pointer value in the thunks data section with the value of the common stub address
+            *(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) = _commonStubAddress;
+            Debug.Assert(*(IntPtr*)(thunkDataBlock + (int)(Constants.PageSize - IntPtr.Size)) == _commonStubAddress);
 
-                newBlockInfo._blockBaseAddress = thunkStubsBlock;
-                newBlockInfo._nextBlock = _allocatedBlocks;
+            // Link the last entry in the old list to the first entry in the new list
+            *((IntPtr*)_lastThunkPtr) = thunkDataBlock;
 
-                _allocatedBlocks = newBlockInfo;
+            // Update the pointer to the last entry in the list
+            _lastThunkPtr = *((IntPtr*)_lastThunkPtr) + Constants.ThunkDataSize * (Constants.NumThunksPerBlock - 1);
 
-                return true;
-            }
+            newBlockInfo._blockBaseAddress = thunkStubsBlock;
+            newBlockInfo._nextBlock = _allocatedBlocks;
 
-            return false;
+            _allocatedBlocks = newBlockInfo;
         }
 
         public unsafe IntPtr AllocateThunk()
@@ -192,10 +158,7 @@ public unsafe IntPtr AllocateThunk()
 
                 if (nextNextAvailableThunkPtr == IntPtr.Zero)
                 {
-                    if (!ExpandHeap())
-                    {
-                        return IntPtr.Zero;
-                    }
+                    ExpandHeap();
 
                     nextAvailableThunkPtr = _nextAvailableThunkPtr;
                     nextNextAvailableThunkPtr = *((IntPtr*)(nextAvailableThunkPtr));
@@ -347,19 +310,12 @@ public static unsafe IntPtr GetNewThunksBlock()
             }
             else
             {
-                nextThunksBlock = RuntimeImports.RhAllocateThunksMapping();
-
-                if (nextThunksBlock == IntPtr.Zero)
-                {
-                    // We either ran out of memory and can't do anymore mappings of the thunks templates sections,
-                    // or we are using the managed runtime services fallback, which doesn't provide the
-                    // file mapping feature (ex: older version of mrt100.dll, or no mrt100.dll at all).
-
-                    // The only option is for the caller to attempt and recycle unused thunks to be able to
-                    // find some free entries.
-
-                    return IntPtr.Zero;
-                }
+                nextThunksBlock = IntPtr.Zero;
+                int result = RuntimeImports.RhAllocateThunksMapping(&nextThunksBlock);
+                if (result == HResults.E_OUTOFMEMORY)
+                    throw new OutOfMemoryException();
+                else if (result != HResults.S_OK)
+                    throw new PlatformNotSupportedException(SR.PlatformNotSupported_DynamicEntrypoint);
 
                 // Each mapping consists of multiple blocks of thunk stubs/data pairs. Keep track of those
                 // so that we do not create a new mapping until all blocks in the sections we just mapped are consumed

src/libraries/Common/src/System/HResults.cs

@@ -121,6 +121,7 @@ internal static partial class HResults
         internal const int E_HANDLE = unchecked((int)0x80070006);
         internal const int E_INVALIDARG = unchecked((int)0x80070057);
         internal const int E_NOTIMPL = unchecked((int)0x80004001);
+        internal const int E_OUTOFMEMORY = unchecked((int)0x8007000E);
         internal const int E_POINTER = unchecked((int)0x80004003);
         internal const int ERROR_MRM_MAP_NOT_FOUND = unchecked((int)0x80073B1F);
         internal const int ERROR_TIMEOUT = unchecked((int)0x800705B4);