[Release/8.0] Fix FP state restore on macOS exception forwarding (#109579)

Backport of #105003, #109458 and part of #99255

Author: janvorli

src/coreclr/pal/src/exception/machexception.cpp

@@ -849,7 +849,7 @@ HijackFaultingThread(
     if (fIsStackOverflow)
     {
         // Allocate the minimal stack necessary for handling stack overflow
-        int stackOverflowStackSize = 7 * 4096;
+        int stackOverflowStackSize = 15 * 4096;
         // Align the size to virtual page size and add one virtual page as a stack guard
         stackOverflowStackSize = ALIGN_UP(stackOverflowStackSize, GetVirtualPageSize()) + GetVirtualPageSize();
         void* stackOverflowHandlerStack = mmap(NULL, stackOverflowStackSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
@@ -1325,7 +1325,7 @@ void MachExceptionInfo::RestoreState(mach_port_t thread)
     kern_return_t machret = thread_set_state(thread, x86_THREAD_STATE, (thread_state_t)&ThreadState, x86_THREAD_STATE_COUNT);
     CHECK_MACH("thread_set_state(thread)", machret);
 
-    machret = thread_set_state(thread, x86_FLOAT_STATE, (thread_state_t)&FloatState, x86_FLOAT_STATE_COUNT);
+    machret = thread_set_state(thread, FloatState.ash.flavor, (thread_state_t)&FloatState.ufs, FloatState.ash.count);
     CHECK_MACH("thread_set_state(float)", machret);
 
     machret = thread_set_state(thread, x86_DEBUG_STATE, (thread_state_t)&DebugState, x86_DEBUG_STATE_COUNT);

src/coreclr/pal/src/exception/machmessage.h

@@ -186,11 +186,6 @@ class MachMessage
     void ReplyToNotification(MachMessage& message, kern_return_t eResult);
 
 private:
-    // The maximum size in bytes of any Mach message we can send or receive. Calculating an exact size for
-    // this is non trivial (basically because of the security trailers that Mach appends) but the current
-    // value has proven to be more than enough so far.
-    static const size_t kcbMaxMessageSize = 1500;
-
     // The following are structures describing the formats of the Mach messages we understand.
 
     // Request to set the register context on a particular thread.
@@ -298,6 +293,9 @@ class MachMessage
         } data;
     } __attribute__((packed));;
 
+    // The maximum size in bytes of any Mach message we can send or receive including possible trailers
+    static const size_t kcbMaxMessageSize = sizeof(mach_message_t) + MAX_TRAILER_SIZE;
+
     // Re-initializes this data structure (to the same state as default construction, containing no message).
     void ResetMessage();