diff --git a/src/coreclr/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs index 2a375e74035f0d..69c7ee7b4abb91 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs @@ -10,13 +10,13 @@ namespace System public partial class Buffer { [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "Buffer_Clear")] - private static unsafe partial void __ZeroMemory(void* b, nuint byteLength); + private static unsafe partial void ZeroMemoryInternal(void* b, nuint byteLength); [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "Buffer_MemMove")] - private static unsafe partial void __Memmove(byte* dest, byte* src, nuint len); + private static unsafe partial void MemmoveInternal(byte* dest, byte* src, nuint len); [MethodImpl(MethodImplOptions.InternalCall)] - private static extern void __BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount); + private static extern void BulkMoveWithWriteBarrierInternal(ref byte destination, ref byte source, nuint byteCount); // Used by ilmarshalers.cpp internal static unsafe void Memcpy(byte* dest, byte* src, int len) diff --git a/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs index babcf894736ddf..184ce80f44b899 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs @@ -2,14 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. /*============================================================ -** -** -** ** Purpose: Exposes features of the Garbage Collector through ** the class libraries. This is a class which cannot be ** instantiated. -** -** ===========================================================*/ using System.Collections.Generic; @@ -347,13 +342,17 @@ public static void WaitForPendingFinalizers() // Indicates that the system should not call the Finalize() method on // an object that would normally require this call. [MethodImpl(MethodImplOptions.InternalCall)] - private static extern void _SuppressFinalize(object o); + private static extern void SuppressFinalizeInternal(object o); - public static void SuppressFinalize(object obj) + public static unsafe void SuppressFinalize(object obj) { ArgumentNullException.ThrowIfNull(obj); - _SuppressFinalize(obj); + MethodTable* pMT = RuntimeHelpers.GetMethodTable(obj); + if (pMT->HasFinalizer) + { + SuppressFinalizeInternal(obj); + } } // Indicates that the system should call the Finalize() method on an object @@ -600,12 +599,14 @@ private static bool InvokeMemoryLoadChangeNotifications() return true; } - // We need to take a snapshot of s_notifications.Count, so that in the case that s_notifications[i].Notification() registers new notifications, - // we neither get rid of them nor iterate over them + // We need to take a snapshot of s_notifications.Count, so that in the case that + // s_notifications[i].Notification() registers new notifications, we neither get rid + // of them nor iterate over them. int count = s_notifications.Count; - // If there is no existing notifications, we won't be iterating over any and we won't be adding any new one. Also, there wasn't any added since - // we last invoked this method so it's safe to assume we can reset s_previousMemoryLoad. + // If there is no existing notifications, we won't be iterating over any and we won't + // be adding any new one. Also, there wasn't any added since we last invoked this + // method so it's safe to assume we can reset s_previousMemoryLoad. if (count == 0) { s_previousMemoryLoad = float.MaxValue; diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs index c646bada45e01c..9b78cde00d6141 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs @@ -491,19 +491,24 @@ private void ResetFinalizerThreadSlow() } } + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern bool CatchAtSafePoint(); + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ThreadNative_PollGC")] - private static partial void ThreadNative_PollGC(); + private static partial void PollGCInternal(); // GC Suspension is done by simply dropping into native code via p/invoke, and we reuse the p/invoke // mechanism for suspension. On all architectures we should have the actual stub used for the check be implemented // as a small assembly stub which checks the global g_TrapReturningThreads flag and tail-call to this helper private static unsafe void PollGC() { - NativeThreadState catchAtSafePoint = ((NativeThreadClass*)Thread.DirectOnThreadLocalData.pNativeThread)->m_State & NativeThreadState.TS_CatchAtSafePoint; - if (catchAtSafePoint != NativeThreadState.None) + if (CatchAtSafePoint()) { - ThreadNative_PollGC(); + PollGCWorker(); } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void PollGCWorker() => PollGCInternal(); } [StructLayout(LayoutKind.Sequential)] diff --git a/src/coreclr/classlibnative/bcltype/objectnative.cpp b/src/coreclr/classlibnative/bcltype/objectnative.cpp index 56ed9d3ee84515..1ee90e337c2cdf 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.cpp +++ b/src/coreclr/classlibnative/bcltype/objectnative.cpp @@ -83,8 +83,6 @@ FCIMPL2(FC_BOOL_RET, ObjectNative::ContentEquals, Object *pThisRef, Object *pCom pCompareRef->GetData(), pThisMT->GetNumInstanceFieldBytes()) == 0; - FC_GC_POLL_RET(); - FC_RETURN_BOOL(ret); } FCIMPLEND diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 1be19a765767c7..a650d3cc2a596f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11878,6 +11878,7 @@ class GenTreeVisitor case GT_IL_OFFSET: case GT_NOP: case GT_SWIFT_ERROR: + case GT_GCPOLL: break; // Lclvar unary operators diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index ec44e86e45ef04..ce51a90ca02a99 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4408,6 +4408,7 @@ void GenTree::VisitOperands(TVisitor visitor) case GT_IL_OFFSET: case GT_NOP: case GT_SWIFT_ERROR: + case GT_GCPOLL: return; // Unary operators with an optional operand diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index 08ac83188ccf85..1f295a08d47158 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -48,6 +48,10 @@ static bool blockNeedsGCPoll(BasicBlock* block) blockMayNeedGCPoll = true; } } + else if (tree->OperGet() == GT_GCPOLL) + { + blockMayNeedGCPoll = true; + } } } } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 022c750bdc89ea..3d938a381013fd 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -2670,6 +2670,7 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) case GT_NOP: case GT_LABEL: case GT_SWIFT_ERROR: + case GT_GCPOLL: return true; default: @@ -5239,6 +5240,7 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) break; case GT_NOP: + case GT_GCPOLL: level = 0; costEx = 0; costSz = 0; @@ -6584,6 +6586,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) case GT_IL_OFFSET: case GT_NOP: case GT_SWIFT_ERROR: + case GT_GCPOLL: return false; // Standard unary operators @@ -6912,6 +6915,7 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) const case GT_CALL: return true; + case GT_GCPOLL: case GT_KEEPALIVE: return true; @@ -7243,6 +7247,7 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const case GT_MEMORYBARRIER: case GT_KEEPALIVE: case GT_SWIFT_ERROR: + case GT_GCPOLL: return true; case GT_CALL: @@ -9430,6 +9435,7 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) case GT_NOP: case GT_LABEL: case GT_SWIFT_ERROR: + case GT_GCPOLL: copy = new (this, oper) GenTree(oper, tree->gtType); goto DONE; @@ -10201,6 +10207,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_IL_OFFSET: case GT_NOP: case GT_SWIFT_ERROR: + case GT_GCPOLL: m_state = -1; return; @@ -12349,6 +12356,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) case GT_PINVOKE_PROLOG: case GT_JMPTABLE: case GT_SWIFT_ERROR: + case GT_GCPOLL: break; case GT_RET_EXPR: diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 75726fc2ef713f..ff7ed26f311e61 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -37,6 +37,7 @@ GTNODE(LABEL , GenTree ,0,0,GTK_LEAF) // Jump- GTNODE(JMP , GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE) // Jump to another function GTNODE(FTN_ADDR , GenTreeFptrVal ,0,0,GTK_LEAF) // Address of a function GTNODE(RET_EXPR , GenTreeRetExpr ,0,0,GTK_LEAF|DBK_NOTLIR) // Place holder for the return expression from an inline candidate +GTNODE(GCPOLL , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTLIR) //----------------------------------------------------------------------------- // Constant nodes: diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 4db4d09c751871..da79c603b67b11 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4090,6 +4090,22 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, break; } + case NI_System_Threading_Thread_FastPollGC: + { + optMethodFlags |= OMF_NEEDS_GCPOLLS; + compCurBB->SetFlags(BBF_NEEDS_GCPOLL); + + GenTree* gcpoll = new (this, GT_GCPOLL) GenTree(GT_GCPOLL, TYP_VOID); + // Prevent both reordering and removal. Invalid optimizations of Thread.FastPollGC are + // very subtle and hard to observe. Thus we are conservatively marking it with both + // GTF_CALL and GTF_GLOB_REF side-effects even though it may be more than strictly + // necessary. The conservative side-effects are unlikely to have negative impact + // on code quality in this case. + gcpoll->gtFlags |= (GTF_CALL | GTF_GLOB_REF); + retNode = gcpoll; + break; + } + #if defined(TARGET_ARM64) || defined(TARGET_RISCV64) || defined(TARGET_XARCH) case NI_System_Threading_Interlocked_Or: case NI_System_Threading_Interlocked_And: @@ -11145,6 +11161,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_Threading_Thread_get_ManagedThreadId; } + else if (strcmp(methodName, "FastPollGC") == 0) + { + result = NI_System_Threading_Thread_FastPollGC; + } } else if (strcmp(className, "Volatile") == 0) { diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 726b4a5e3b2763..b199d9b6612509 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -1470,6 +1470,7 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR case GT_IL_OFFSET: case GT_KEEPALIVE: case GT_SWIFT_ERROR_RET: + case GT_GCPOLL: // Never remove these nodes, as they are always side-effecting. // // NOTE: the only side-effect of some of these nodes (GT_CMP, GT_SUB_HI) is a write to the flags diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index baf060aaec2e77..ddee451a200bea 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -75,6 +75,7 @@ enum NamedIntrinsic : unsigned short NI_System_Threading_Thread_get_CurrentThread, NI_System_Threading_Thread_get_ManagedThreadId, + NI_System_Threading_Thread_FastPollGC, NI_System_Threading_Volatile_Read, NI_System_Threading_Volatile_Write, NI_System_Threading_Volatile_ReadBarrier, diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp index 5e298345fc4505..9afb109802e85f 100644 --- a/src/coreclr/jit/optcse.cpp +++ b/src/coreclr/jit/optcse.cpp @@ -1908,6 +1908,7 @@ bool CSE_HeuristicCommon::CanConsiderTree(GenTree* tree, bool isReturn) case GT_COLON: case GT_QMARK: case GT_NOP: + case GT_GCPOLL: case GT_RETURN: return false; // Currently the only special nodes that we hit // that we know that we don't want to CSE diff --git a/src/coreclr/jit/rationalize.cpp b/src/coreclr/jit/rationalize.cpp index 9c7a8c6a112c28..be2e6f619447b0 100644 --- a/src/coreclr/jit/rationalize.cpp +++ b/src/coreclr/jit/rationalize.cpp @@ -738,6 +738,13 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, Compiler::Ge } break; + case GT_GCPOLL: + { + // GCPOLL is essentially a no-op, we used it as a hint for fgCreateGCPoll + node->gtBashToNOP(); + return Compiler::WALK_CONTINUE; + } + case GT_COMMA: { GenTree* op1 = node->gtGetOp1(); diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 9e223650129306..8f34f02d9b97c9 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -12276,6 +12276,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) // These do not represent values. case GT_NO_OP: case GT_NOP: + case GT_GCPOLL: case GT_JMP: // Control flow case GT_LABEL: // Control flow #if defined(FEATURE_EH_WINDOWS_X86) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Buffer.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Buffer.NativeAot.cs index caf6d9b7089a3d..4ceecdf70f664b 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Buffer.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Buffer.NativeAot.cs @@ -12,15 +12,15 @@ namespace System public static partial class Buffer { [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe void __ZeroMemory(void* b, nuint byteLength) => + private static unsafe void ZeroMemoryInternal(void* b, nuint byteLength) => RuntimeImports.memset((byte*)b, 0, byteLength); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe void __Memmove(byte* dest, byte* src, nuint len) => + private static unsafe void MemmoveInternal(byte* dest, byte* src, nuint len) => RuntimeImports.memmove(dest, src, len); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void __BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) => + private static void BulkMoveWithWriteBarrierInternal(ref byte destination, ref byte source, nuint byteCount) => RuntimeImports.RhBulkMoveWithWriteBarrier(ref destination, ref source, byteCount); } } diff --git a/src/coreclr/vm/comsynchronizable.cpp b/src/coreclr/vm/comsynchronizable.cpp index 2e4170a81ef838..c759df307a71be 100644 --- a/src/coreclr/vm/comsynchronizable.cpp +++ b/src/coreclr/vm/comsynchronizable.cpp @@ -714,6 +714,14 @@ FCIMPL1(void, ThreadNative::Finalize, ThreadBaseObject* pThisUNSAFE) } FCIMPLEND +FCIMPL0(FC_BOOL_RET, ThreadNative::CatchAtSafePoint) +{ + FCALL_CONTRACT; + + FC_RETURN_BOOL(GetThread()->CatchAtSafePoint()); +} +FCIMPLEND + // Get whether or not this is a background thread. extern "C" BOOL QCALLTYPE ThreadNative_GetIsBackground(QCall::ThreadHandle thread) { diff --git a/src/coreclr/vm/comsynchronizable.h b/src/coreclr/vm/comsynchronizable.h index 9b17981e16e7a3..004da5691c7a65 100644 --- a/src/coreclr/vm/comsynchronizable.h +++ b/src/coreclr/vm/comsynchronizable.h @@ -39,8 +39,9 @@ class ThreadNative ThreadAbortRequested = 128, }; - static FCDECL0(INT32, GetOptimalMaxSpinWaitsPerSpinIteration); - static FCDECL1(void, Finalize, ThreadBaseObject* pThis); + static FCDECL0(INT32, GetOptimalMaxSpinWaitsPerSpinIteration); + static FCDECL1(void, Finalize, ThreadBaseObject* pThis); + static FCDECL0(FC_BOOL_RET, CatchAtSafePoint); }; extern "C" void QCALLTYPE ThreadNative_Start(QCall::ThreadHandle thread, int threadStackSize, int priority, BOOL isThreadPool, PCWSTR pThreadName); diff --git a/src/coreclr/vm/comutilnative.cpp b/src/coreclr/vm/comutilnative.cpp index 91a1c8c68f2166..d039226c328cad 100644 --- a/src/coreclr/vm/comutilnative.cpp +++ b/src/coreclr/vm/comutilnative.cpp @@ -547,24 +547,22 @@ extern "C" void QCALLTYPE Buffer_Clear(void *dst, size_t length) memset(dst, 0, length); } +extern "C" void QCALLTYPE Buffer_MemMove(void *dst, void *src, size_t length) +{ + QCALL_CONTRACT; + + memmove(dst, src, length); +} + FCIMPL3(VOID, Buffer::BulkMoveWithWriteBarrier, void *dst, void *src, size_t byteCount) { FCALL_CONTRACT; if (dst != src && byteCount != 0) InlinedMemmoveGCRefsHelper(dst, src, byteCount); - - FC_GC_POLL(); } FCIMPLEND -extern "C" void QCALLTYPE Buffer_MemMove(void *dst, void *src, size_t length) -{ - QCALL_CONTRACT; - - memmove(dst, src, length); -} - // // GCInterface // @@ -736,9 +734,7 @@ FCIMPL1(int, GCInterface::GetGenerationInternal, Object* objUNSAFE) { FCALL_CONTRACT; _ASSERTE(objUNSAFE != NULL); - int result = (INT32)GCHeapUtilities::GetGCHeap()->WhichGeneration(objUNSAFE); - FC_GC_POLL_RET(); - return result; + return (INT32)GCHeapUtilities::GetGCHeap()->WhichGeneration(objUNSAFE); } FCIMPLEND @@ -756,8 +752,6 @@ FCIMPL0(UINT64, GCInterface::GetSegmentSize) _ASSERTE(segment_size < SIZE_T_MAX && large_segment_size < SIZE_T_MAX); if (segment_size < large_segment_size) segment_size = large_segment_size; - - FC_GC_POLL_RET(); return (UINT64) segment_size; } FCIMPLEND @@ -776,9 +770,7 @@ FCIMPL2(int, GCInterface::CollectionCount, INT32 generation, INT32 getSpecialGCC _ASSERTE(generation >= 0); //We don't need to check the top end because the GC will take care of that. - int result = (INT32)GCHeapUtilities::GetGCHeap()->CollectionCount(generation, getSpecialGCCount); - FC_GC_POLL_RET(); - return result; + return (INT32)GCHeapUtilities::GetGCHeap()->CollectionCount(generation, getSpecialGCCount); } FCIMPLEND @@ -1110,14 +1102,8 @@ FCIMPL1(void, GCInterface::SuppressFinalize, Object *obj) { FCALL_CONTRACT; - // Checked by the caller - _ASSERTE(obj != NULL); - - if (!obj->GetMethodTable ()->HasFinalizer()) - return; - + _ASSERTE(obj->GetMethodTable ()->HasFinalizer()); GCHeapUtilities::GetGCHeap()->SetFinalizationRun(obj); - FC_GC_POLL(); } FCIMPLEND diff --git a/src/coreclr/vm/comutilnative.h b/src/coreclr/vm/comutilnative.h index 2d9116e135c590..1f5fda9ed02ccc 100644 --- a/src/coreclr/vm/comutilnative.h +++ b/src/coreclr/vm/comutilnative.h @@ -87,8 +87,8 @@ class Buffer static FCDECL3(VOID, BulkMoveWithWriteBarrier, void *dst, void *src, size_t byteCount); }; -extern "C" void QCALLTYPE Buffer_MemMove(void *dst, void *src, size_t length); extern "C" void QCALLTYPE Buffer_Clear(void *dst, size_t length); +extern "C" void QCALLTYPE Buffer_MemMove(void *dst, void *src, size_t length); const UINT MEM_PRESSURE_COUNT = 4; diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h index 6758a6f007d132..0ec9409cb9676d 100644 --- a/src/coreclr/vm/ecalllist.h +++ b/src/coreclr/vm/ecalllist.h @@ -238,6 +238,7 @@ FCFuncEnd() FCFuncStart(gThreadFuncs) FCFuncElement("InternalFinalize", ThreadNative::Finalize) + FCFuncElement("CatchAtSafePoint", ThreadNative::CatchAtSafePoint) FCFuncElement("get_OptimalMaxSpinWaitsPerSpinIteration", ThreadNative::GetOptimalMaxSpinWaitsPerSpinIteration) FCFuncEnd() @@ -250,7 +251,7 @@ FCFuncStart(gArrayFuncs) FCFuncEnd() FCFuncStart(gBufferFuncs) - FCFuncElement("__BulkMoveWithWriteBarrier", Buffer::BulkMoveWithWriteBarrier) + FCFuncElement("BulkMoveWithWriteBarrierInternal", Buffer::BulkMoveWithWriteBarrier) FCFuncEnd() FCFuncStart(gGCFrameRegistration) @@ -270,7 +271,7 @@ FCFuncStart(gGCInterfaceFuncs) FCFuncElement("GetGenerationSize", GCInterface::GetGenerationSize) FCFuncElement("GetGenerationInternal", GCInterface::GetGenerationInternal) FCFuncElement("GetMaxGeneration", GCInterface::GetMaxGeneration) - FCFuncElement("_SuppressFinalize", GCInterface::SuppressFinalize) + FCFuncElement("SuppressFinalizeInternal", GCInterface::SuppressFinalize) FCFuncElement("GetAllocatedBytesForCurrentThread", GCInterface::GetAllocatedBytesForCurrentThread) FCFuncElement("GetTotalAllocatedBytesApproximate", GCInterface::GetTotalAllocatedBytesApproximate) diff --git a/src/coreclr/vm/fcall.cpp b/src/coreclr/vm/fcall.cpp index fc2e16f98a49b3..8dc242e934db0e 100644 --- a/src/coreclr/vm/fcall.cpp +++ b/src/coreclr/vm/fcall.cpp @@ -65,48 +65,6 @@ NOINLINE LPVOID __FCThrow(LPVOID __me, RuntimeExceptionKind reKind, UINT resID, return NULL; } -/**************************************************************************************/ -/* erect a frame in the FCALL and then poll the GC, objToProtect will be protected - during the poll and the updated object returned. */ - -NOINLINE Object* FC_GCPoll(void* __me, Object* objToProtect) -{ - CONTRACTL { - THROWS; - // This isn't strictly true... But the guarantee that we make here is - // that we won't trigger without having setup a frame. - UNCHECKED(GC_NOTRIGGER); - } CONTRACTL_END; - - FC_CAN_TRIGGER_GC(); - INCONTRACT(FCallCheck __fCallCheck(__FILE__, __LINE__)); - - Thread *thread = GetThread(); - if (thread->CatchAtSafePoint()) // Does someone want this thread stopped? - { - HELPER_METHOD_FRAME_BEGIN_RET_ATTRIB_1(Frame::FRAME_ATTR_CAPTURE_DEPTH_2, objToProtect); - -#ifdef _DEBUG - BOOL GCOnTransition = FALSE; - if (g_pConfig->FastGCStressLevel()) { - GCOnTransition = GC_ON_TRANSITIONS (FALSE); - } -#endif - CommonTripThread(); -#ifdef _DEBUG - if (g_pConfig->FastGCStressLevel()) { - GC_ON_TRANSITIONS (GCOnTransition); - } -#endif - - HELPER_METHOD_FRAME_END(); - } - - FC_CAN_TRIGGER_GC_END(); - - return objToProtect; -} - #ifdef ENABLE_CONTRACTS /**************************************************************************************/ @@ -177,20 +135,6 @@ DEBUG_NOINLINE FCallCheck::~FCallCheck() // // Call HELPER_METHOD_POLL() // or use HELPER_METHOD_FRAME_END_POLL - // - // If you don't have a helper frame you can used - // - // FC_GC_POLL_AND_RETURN_OBJREF or - // FC_GC_POLL or - // FC_GC_POLL_RET - // - // Note that these must be at GC safe points. In particular - // all object references that are NOT protected will be trashed. - - - // There is a special poll called FC_GC_POLL_NOT_NEEDED - // which says the code path is short enough that a GC poll is not need - // you should not use this in most cases. _ASSERTE(unbreakableLockCount == m_pThread->GetUnbreakableLockCount() || (!m_pThread->HasUnbreakableLock() && !m_pThread->HasThreadStateNC(Thread::TSNC_OwnsSpinLock))); diff --git a/src/coreclr/vm/fcall.h b/src/coreclr/vm/fcall.h index 0b362c0cc71696..58d6641304657d 100644 --- a/src/coreclr/vm/fcall.h +++ b/src/coreclr/vm/fcall.h @@ -59,9 +59,6 @@ // to do the poll at the end. If somewhere in the middle is the best // place you can do that too with HELPER_METHOD_POLL() -// You don't need to erect a helper method frame to do a poll. FC_GC_POLL -// can do this (remember all your GC refs will be trashed). - // Finally if your method is VERY small, you can get away without a poll, // you have to use FC_GC_POLL_NOT_NEEDED to mark this. // Use sparingly! @@ -801,35 +798,6 @@ LPVOID __FCThrow(LPVOID me, enum RuntimeExceptionKind reKind, UINT resID, LPCWST // don't need to poll the GC. USE VERY SPARINGLY!!! #define FC_GC_POLL_NOT_NEEDED() INCONTRACT(__fCallCheck.SetNotNeeded()) -Object* FC_GCPoll(void* me, Object* objToProtect = NULL); - -#define FC_GC_POLL_EX(ret) \ - { \ - INCONTRACT(Thread::TriggersGC(GetThread());) \ - INCONTRACT(__fCallCheck.SetDidPoll();) \ - if (g_TrapReturningThreads) \ - { \ - if (FC_GCPoll(__me)) \ - return ret; \ - while (0 == FC_NO_TAILCALL) { }; /* side effect the compile can't remove */ \ - } \ - } - -#define FC_GC_POLL() FC_GC_POLL_EX(;) -#define FC_GC_POLL_RET() FC_GC_POLL_EX(0) - -#define FC_GC_POLL_AND_RETURN_OBJREF(obj) \ - { \ - INCONTRACT(__fCallCheck.SetDidPoll();) \ - Object* __temp = OBJECTREFToObject(obj); \ - if (g_TrapReturningThreads) \ - { \ - __temp = FC_GCPoll(__me, __temp); \ - while (0 == FC_NO_TAILCALL) { }; /* side effect the compile can't remove */ \ - } \ - return __temp; \ - } - #if defined(ENABLE_CONTRACTS) #define FC_CAN_TRIGGER_GC() FCallGCCanTrigger::Enter() #define FC_CAN_TRIGGER_GC_END() FCallGCCanTrigger::Leave(__FUNCTION__, __FILE__, __LINE__) diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs index d3fcd04dcb46ff..7de71b86ea6d04 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Threading; namespace System { @@ -126,21 +127,21 @@ public static unsafe void MemoryCopy(void* source, void* destination, ulong dest // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)] - internal static unsafe void _Memmove(ref byte dest, ref byte src, nuint len) + internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) { fixed (byte* pDest = &dest) fixed (byte* pSrc = &src) - __Memmove(pDest, pSrc, len); + MemmoveInternal(pDest, pSrc, len); } // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)] - internal static unsafe void _ZeroMemory(ref byte b, nuint byteLength) + internal static unsafe void ZeroMemory(ref byte b, nuint byteLength) { fixed (byte* bytePointer = &b) { - __ZeroMemory(bytePointer, byteLength); + ZeroMemoryInternal(bytePointer, byteLength); } } @@ -168,7 +169,7 @@ ref Unsafe.As(ref source), } } - // The maximum block size to for __BulkMoveWithWriteBarrier FCall. This is required to avoid GC starvation. + // The maximum block size to for BulkMoveWithWriteBarrierInternal FCall. This is required to avoid GC starvation. #if DEBUG // Stress the mechanism in debug builds private const uint BulkMoveWithWriteBarrierChunk = 0x400; #else @@ -178,14 +179,19 @@ ref Unsafe.As(ref source), internal static void BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) { if (byteCount <= BulkMoveWithWriteBarrierChunk) - __BulkMoveWithWriteBarrier(ref destination, ref source, byteCount); + { + BulkMoveWithWriteBarrierInternal(ref destination, ref source, byteCount); + Thread.FastPollGC(); + } else - _BulkMoveWithWriteBarrier(ref destination, ref source, byteCount); + { + BulkMoveWithWriteBarrierBatch(ref destination, ref source, byteCount); + } } // Non-inlinable wrapper around the loop for copying large blocks in chunks [MethodImpl(MethodImplOptions.NoInlining)] - private static void _BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) + private static void BulkMoveWithWriteBarrierBatch(ref byte destination, ref byte source, nuint byteCount) { Debug.Assert(byteCount > BulkMoveWithWriteBarrierChunk); @@ -199,7 +205,8 @@ private static void _BulkMoveWithWriteBarrier(ref byte destination, ref byte sou do { byteCount -= BulkMoveWithWriteBarrierChunk; - __BulkMoveWithWriteBarrier(ref destination, ref source, BulkMoveWithWriteBarrierChunk); + BulkMoveWithWriteBarrierInternal(ref destination, ref source, BulkMoveWithWriteBarrierChunk); + Thread.FastPollGC(); destination = ref Unsafe.AddByteOffset(ref destination, BulkMoveWithWriteBarrierChunk); source = ref Unsafe.AddByteOffset(ref source, BulkMoveWithWriteBarrierChunk); } @@ -211,11 +218,13 @@ private static void _BulkMoveWithWriteBarrier(ref byte destination, ref byte sou do { byteCount -= BulkMoveWithWriteBarrierChunk; - __BulkMoveWithWriteBarrier(ref Unsafe.AddByteOffset(ref destination, byteCount), ref Unsafe.AddByteOffset(ref source, byteCount), BulkMoveWithWriteBarrierChunk); + BulkMoveWithWriteBarrierInternal(ref Unsafe.AddByteOffset(ref destination, byteCount), ref Unsafe.AddByteOffset(ref source, byteCount), BulkMoveWithWriteBarrierChunk); + Thread.FastPollGC(); } while (byteCount > BulkMoveWithWriteBarrierChunk); } - __BulkMoveWithWriteBarrier(ref destination, ref source, byteCount); + BulkMoveWithWriteBarrierInternal(ref destination, ref source, byteCount); + Thread.FastPollGC(); } #endif // !MONO diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index de4b6e97e427dd..319c23e0bf2152 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -239,7 +239,7 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) Debug.Assert(len > 0); _ = Unsafe.ReadUnaligned(ref dest); _ = Unsafe.ReadUnaligned(ref src); - Buffer._Memmove(ref dest, ref src, len); + Buffer.Memmove(ref dest, ref src, len); } [Intrinsic] // Unrolled for small sizes @@ -425,7 +425,7 @@ public static void ClearWithoutReferences(ref byte dest, nuint len) PInvoke: // Implicit nullchecks _ = Unsafe.ReadUnaligned(ref dest); - Buffer._ZeroMemory(ref dest, len); + Buffer.ZeroMemory(ref dest, len); } internal static void Fill(ref byte dest, byte value, nuint len) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs index 7157468c7528fc..4dc0a5002e7e78 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs @@ -374,6 +374,11 @@ public static void Sleep(int millisecondsTimeout) internal static ulong CurrentOSThreadId => GetCurrentOSThreadId(); #endif +#if !MONO + [Intrinsic] + internal static void FastPollGC() => FastPollGC(); +#endif + public ExecutionContext? ExecutionContext => ExecutionContext.Capture(); public string? Name diff --git a/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs index a74166daba159c..4139326cc1a370 100644 --- a/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs @@ -8,10 +8,10 @@ namespace System public partial class Buffer { [MethodImpl(MethodImplOptions.InternalCall)] - private static extern unsafe void __ZeroMemory(void* b, nuint byteLength); + private static extern unsafe void ZeroMemoryInternal(void* b, nuint byteLength); [MethodImpl(MethodImplOptions.InternalCall)] - private static extern unsafe void __Memmove(byte* dest, byte* src, nuint len); + private static extern unsafe void MemmoveInternal(byte* dest, byte* src, nuint len); [MethodImpl(MethodImplOptions.InternalCall)] private static extern void BulkMoveWithWriteBarrier(ref byte dmem, ref byte smem, nuint len, IntPtr type_handle); diff --git a/src/mono/mono/metadata/icall-def.h b/src/mono/mono/metadata/icall-def.h index 41a0481a7f047e..2661ebd4bc0989 100644 --- a/src/mono/mono/metadata/icall-def.h +++ b/src/mono/mono/metadata/icall-def.h @@ -140,8 +140,8 @@ HANDLES(ARRAY_14, "SetValueRelaxedImpl", ves_icall_System_Array_SetValueRelaxed ICALL_TYPE(BUFFER, "System.Buffer", BUFFER_0) NOHANDLES(ICALL(BUFFER_0, "BulkMoveWithWriteBarrier", ves_icall_System_Buffer_BulkMoveWithWriteBarrier)) -NOHANDLES(ICALL(BUFFER_2, "__Memmove", ves_icall_System_Runtime_RuntimeImports_Memmove)) -NOHANDLES(ICALL(BUFFER_3, "__ZeroMemory", ves_icall_System_Runtime_RuntimeImports_ZeroMemory)) +NOHANDLES(ICALL(BUFFER_2, "MemmoveInternal", ves_icall_System_Runtime_RuntimeImports_Memmove)) +NOHANDLES(ICALL(BUFFER_3, "ZeroMemoryInternal", ves_icall_System_Runtime_RuntimeImports_ZeroMemory)) ICALL_TYPE(DELEGATE, "System.Delegate", DELEGATE_1) HANDLES(DELEGATE_1, "AllocDelegateLike_internal", ves_icall_System_Delegate_AllocDelegateLike_internal, MonoMulticastDelegate, 1, (MonoDelegate))