From cd7e8718fcf5d5e55f9529c206a8c7205b45af08 Mon Sep 17 00:00:00 2001 From: Vladimir Sadov Date: Thu, 10 Nov 2022 09:41:51 -0800 Subject: [PATCH] [NativeAOT] Enable software writewatch and card bundles on Windows. (#77934) * amd64 * arm64 * fix ARM64 build * import conditionally --- src/coreclr/nativeaot/Runtime/CMakeLists.txt | 7 +- .../nativeaot/Runtime/amd64/AsmMacros.inc | 10 +- .../nativeaot/Runtime/amd64/WriteBarriers.S | 17 +- .../nativeaot/Runtime/amd64/WriteBarriers.asm | 94 ++++-- .../nativeaot/Runtime/arm64/AsmMacros.h | 17 ++ .../nativeaot/Runtime/arm64/WriteBarriers.S | 267 ++++++++---------- .../nativeaot/Runtime/arm64/WriteBarriers.asm | 260 ++++++++--------- src/coreclr/nativeaot/Runtime/gcrhenv.cpp | 4 - 8 files changed, 343 insertions(+), 333 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index d4fcf662ec66b..5e49efd8f5fa0 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -204,11 +204,8 @@ include_directories(${ARCH_SOURCES_DIR}) add_definitions(-DFEATURE_BASICFREEZE) add_definitions(-DFEATURE_CONSERVATIVE_GC) - -if(CLR_CMAKE_TARGET_UNIX) - add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP) - add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES) -endif() +add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP) +add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES) add_definitions(-DFEATURE_CUSTOM_IMPORTS) add_definitions(-DFEATURE_DYNAMIC_CODE) diff --git a/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc b/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc index d44537b2456c1..5db25fd935138 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc +++ b/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc @@ -406,10 +406,18 @@ EXTERN REDHAWKGCINTERFACE__STRESSGC : PROC EXTERN THREAD__HIJACKFORGCSTRESS : PROC endif ;; FEATURE_GC_STRESS +EXTERN RhpTrapThreads : DWORD EXTERN g_lowest_address : QWORD EXTERN g_highest_address : QWORD EXTERN g_ephemeral_low : QWORD EXTERN g_ephemeral_high : QWORD EXTERN g_card_table : QWORD -EXTERN RhpTrapThreads : DWORD + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN g_card_bundle_table : QWORD +endif + +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN g_write_watch_table : QWORD +endif diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S index c31a95c9bec5a..3888d0a8adfea 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S +++ b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S @@ -255,11 +255,10 @@ LEAF_END RhpCheckedXchg, _TEXT // On entry: // rdi: address of ref-field (assigned to) // rsi: address of the data (source) -// rcx: be trashed // // On exit: // rdi, rsi are incremented by 8, -// rcx: trashed +// rcx, r10, r11: trashed // LEAF_ENTRY RhpByRefAssignRef, _TEXT mov rcx, [rsi] @@ -267,9 +266,9 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT // Check whether the writes were even into the heap. If not there's no card update required. cmp rdi, [C_VAR(g_lowest_address)] - jb LOCAL_LABEL(RhpByRefAssignRef_NotInHeap) + jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) cmp rdi, [C_VAR(g_highest_address)] - jae LOCAL_LABEL(RhpByRefAssignRef_NotInHeap) + jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless // we're in a debug build and write barrier checking has been enabled). @@ -293,9 +292,9 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable): // If the reference is to an object that's not in an ephemeral generation we have no need to track it // (since the object won't be collected or moved by an ephemeral collection). cmp rcx, [C_VAR(g_ephemeral_low)] - jb LOCAL_LABEL(RhpByRefAssignRef_NotInHeap) + jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) cmp rcx, [C_VAR(g_ephemeral_high)] - jae LOCAL_LABEL(RhpByRefAssignRef_NotInHeap) + jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) // move current rdi value into rcx, we need to keep rdi and eventually increment by 8 mov rcx, rdi @@ -307,7 +306,7 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable): shr rcx, 0x0B mov r10, [C_VAR(g_card_table)] cmp byte ptr [rcx + r10], 0x0FF - je LOCAL_LABEL(RhpByRefAssignRef_NotInHeap) + je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) // We get here if it's necessary to update the card table. mov byte ptr [rcx + r10], 0xFF @@ -317,12 +316,12 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable): shr rcx, 0x0A add rcx, [C_VAR(g_card_bundle_table)] cmp byte ptr [rcx], 0xFF - je LOCAL_LABEL(RhpByRefAssignRef_NotInHeap) + je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) mov byte ptr [rcx], 0xFF #endif -LOCAL_LABEL(RhpByRefAssignRef_NotInHeap): +LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired): // Increment the pointers before leaving add rdi, 0x8 add rsi, 0x8 diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm index 5a6dcc666fec5..753ca45609797 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm @@ -100,6 +100,21 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG ;; we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW BASENAME, REFREG, rcx +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + mov r11, [g_write_watch_table] + cmp r11, 0 + je &BASENAME&_CheckCardTable_&REFREG& + + mov r10, rcx + shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift + add r10, r11 + cmp byte ptr [r10], 0 + jne &BASENAME&_CheckCardTable_&REFREG& + mov byte ptr [r10], 0FFh +endif + +&BASENAME&_CheckCardTable_&REFREG&: + ;; If the reference is to an object that's not in an ephemeral generation we have no need to track it ;; (since the object won't be collected or moved by an ephemeral collection). cmp REFREG, [g_ephemeral_low] @@ -111,17 +126,25 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG ;; track this write. The location address is translated into an offset in the card table bitmap. We set ;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write ;; the byte if it hasn't already been done since writes are expensive and impact scaling. - shr rcx, 11 - add rcx, [g_card_table] + shr rcx, 0Bh + mov r10, [g_card_table] + cmp byte ptr [rcx + r10], 0FFh + je &BASENAME&_NoBarrierRequired_&REFREG& + + ;; We get here if it's necessary to update the card table. + mov byte ptr [rcx + r10], 0FFh + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0x0B already) + shr rcx, 0Ah + add rcx, [g_card_bundle_table] cmp byte ptr [rcx], 0FFh - jne &BASENAME&_UpdateCardTable_&REFREG& - -&BASENAME&_NoBarrierRequired_&REFREG&: - ret + je &BASENAME&_NoBarrierRequired_&REFREG& -;; We get here if it's necessary to update the card table. -&BASENAME&_UpdateCardTable_&REFREG&: mov byte ptr [rcx], 0FFh +endif + +&BASENAME&_NoBarrierRequired_&REFREG&: ret endm @@ -248,11 +271,10 @@ LEAF_END RhpCheckedXchg, _TEXT ;; On entry: ;; rdi: address of ref-field (assigned to) ;; rsi: address of the data (source) -;; rcx: be trashed ;; ;; On exit: ;; rdi, rsi are incremented by 8, -;; rcx: trashed +;; rcx, r10, r11: trashed ;; LEAF_ENTRY RhpByRefAssignRef, _TEXT mov rcx, [rsi] @@ -260,43 +282,63 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT ;; Check whether the writes were even into the heap. If not there's no card update required. cmp rdi, [g_lowest_address] - jb RhpByRefAssignRef_NotInHeap + jb RhpByRefAssignRef_NoBarrierRequired cmp rdi, [g_highest_address] - jae RhpByRefAssignRef_NotInHeap + jae RhpByRefAssignRef_NoBarrierRequired ;; Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless ;; we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW BASENAME, rcx, rdi +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + mov r11, [g_write_watch_table] + cmp r11, 0 + je RhpByRefAssignRef_CheckCardTable + + mov r10, rdi + shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift + add r10, r11 + cmp byte ptr [r10], 0 + jne RhpByRefAssignRef_CheckCardTable + mov byte ptr [r10], 0FFh +endif + +RhpByRefAssignRef_CheckCardTable: + ;; If the reference is to an object that's not in an ephemeral generation we have no need to track it ;; (since the object won't be collected or moved by an ephemeral collection). cmp rcx, [g_ephemeral_low] - jb RhpByRefAssignRef_NotInHeap + jb RhpByRefAssignRef_NoBarrierRequired cmp rcx, [g_ephemeral_high] - jae RhpByRefAssignRef_NotInHeap + jae RhpByRefAssignRef_NoBarrierRequired - ;; move current rdi value into rcx and then increment the pointers + ;; move current rdi value into rcx, we need to keep rdi and eventually increment by 8 mov rcx, rdi - add rsi, 8h - add rdi, 8h ;; We have a location on the GC heap being updated with a reference to an ephemeral object so we must ;; track this write. The location address is translated into an offset in the card table bitmap. We set ;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write ;; the byte if it hasn't already been done since writes are expensive and impact scaling. - shr rcx, 11 - add rcx, [g_card_table] - cmp byte ptr [rcx], 0FFh - jne RhpByRefAssignRef_UpdateCardTable - ret + shr rcx, 0Bh + mov r10, [g_card_table] + cmp byte ptr [rcx + r10], 0FFh + je RhpByRefAssignRef_NoBarrierRequired ;; We get here if it's necessary to update the card table. -RhpByRefAssignRef_UpdateCardTable: + mov byte ptr [rcx + r10], 0FFh + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0Bh already) + shr rcx, 0Ah + add rcx, [g_card_bundle_table] + cmp byte ptr [rcx], 0FFh + je RhpByRefAssignRef_NoBarrierRequired + mov byte ptr [rcx], 0FFh - ret +endif -RhpByRefAssignRef_NotInHeap: - ; Increment the pointers before leaving +RhpByRefAssignRef_NoBarrierRequired: + ;; Increment the pointers before leaving add rdi, 8h add rsi, 8h ret diff --git a/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h b/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h index d6e3074fbfcc2..de4aef3d26777 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h +++ b/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h @@ -108,6 +108,14 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit equ OFFSETOF__Thread__m_rgbA EXTERN g_ephemeral_high EXTERN g_card_table +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + EXTERN g_card_bundle_table +#endif + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + EXTERN g_write_watch_table +#endif + ;; ----------------------------------------------------------------------------- ;; Macro used to assign an alternate name to a symbol containing characters normally disallowed in a symbol @@ -146,6 +154,15 @@ MovInstr SETS "movk" $MovInstr $Reg, #(($ConstantLo):AND:0xffff) MEND +;;----------------------------------------------------------------------------- +;; Macro for loading a 64bit value of a global variable into a register + MACRO + PREPARE_EXTERNAL_VAR_INDIRECT $Name, $Reg + + adrp $Reg, $Name + ldr $Reg, [$Reg, $Name] + MEND + ;; ----------------------------------------------------------------------------- ;; ;; Macro to export a pointer to an address inside a stub as a 64-bit variable diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S index 8d908d993aebf..8fbbce7fd8186 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S +++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S @@ -22,29 +22,29 @@ // $refReg: objectref to be stored // // On exit: - // x9,x10: trashed + // x12,x17: trashed // other registers are preserved // .macro UPDATE_GC_SHADOW destReg, refReg // If g_GCShadow is 0, don't perform the check. - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9 - cbz x9, 1f + PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X12 + cbz x12, 1f // Save destReg since we're about to modify it (and we need the original value both within the macro and // once we exit the macro). - mov x10, \destReg + mov x17, \destReg // Transform destReg into the equivalent address in the shadow heap. - PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, X9 - subs \destReg, \destReg, x9 + PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, X12 + subs \destReg, \destReg, x12 blo 0f - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9 - add \destReg, \destReg, x9 + PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X12 + add \destReg, \destReg, x12 - PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, X9 - cmp \destReg, x9 + PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, X12 + cmp \destReg, x12 bhs 0f // Update the shadow heap. @@ -55,20 +55,20 @@ dmb ish // Now check that the real heap location still contains the value we just wrote into the shadow heap. - mov x9, x10 - ldr x9, [x9] - cmp x9, \refReg + mov x12, x17 + ldr x12, [x12] + cmp x12, \refReg beq 0f // Someone went and updated the real heap. We need to invalidate INVALIDGCVALUE the shadow location since we can not // guarantee whose shadow update won. - movz x9, (INVALIDGCVALUE & 0xFFFF) // #0xcccd - movk x9, ((INVALIDGCVALUE >> 16) & 0xFFFF), LSL #16 - str x9, [\destReg] + movz x12, (INVALIDGCVALUE & 0xFFFF) // #0xcccd + movk x12, ((INVALIDGCVALUE >> 16) & 0xFFFF), LSL #16 + str x12, [\destReg] 0: // Restore original destReg value - mov \destReg, x10 + mov \destReg, x17 1: .endm @@ -89,14 +89,13 @@ // some interlocked helpers that need an inline barrier. // On entry: - // destReg: location to be updated - // refReg: objectref to be stored - // trash: register nr than can be trashed + // destReg: location to be updated (cannot be x12,x17) + // refReg: objectref to be stored (cannot be x12,x17) // // On exit: - // destReg: trashed + // x12,x17: trashed // - .macro INSERT_UNCHECKED_WRITE_BARRIER_CORE destReg, refReg, trash + .macro INSERT_UNCHECKED_WRITE_BARRIER_CORE destReg, refReg // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless // we are in a debug build and write barrier checking has been enabled). @@ -104,51 +103,50 @@ #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Update the write watch table if necessary - PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x\trash - - cbz x\trash, 2f - add x\trash, x\trash, \destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift - ldrb w17, [x\trash] - cbnz x17, 2f - mov w17, #0xFF - strb w17, [x\trash] + PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12 + + cbz x12, 2f + add x12, x12, \destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift + ldrb w17, [x12] + cbnz x17, 2f + mov w17, #0xFF + strb w17, [x12] #endif 2: // We can skip the card table write if the reference is to // an object not on the epehemeral segment. - PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x\trash - cmp \refReg, x\trash + PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x12 + cmp \refReg, x12 blo 0f - PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x\trash - cmp \refReg, x\trash + PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x12 + cmp \refReg, x12 bhs 0f // Set this objects card, if it has not already been set. - - PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x\trash - add x17, x\trash, \destReg, lsr #11 + PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12 + add x17, x12, \destReg, lsr #11 // Check that this card has not already been written. Avoiding useless writes is a big win on // multi-proc systems since it avoids cache thrashing. - ldrb w\trash, [x17] - cmp x\trash, 0xFF + ldrb w12, [x17] + cmp x12, 0xFF beq 0f - mov x\trash, 0xFF - strb w\trash, [x17] + mov x12, 0xFF + strb w12, [x17] #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Check if we need to update the card bundle table - PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x\trash - add x17, x\trash, \destReg, lsr #21 - ldrb w\trash, [x17] - cmp x\trash, 0xFF - beq 0f - - mov x\trash, 0xFF - strb w\trash, [x17] + PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12 + add x17, x12, \destReg, lsr #21 + ldrb w12, [x17] + cmp x12, 0xFF + beq 0f + + mov x12, 0xFF + strb w12, [x17] #endif 0: @@ -158,73 +156,111 @@ // On entry: // destReg: location to be updated // refReg: objectref to be stored - // trash: register nr than can be trashed // // On exit: - // destReg: trashed + // x12, x17: trashed // - .macro INSERT_CHECKED_WRITE_BARRIER_CORE destReg, refReg, trash + .macro INSERT_CHECKED_WRITE_BARRIER_CORE destReg, refReg // The "check" of this checked write barrier - is destReg // within the heap? if no, early out. - PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x\trash - cmp \destReg, x\trash + PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12 + cmp \destReg, x12 - PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x\trash + PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12 // If \destReg >= g_lowest_address, compare \destReg to g_highest_address. // Otherwise, set the C flag (0x2) to take the next branch. - ccmp \destReg, x\trash, #0x2, hs + ccmp \destReg, x12, #0x2, hs bhs 0f - INSERT_UNCHECKED_WRITE_BARRIER_CORE \destReg, \refReg, \trash + INSERT_UNCHECKED_WRITE_BARRIER_CORE \destReg, \refReg 0: // Exit label .endm -// RhpCheckedAssignRef(Object** dst, Object* src) +// void JIT_ByRefWriteBarrier +// On entry: +// x13 : the source address (points to object reference to write) +// x14 : the destination address (object reference written here) +// +// On exit: +// x13 : incremented by 8 +// x14 : incremented by 8 +// x15 : trashed +// x12, x17 : trashed +// +LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT + + ldr x15, [x13], 8 + b C_FUNC(RhpCheckedAssignRefArm64) + +LEAF_END RhpByRefAssignRefArm64, _TEXT + +// JIT_CheckedWriteBarrier(Object** dst, Object* src) // // Write barrier for writes to objects that may reside // on the managed heap. // // On entry: -// x0 : the destination address (LHS of the assignment). -// May not be an object reference (hence the checked). -// x1 : the object reference (RHS of the assignment). +// x14 : the destination address (LHS of the assignment). +// May not be a heap location (hence the checked). +// x15 : the object reference (RHS of the assignment). +// // On exit: -// x1 : trashed -// x9 : trashed - LEAF_ENTRY RhpCheckedAssignRef, _TEXT - ALTERNATE_ENTRY RhpCheckedAssignRefX1 +// x12, x17 : trashed +// x14 : incremented by 8 + LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT - mov x14, x0 ; x14 = dst - mov x15, x1 ; x15 = val - b C_FUNC(RhpCheckedAssignRefArm64) + // is destReg within the heap? + PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12 + cmp x14, x12 -LEAF_END RhpCheckedAssignRef, _TEXT + PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12 + ccmp x14, x12, #0x2, hs + blo C_FUNC(RhpAssignRefArm64) + +NotInHeap: + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation + str x15, [x14], 8 + ret -// RhpAssignRef(Object** dst, Object* src) +LEAF_END RhpCheckedAssignRefArm64, _TEXT + +// JIT_WriteBarrier(Object** dst, Object* src) // // Write barrier for writes to objects that are known to // reside on the managed heap. // // On entry: -// x0 : the destination address (LHS of the assignment). -// x1 : the object reference (RHS of the assignment). +// x14 : the destination address (LHS of the assignment). +// x15 : the object reference (RHS of the assignment). +// // On exit: -// x1 : trashed -// x9 : trashed -LEAF_ENTRY RhpAssignRef, _TEXT - ALTERNATE_ENTRY RhpAssignRefX1 +// x12, x17 : trashed +// x14 : incremented by 8 +LEAF_ENTRY RhpAssignRefArm64, _TEXT + ALTERNATE_ENTRY RhpAssignRefAVLocation + stlr x15, [x14] + + INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 + + add x14, x14, 8 + ret + +LEAF_END RhpAssignRefArm64, _TEXT + +// Same as RhpAssignRefArm64, but with standard ABI. +LEAF_ENTRY RhpAssignRef, _TEXT mov x14, x0 ; x14 = dst mov x15, x1 ; x15 = val b C_FUNC(RhpAssignRefArm64) - LEAF_END RhpAssignRef, _TEXT + // Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon // successful updates. @@ -243,27 +279,26 @@ LEAF_END RhpAssignRef, _TEXT // // On exit: // x0: original value of objectref -// x9: trashed -// x10: trashed +// x10, x12, x17: trashed // LEAF_ENTRY RhpCheckedLockCmpXchg - ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation CmpXchgRetry: // Check location value is what we expect. + ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation ldaxr x10, [x0] cmp x10, x2 bne CmpXchgNoUpdate // Current value matches comparand, attempt to update with the new value. - stlxr w9, x1, [x0] - cbnz w9, CmpXchgRetry + stlxr w12, x1, [x0] + cbnz w12, CmpXchgRetry // We have successfully updated the value of the objectref so now we need a GC write barrier. // The following barrier code takes the destination in x0 and the value in x1 so the arguments are // already correctly set up. - INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9 + INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1 CmpXchgNoUpdate: // x10 still contains the original value. @@ -287,25 +322,25 @@ CmpXchgNoUpdate: // // On exit: // x0: original value of objectref -// x9: trashed // x10: trashed +// x12, x17: trashed // LEAF_ENTRY RhpCheckedXchg, _TEXT - ALTERNATE_ENTRY RhpCheckedXchgAVLocation ExchangeRetry: // Read the existing memory location. + ALTERNATE_ENTRY RhpCheckedXchgAVLocation ldaxr x10, [x0] // Attempt to update with the new value. - stlxr w9, x1, [x0] - cbnz w9, ExchangeRetry + stlxr w12, x1, [x0] + cbnz w12, ExchangeRetry // We have successfully updated the value of the objectref so now we need a GC write barrier. // The following barrier code takes the destination in x0 and the value in x1 so the arguments are // already correctly set up. - INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9 + INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1 // x10 still contains the original value. mov x0, x10 @@ -313,59 +348,3 @@ ExchangeRetry: ret LEAF_END RhpCheckedXchg, _TEXT - -LEAF_ENTRY RhpAssignRefArm64, _TEXT - ALTERNATE_ENTRY RhpAssignRefAVLocation - ALTERNATE_ENTRY RhpAssignRefX1AVLocation - stlr x15, [x14] - - INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15, 12 - - ret -LEAF_END RhpAssignRefArm64, _TEXT - -// void JIT_CheckedWriteBarrier(Object** dst, Object* src) -// On entry: -// x14 : the destination address (LHS of the assignment) -// x15 : the object reference (RHS of the assignment) -// -// On exit: -// x12 : trashed -// x14 : trashed (incremented by 8 to implement JIT_ByRefWriteBarrier contract) -// x15 : trashed -// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -// -LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation - ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation - - stlr x15, [x14] - - INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, 12 - - ret -LEAF_END RhpCheckedAssignRefArm64, _TEXT - -// void JIT_ByRefWriteBarrier -// On entry: -// x13 : the source address (points to object reference to write) -// x14 : the destination address (object reference written here) -// -// On exit: -// x12 : trashed -// x13 : incremented by 8 -// x14 : incremented by 8 -// x15 : trashed -// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -// -LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT - ldr x15, [x13] - stlr x15, [x14] - - INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, 12 - - add X13, x13, #8 - add x14, x14, #8 - - ret -LEAF_END RhpByRefAssignRefArm64, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm index 005e80a12cd01..62639a7a6de42 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm @@ -42,8 +42,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD UPDATE_GC_SHADOW $destReg, $refReg ;; If g_GCShadow is 0, don't perform the check. - adrp x12, $g_GCShadow - ldr x12, [x12, $g_GCShadow] + PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadow, x12 cbz x12, %ft1 ;; Save $destReg since we're about to modify it (and we need the original value both within the macro and @@ -51,17 +50,14 @@ INVALIDGCVALUE EQU 0xCCCCCCCD mov x17, $destReg ;; Transform $destReg into the equivalent address in the shadow heap. - adrp x12, g_lowest_address - ldr x12, [x12, g_lowest_address] + PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12 subs $destReg, $destReg, x12 blo %ft0 - adrp x12, $g_GCShadow - ldr x12, [x12, $g_GCShadow] + PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadow, x12 add $destReg, $destReg, x12 - adrp x12, $g_GCShadowEnd - ldr x12, [x12, $g_GCShadowEnd] + PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, x12 cmp $destReg, x12 bhs %ft0 @@ -109,44 +105,63 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; On entry: ;; $destReg: location to be updated (cannot be x12,x17) ;; $refReg: objectref to be stored (cannot be x12,x17) - ;; $trashReg: register that can be trashed (can be $destReg or $refReg) ;; ;; On exit: - ;; $trashReg: trashed - ;; x12: trashed - ;; x17: trashed if WRITE_BARRIER_CHECK + ;; x12,x17: trashed ;; - INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg + INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg ;; Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless ;; we're in a debug build and write barrier checking has been enabled). UPDATE_GC_SHADOW $destReg, $refReg +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + // Update the write watch table if necessary + PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12 + + cbz x12, %ft2 + add x12, x12, $destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift + ldrb w17, [x12] + cbnz x17, %ft2 + mov w17, #0xFF + strb w17, [x12] +#endif + +2 ;; We can skip the card table write if the reference is to ;; an object not on the epehemeral segment. - adrp x12, g_ephemeral_low - ldr x12, [x12, g_ephemeral_low] + PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x12 cmp $refReg, x12 blo %ft0 - adrp x12, g_ephemeral_high - ldr x12, [x12, g_ephemeral_high] + PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x12 cmp $refReg, x12 bhs %ft0 ;; Set this object's card, if it hasn't already been set. - adrp x12, g_card_table - ldr x12, [x12, g_card_table] - add $trashReg, x12, $destReg lsr #11 + PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12 + add x17, x12, $destReg lsr #11 ;; Check that this card hasn't already been written. Avoiding useless writes is a big win on ;; multi-proc systems since it avoids cache trashing. - ldrb w12, [$trashReg] + ldrb w12, [x17] + cmp x12, 0xFF + beq %ft0 + + mov x12, 0xFF + strb w12, [x17] + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + // Check if we need to update the card bundle table + PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12 + add x17, x12, $destReg, lsr #21 + ldrb w12, [x17] cmp x12, 0xFF beq %ft0 mov x12, 0xFF - strb w12, [$trashReg] + strb w12, [x17] +#endif 0 ;; Exit label @@ -156,82 +171,111 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; On entry: ;; $destReg: location to be updated (cannot be x12,x17) ;; $refReg: objectref to be stored (cannot be x12,x17) - ;; $trashReg: register that can be trashed (can be $destReg or $refReg) ;; ;; On exit: - ;; $trashReg: trashed - ;; x12: trashed - ;; x17: trashed if WRITE_BARRIER_CHECK + ;; x12, x17: trashed ;; - INSERT_CHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg + INSERT_CHECKED_WRITE_BARRIER_CORE $destReg, $refReg ;; The "check" of this checked write barrier - is $destReg ;; within the heap? if no, early out. - adrp x12, g_lowest_address - ldr x12, [x12, g_lowest_address] + PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12 cmp $destReg, x12 - adrp x12, g_highest_address - ldr x12, [x12, g_highest_address] + PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12 ;; If $destReg >= g_lowest_address, compare $destReg to g_highest_address. ;; Otherwise, set the C flag (0x2) to take the next branch. ccmp $destReg, x12, #0x2, hs bhs %ft0 - INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg + INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg 0 ;; Exit label MEND -;; RhpCheckedAssignRef(Object** dst, Object* src) +;; void JIT_ByRefWriteBarrier +;; On entry: +;; x13 : the source address (points to object reference to write) +;; x14 : the destination address (object reference written here) +;; +;; On exit: +;; x13 : incremented by 8 +;; x14 : incremented by 8 +;; x15 : trashed +;; x12, x17 : trashed +;; + LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT + + ldr x15, [x13], 8 + b RhpCheckedAssignRefArm64 + + LEAF_END RhpByRefAssignRefArm64 + + +;; JIT_CheckedWriteBarrier(Object** dst, Object* src) ;; ;; Write barrier for writes to objects that may reside ;; on the managed heap. ;; ;; On entry: -;; x0 : the destination address (LHS of the assignment). -;; May not be an object reference (hence the checked). -;; x1 : the object reference (RHS of the assignment) +;; x14 : the destination address (LHS of the assignment). +;; May not be a heap location (hence the checked). +;; x15 : the object reference (RHS of the assignment) ;; ;; On exit: -;; x12 : trashed -;; x14 : trashed -;; x15 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK - LEAF_ENTRY RhpCheckedAssignRef - ALTERNATE_ENTRY RhpCheckedAssignRefX1 - - mov x14, x0 ; x14 = dst - mov x15, x1 ; x15 = val - b RhpCheckedAssignRefArm64 +;; x12, x17 : trashed +;; x14 : incremented by 8 + LEAF_ENTRY RhpCheckedAssignRefArm64 + + ;; is destReg within the heap? + PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12 + cmp x14, x12 + + PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12 + ccmp x14, x12, #0x2, hs + blo RhpAssignRefArm64 + +NotInHeap + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation + str x15, [x14], 8 + ret - LEAF_END RhpCheckedAssignRef + LEAF_END RhpCheckedAssignRefArm64 -;; RhpAssignRef(Object** dst, Object* src) +;; JIT_WriteBarrier(Object** dst, Object* src) ;; ;; Write barrier for writes to objects that are known to ;; reside on the managed heap. ;; ;; On entry: -;; x0 : the destination address (LHS of the assignment) -;; x1 : the object reference (RHS of the assignment) +;; x14 : the destination address (LHS of the assignment) +;; x15 : the object reference (RHS of the assignment) ;; ;; On exit: -;; x12 : trashed -;; x14 : trashed -;; x15 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK - LEAF_ENTRY RhpAssignRef - ALTERNATE_ENTRY RhpAssignRefX1 +;; x12, x17 : trashed +;; x14 : incremented by 8 + LEAF_ENTRY RhpAssignRefArm64 - mov x14, x0 ; x14 = dst - mov x15, x1 ; x15 = val - b RhpAssignRefArm64 + ALTERNATE_ENTRY RhpAssignRefAVLocation + stlr x15, [x14] + + INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15 + add x14, x14, 8 + ret + + LEAF_END RhpAssignRefArm64 + +;; same as RhpAssignRefArm64, but with standard ABI. + LEAF_ENTRY RhpAssignRef + mov x14, x0 ; x14 = dst + mov x15, x1 ; x15 = val + b RhpAssignRefArm64 LEAF_END RhpAssignRef + ;; Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon ;; successful updates. @@ -249,30 +293,27 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; x2 : comparand ;; ;; On exit: -;; x0 : original value of objectref -;; x9 : trashed -;; x10 : trashed -;; x12 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK +;; x0: original value of objectref +;; x10, x12, x17: trashed ;; LEAF_ENTRY RhpCheckedLockCmpXchg - ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation CmpXchgRetry ;; Check location value is what we expect. + ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation ldaxr x10, [x0] cmp x10, x2 bne CmpXchgNoUpdate ;; Current value matches comparand, attempt to update with the new value. - stlxr w9, x1, [x0] - cbnz w9, CmpXchgRetry + stlxr w12, x1, [x0] + cbnz w12, CmpXchgRetry ;; We've successfully updated the value of the objectref so now we need a GC write barrier. ;; The following barrier code takes the destination in x0 and the value in x1 so the arguments are ;; already correctly set up. - INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, x0 + INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1 CmpXchgNoUpdate ;; x10 still contains the original value. @@ -295,28 +336,26 @@ CmpXchgNoUpdate ;; x1 : exchange value ;; ;; On exit: -;; x0 : original value of objectref -;; x9 : trashed -;; x10 : trashed -;; x12 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK +;; x0: original value of objectref +;; x10: trashed +;; x12, x17: trashed ;; LEAF_ENTRY RhpCheckedXchg - ALTERNATE_ENTRY RhpCheckedXchgAVLocation ExchangeRetry ;; Read the existing memory location. + ALTERNATE_ENTRY RhpCheckedXchgAVLocation ldaxr x10, [x0] ;; Attempt to update with the new value. - stlxr w9, x1, [x0] - cbnz w9, ExchangeRetry + stlxr w12, x1, [x0] + cbnz w12, ExchangeRetry ;; We've successfully updated the value of the objectref so now we need a GC write barrier. ;; The following barrier code takes the destination in x0 and the value in x1 so the arguments are ;; already correctly set up. - INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, x0 + INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1 ;; x10 still contains the original value. mov x0, x10 @@ -325,71 +364,4 @@ ExchangeRetry LEAF_END RhpCheckedXchg -;; RhpAssignRefArm64(Object** dst, Object* src) -;; -;; On entry: -;; x14 : the destination address (LHS of the assignment). -;; x15 : the object reference (RHS of the assignment). -;; -;; On exit: -;; x12 : trashed -;; x15 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK -;; - LEAF_ENTRY RhpAssignRefArm64, _TEXT - ALTERNATE_ENTRY RhpAssignRefAVLocation - ALTERNATE_ENTRY RhpAssignRefX1AVLocation - - stlr x15, [x14] - - INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15, x15 - - ret - LEAF_END RhpAssignRefArm64 - -;; void JIT_CheckedWriteBarrier(Object** dst, Object* src) -;; On entry: -;; x14 : the destination address (LHS of the assignment) -;; x15 : the object reference (RHS of the assignment) -;; -;; On exit: -;; x12 : trashed -;; x15 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK -;; - LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT - ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation - ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation - - stlr x15, [x14] - - INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, x15 - - ret - LEAF_END RhpCheckedAssignRefArm64 - -;; void JIT_ByRefWriteBarrier -;; On entry: -;; x13 : the source address (points to object reference to write) -;; x14 : the destination address (object reference written here) -;; -;; On exit: -;; x12 : trashed -;; x13 : incremented by 8 -;; x14 : incremented by 8 -;; x15 : trashed -;; x17 : trashed if WRITE_BARRIER_CHECK -;; - LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT - ldr x15, [x13] - stlr x15, [x14] - - INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, x15 - - add x13, x13, #8 - add x14, x14, #8 - - ret - LEAF_END RhpByRefAssignRefArm64 - end diff --git a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp index ead3936f3bdf0..6d8b819b3a096 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp +++ b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp @@ -1029,10 +1029,6 @@ void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext) #endif // FEATURE_EVENT_TRACE } -#if defined(FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP) && !defined(TARGET_UNIX) -#error FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP is only implemented for UNIX -#endif - void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) { // NativeAOT doesn't patch the write barrier like CoreCLR does, but it