Skip to content

Commit

Permalink
[NativeAOT] Enable software writewatch and card bundles on Windows. (d…
Browse files Browse the repository at this point in the history
…otnet#77934)

* amd64

* arm64

* fix ARM64 build

* import conditionally
  • Loading branch information
VSadov authored Nov 10, 2022
1 parent 6a67628 commit cd7e871
Show file tree
Hide file tree
Showing 8 changed files with 343 additions and 333 deletions.
7 changes: 2 additions & 5 deletions src/coreclr/nativeaot/Runtime/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -204,11 +204,8 @@ include_directories(${ARCH_SOURCES_DIR})

add_definitions(-DFEATURE_BASICFREEZE)
add_definitions(-DFEATURE_CONSERVATIVE_GC)

if(CLR_CMAKE_TARGET_UNIX)
add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES)
endif()
add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES)

add_definitions(-DFEATURE_CUSTOM_IMPORTS)
add_definitions(-DFEATURE_DYNAMIC_CODE)
Expand Down
10 changes: 9 additions & 1 deletion src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
Original file line number Diff line number Diff line change
Expand Up @@ -406,10 +406,18 @@ EXTERN REDHAWKGCINTERFACE__STRESSGC : PROC
EXTERN THREAD__HIJACKFORGCSTRESS : PROC
endif ;; FEATURE_GC_STRESS

EXTERN RhpTrapThreads : DWORD
EXTERN g_lowest_address : QWORD
EXTERN g_highest_address : QWORD
EXTERN g_ephemeral_low : QWORD
EXTERN g_ephemeral_high : QWORD
EXTERN g_card_table : QWORD
EXTERN RhpTrapThreads : DWORD

ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
EXTERN g_card_bundle_table : QWORD
endif

ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
EXTERN g_write_watch_table : QWORD
endif

17 changes: 8 additions & 9 deletions src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S
Original file line number Diff line number Diff line change
Expand Up @@ -255,21 +255,20 @@ LEAF_END RhpCheckedXchg, _TEXT
// On entry:
// rdi: address of ref-field (assigned to)
// rsi: address of the data (source)
// rcx: be trashed
//
// On exit:
// rdi, rsi are incremented by 8,
// rcx: trashed
// rcx, r10, r11: trashed
//
LEAF_ENTRY RhpByRefAssignRef, _TEXT
mov rcx, [rsi]
mov [rdi], rcx

// Check whether the writes were even into the heap. If not there's no card update required.
cmp rdi, [C_VAR(g_lowest_address)]
jb LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
cmp rdi, [C_VAR(g_highest_address)]
jae LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)

// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we're in a debug build and write barrier checking has been enabled).
Expand All @@ -293,9 +292,9 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
// If the reference is to an object that's not in an ephemeral generation we have no need to track it
// (since the object won't be collected or moved by an ephemeral collection).
cmp rcx, [C_VAR(g_ephemeral_low)]
jb LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
cmp rcx, [C_VAR(g_ephemeral_high)]
jae LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)

// move current rdi value into rcx, we need to keep rdi and eventually increment by 8
mov rcx, rdi
Expand All @@ -307,7 +306,7 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
shr rcx, 0x0B
mov r10, [C_VAR(g_card_table)]
cmp byte ptr [rcx + r10], 0x0FF
je LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)

// We get here if it's necessary to update the card table.
mov byte ptr [rcx + r10], 0xFF
Expand All @@ -317,12 +316,12 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
shr rcx, 0x0A
add rcx, [C_VAR(g_card_bundle_table)]
cmp byte ptr [rcx], 0xFF
je LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)

mov byte ptr [rcx], 0xFF
#endif

LOCAL_LABEL(RhpByRefAssignRef_NotInHeap):
LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired):
// Increment the pointers before leaving
add rdi, 0x8
add rsi, 0x8
Expand Down
94 changes: 68 additions & 26 deletions src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,21 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG
;; we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW BASENAME, REFREG, rcx

ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
mov r11, [g_write_watch_table]
cmp r11, 0
je &BASENAME&_CheckCardTable_&REFREG&

mov r10, rcx
shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift
add r10, r11
cmp byte ptr [r10], 0
jne &BASENAME&_CheckCardTable_&REFREG&
mov byte ptr [r10], 0FFh
endif

&BASENAME&_CheckCardTable_&REFREG&:

;; If the reference is to an object that's not in an ephemeral generation we have no need to track it
;; (since the object won't be collected or moved by an ephemeral collection).
cmp REFREG, [g_ephemeral_low]
Expand All @@ -111,17 +126,25 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG
;; track this write. The location address is translated into an offset in the card table bitmap. We set
;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
;; the byte if it hasn't already been done since writes are expensive and impact scaling.
shr rcx, 11
add rcx, [g_card_table]
shr rcx, 0Bh
mov r10, [g_card_table]
cmp byte ptr [rcx + r10], 0FFh
je &BASENAME&_NoBarrierRequired_&REFREG&

;; We get here if it's necessary to update the card table.
mov byte ptr [rcx + r10], 0FFh

ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0x0B already)
shr rcx, 0Ah
add rcx, [g_card_bundle_table]
cmp byte ptr [rcx], 0FFh
jne &BASENAME&_UpdateCardTable_&REFREG&

&BASENAME&_NoBarrierRequired_&REFREG&:
ret
je &BASENAME&_NoBarrierRequired_&REFREG&

;; We get here if it's necessary to update the card table.
&BASENAME&_UpdateCardTable_&REFREG&:
mov byte ptr [rcx], 0FFh
endif

&BASENAME&_NoBarrierRequired_&REFREG&:
ret

endm
Expand Down Expand Up @@ -248,55 +271,74 @@ LEAF_END RhpCheckedXchg, _TEXT
;; On entry:
;; rdi: address of ref-field (assigned to)
;; rsi: address of the data (source)
;; rcx: be trashed
;;
;; On exit:
;; rdi, rsi are incremented by 8,
;; rcx: trashed
;; rcx, r10, r11: trashed
;;
LEAF_ENTRY RhpByRefAssignRef, _TEXT
mov rcx, [rsi]
mov [rdi], rcx

;; Check whether the writes were even into the heap. If not there's no card update required.
cmp rdi, [g_lowest_address]
jb RhpByRefAssignRef_NotInHeap
jb RhpByRefAssignRef_NoBarrierRequired
cmp rdi, [g_highest_address]
jae RhpByRefAssignRef_NotInHeap
jae RhpByRefAssignRef_NoBarrierRequired

;; Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
;; we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW BASENAME, rcx, rdi

ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
mov r11, [g_write_watch_table]
cmp r11, 0
je RhpByRefAssignRef_CheckCardTable

mov r10, rdi
shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift
add r10, r11
cmp byte ptr [r10], 0
jne RhpByRefAssignRef_CheckCardTable
mov byte ptr [r10], 0FFh
endif

RhpByRefAssignRef_CheckCardTable:

;; If the reference is to an object that's not in an ephemeral generation we have no need to track it
;; (since the object won't be collected or moved by an ephemeral collection).
cmp rcx, [g_ephemeral_low]
jb RhpByRefAssignRef_NotInHeap
jb RhpByRefAssignRef_NoBarrierRequired
cmp rcx, [g_ephemeral_high]
jae RhpByRefAssignRef_NotInHeap
jae RhpByRefAssignRef_NoBarrierRequired

;; move current rdi value into rcx and then increment the pointers
;; move current rdi value into rcx, we need to keep rdi and eventually increment by 8
mov rcx, rdi
add rsi, 8h
add rdi, 8h

;; We have a location on the GC heap being updated with a reference to an ephemeral object so we must
;; track this write. The location address is translated into an offset in the card table bitmap. We set
;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
;; the byte if it hasn't already been done since writes are expensive and impact scaling.
shr rcx, 11
add rcx, [g_card_table]
cmp byte ptr [rcx], 0FFh
jne RhpByRefAssignRef_UpdateCardTable
ret
shr rcx, 0Bh
mov r10, [g_card_table]
cmp byte ptr [rcx + r10], 0FFh
je RhpByRefAssignRef_NoBarrierRequired

;; We get here if it's necessary to update the card table.
RhpByRefAssignRef_UpdateCardTable:
mov byte ptr [rcx + r10], 0FFh

ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0Bh already)
shr rcx, 0Ah
add rcx, [g_card_bundle_table]
cmp byte ptr [rcx], 0FFh
je RhpByRefAssignRef_NoBarrierRequired

mov byte ptr [rcx], 0FFh
ret
endif

RhpByRefAssignRef_NotInHeap:
; Increment the pointers before leaving
RhpByRefAssignRef_NoBarrierRequired:
;; Increment the pointers before leaving
add rdi, 8h
add rsi, 8h
ret
Expand Down
17 changes: 17 additions & 0 deletions src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,14 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit equ OFFSETOF__Thread__m_rgbA
EXTERN g_ephemeral_high
EXTERN g_card_table

#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
EXTERN g_card_bundle_table
#endif

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
EXTERN g_write_watch_table
#endif


;; -----------------------------------------------------------------------------
;; Macro used to assign an alternate name to a symbol containing characters normally disallowed in a symbol
Expand Down Expand Up @@ -146,6 +154,15 @@ MovInstr SETS "movk"
$MovInstr $Reg, #(($ConstantLo):AND:0xffff)
MEND

;;-----------------------------------------------------------------------------
;; Macro for loading a 64bit value of a global variable into a register
MACRO
PREPARE_EXTERNAL_VAR_INDIRECT $Name, $Reg

adrp $Reg, $Name
ldr $Reg, [$Reg, $Name]
MEND

;; -----------------------------------------------------------------------------
;;
;; Macro to export a pointer to an address inside a stub as a 64-bit variable
Expand Down
Loading

0 comments on commit cd7e871

Please sign in to comment.