From 3c0961a52402cbb64b79e362053a567c4abe6805 Mon Sep 17 00:00:00 2001 From: xtqqczze <45661989+xtqqczze@users.noreply.github.com> Date: Mon, 15 Apr 2024 07:34:05 +0100 Subject: [PATCH] Replace unaligned casts in `SpanHelpers.Memmove` (#98812) Fix https://github.com/dotnet/runtime/issues/83709 IL size is reduced by ~140 bytes. --- .../src/System/SpanHelpers.ByteMemOps.cs | 140 +++++++++--------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index b4aa563b277478..c4831969ccb0c5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -58,56 +58,56 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) // Copy bytes which are multiples of 16 and leave the remainder for MCPY01 to handle. Debug.Assert(len > 16 && len <= 64); #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref dest) = Unsafe.As(ref src); // [0,16] + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); #elif TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); // [0,16] + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); #else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); // [0,16] + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 12))); #endif if (len <= 32) goto MCPY01; #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); // [0,32] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); // [0,32] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); #else - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); // [0,32] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 20), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 20))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 28), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 28))); #endif if (len <= 48) goto MCPY01; #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); // [0,48] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); // [0,48] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); #else - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); // [0,48] + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 36), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 36))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 44), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 44))); #endif MCPY01: // Unconditionally copy the last 16 bytes using destEnd and srcEnd and return. Debug.Assert(len > 16 && len <= 64); #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); #else - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -12))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); #endif return; @@ -117,13 +117,13 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) goto MCPY03; Debug.Assert(len >= 8 && len <= 16); #if TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); #else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); #endif return; @@ -132,8 +132,8 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) if ((len & 4) == 0) goto MCPY04; Debug.Assert(len >= 4 && len < 8); - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); return; MCPY04: @@ -144,7 +144,7 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) dest = src; if ((len & 2) == 0) return; - Unsafe.As(ref Unsafe.Add(ref destEnd, -2)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -2)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -2), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -2))); return; MCPY05: @@ -163,7 +163,7 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) // dest is more important to align than src because an unaligned store is more expensive // than an unaligned load. nuint misalignedElements = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); - Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); src = ref Unsafe.Add(ref src, misalignedElements); dest = ref Unsafe.Add(ref dest, misalignedElements); len -= misalignedElements; @@ -177,33 +177,33 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) MCPY06: #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); #elif TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); - Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 48), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 48))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 56), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 56))); #else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); - Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); - Unsafe.As(ref Unsafe.Add(ref dest, 52)) = Unsafe.As(ref Unsafe.Add(ref src, 52)); - Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); - Unsafe.As(ref Unsafe.Add(ref dest, 60)) = Unsafe.As(ref Unsafe.Add(ref src, 60)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 12))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 20), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 20))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 24))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 28), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 28))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 32))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 36), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 36))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 40))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 44), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 44))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 48), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 48))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 52), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 52))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 56), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 56))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 60), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 60))); #endif dest = ref Unsafe.Add(ref dest, 64); src = ref Unsafe.Add(ref src, 64); @@ -215,15 +215,15 @@ internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) if (len > 16) goto MCPY00; #if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); #elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); #else - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -16))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -12), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -12))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -8))); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), Unsafe.ReadUnaligned(ref Unsafe.Add(ref srcEnd, -4))); #endif return;