diff --git a/docs/coding-guidelines/vectorization-guidelines.md b/docs/coding-guidelines/vectorization-guidelines.md index ab85676263afd6..9ac773872947a9 100644 --- a/docs/coding-guidelines/vectorization-guidelines.md +++ b/docs/coding-guidelines/vectorization-guidelines.md @@ -523,7 +523,7 @@ int ManagedReferencesSum(int[] buffer) Vector128 sum = Vector128.Zero; - while (!Unsafe.IsAddressGreaterThan(ref current, ref oneVectorAwayFromEnd)) + while (Unsafe.IsAddressLessThanOrEqualTo(ref current, ref oneVectorAwayFromEnd)) { sum += Vector128.LoadUnsafe(ref current); @@ -561,7 +561,7 @@ do return ...; } -while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); +while (Unsafe.IsAddressGreaterThanOrEqualTo(ref currentSearchSpace, ref searchSpace)); ``` It was part of `LastIndexOf` implementation, where we were iterating from the end to the beginning of the buffer. In the last iteration of the loop, `currentSearchSpace` could become a pointer to unknown memory that lied before the beginning of the buffer: @@ -573,7 +573,7 @@ currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // clt.un + // ldc.i4.0 + // ceq + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_GE, TYP_INT, op1, op2); + tmp->gtFlags |= GTF_UNSIGNED; + return gtFoldExpr(tmp); + } + case NI_SRCS_UNSAFE_IsAddressLessThan: { assert(sig->sigInst.methInstCount == 1); @@ -5517,6 +5536,25 @@ GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic, return gtFoldExpr(tmp); } + case NI_SRCS_UNSAFE_IsAddressLessThanOrEqualTo: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // cgt.un + // ldc.i4.0 + // ceq + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_LE, TYP_INT, op1, op2); + tmp->gtFlags |= GTF_UNSIGNED; + return gtFoldExpr(tmp); + } + case NI_SRCS_UNSAFE_IsNullRef: { assert(sig->sigInst.methInstCount == 1); @@ -10720,10 +10758,18 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_SRCS_UNSAFE_IsAddressGreaterThan; } + else if (strcmp(methodName, "IsAddressGreaterThanOrEqualTo") == 0) + { + result = NI_SRCS_UNSAFE_IsAddressGreaterThanOrEqualTo; + } else if (strcmp(methodName, "IsAddressLessThan") == 0) { result = NI_SRCS_UNSAFE_IsAddressLessThan; } + else if (strcmp(methodName, "IsAddressLessThanOrEqualTo") == 0) + { + result = NI_SRCS_UNSAFE_IsAddressLessThanOrEqualTo; + } else if (strcmp(methodName, "IsNullRef") == 0) { result = NI_SRCS_UNSAFE_IsNullRef; diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 71a7124e0974ac..aa56003f25acc1 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -221,7 +221,9 @@ enum NamedIntrinsic : unsigned short NI_SRCS_UNSAFE_InitBlock, NI_SRCS_UNSAFE_InitBlockUnaligned, NI_SRCS_UNSAFE_IsAddressGreaterThan, + NI_SRCS_UNSAFE_IsAddressGreaterThanOrEqualTo, NI_SRCS_UNSAFE_IsAddressLessThan, + NI_SRCS_UNSAFE_IsAddressLessThanOrEqualTo, NI_SRCS_UNSAFE_IsNullRef, NI_SRCS_UNSAFE_NullRef, NI_SRCS_UNSAFE_Read, diff --git a/src/libraries/System.Linq/src/System/Linq/Range.cs b/src/libraries/System.Linq/src/System/Linq/Range.cs index 24da4789b1cb5b..7c5bae3f395e80 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.cs @@ -97,7 +97,7 @@ private static void FillIncrementing(Span destination, T value) where T : current += increment; pos = ref Unsafe.Add(ref pos, Vector.Count); } - while (!Unsafe.IsAddressGreaterThan(ref pos, ref oneVectorFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref pos, ref oneVectorFromEnd)); value = current[0]; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Collections/Generic/ArraySortHelper.cs b/src/libraries/System.Private.CoreLib/src/System/Collections/Generic/ArraySortHelper.cs index 9c336d924c75d8..eb4c3828cd930e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Collections/Generic/ArraySortHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Collections/Generic/ArraySortHelper.cs @@ -487,7 +487,7 @@ private static unsafe int PickPivotAndPartition(Span keys) while (Unsafe.IsAddressGreaterThan(ref rightRef, ref zeroRef) && LessThan(ref pivot, ref rightRef = ref Unsafe.Add(ref rightRef, -1))) ; } - if (!Unsafe.IsAddressLessThan(ref leftRef, ref rightRef)) + if (Unsafe.IsAddressGreaterThanOrEqualTo(ref leftRef, ref rightRef)) { break; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs index 3d97a654124205..0f8358c917fb49 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs @@ -404,6 +404,22 @@ public static bool IsAddressGreaterThan([AllowNull] ref readonly T left, [All // ret } + /// + /// Determines whether the memory address referenced by is greater than + /// or equal to the memory address referenced by . + /// + /// + /// This check is conceptually similar to "(void*)(&left) >= (void*)(&right)". + /// + [Intrinsic] + [NonVersionable] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsAddressGreaterThanOrEqualTo([AllowNull] ref readonly T left, [AllowNull] ref readonly T right) + where T : allows ref struct + { + return !IsAddressLessThan(in left, in right); + } + /// /// Determines whether the memory address referenced by is less than /// the memory address referenced by . @@ -428,6 +444,22 @@ public static bool IsAddressLessThan([AllowNull] ref readonly T left, [AllowN // ret } + /// + /// Determines whether the memory address referenced by is less than + /// or equal to the memory address referenced by . + /// + /// + /// This check is conceptually similar to "(void*)(&left) <= (void*)(&right)". + /// + [Intrinsic] + [NonVersionable] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsAddressLessThanOrEqualTo([AllowNull] ref readonly T left, [AllowNull] ref readonly T right) + where T : allows ref struct + { + return !IsAddressGreaterThan(in left, in right); + } + /// /// Initializes a block of memory at the given location with a given initial value. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs index 7f8393f483179e..3808fb037410c9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs @@ -321,7 +321,7 @@ private static TResult IndexOfAnyCores into a Vector256 is cheap compared to the lookup, we can effectively double the throughput. // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - (2 * Vector256.Count)); do @@ -374,7 +374,7 @@ private static TResult IndexOfAnyCores into a Vector128 is cheap compared to the lookup, we can effectively double the throughput. // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - (2 * Vector128.Count)); do @@ -453,7 +453,7 @@ public static int LastIndexOfAny(ref // As packing two Vector256s into a Vector256 is cheap compared to the lookup, we can effectively double the throughput. // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". + // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "IsAddressGreaterThanOrEqualTo". ref short twoVectorsAfterStart = ref Unsafe.Add(ref searchSpace, 2 * Vector256.Count); do @@ -504,7 +504,7 @@ public static int LastIndexOfAny(ref // As packing two Vector128s into a Vector128 is cheap compared to the lookup, we can effectively double the throughput. // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". + // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "IsAddressGreaterThanOrEqualTo". ref short twoVectorsAfterStart = ref Unsafe.Add(ref searchSpace, 2 * Vector128.Count); do @@ -604,7 +604,7 @@ private static TResult IndexOfAnyCore" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref byte vectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - Vector256.Count); do @@ -653,7 +653,7 @@ private static TResult IndexOfAnyCore" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref byte vectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - Vector128.Count); do @@ -729,7 +729,7 @@ public static int LastIndexOfAny(ref byte searchSpac // Process the input in chunks of 32 bytes. // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". + // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "IsAddressGreaterThanOrEqualTo". ref byte vectorAfterStart = ref Unsafe.Add(ref searchSpace, Vector256.Count); do @@ -778,7 +778,7 @@ public static int LastIndexOfAny(ref byte searchSpac // Process the input in chunks of 16 bytes. // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". + // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "IsAddressGreaterThanOrEqualTo". ref byte vectorAfterStart = ref Unsafe.Add(ref searchSpace, Vector128.Count); do @@ -876,7 +876,7 @@ private static TResult IndexOfAnyCore(ref byte // Process the input in chunks of 32 bytes. // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref byte vectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - Vector256.Count); do @@ -928,7 +928,7 @@ private static TResult IndexOfAnyCore(ref byte // Process the input in chunks of 16 bytes. // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref byte vectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength - Vector128.Count); do @@ -1004,7 +1004,7 @@ public static int LastIndexOfAny(ref byte searchSpace, int searchSpace // Process the input in chunks of 32 bytes. // If the input length is a multiple of 32, don't consume the last 32 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". + // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "IsAddressGreaterThanOrEqualTo". ref byte vectorAfterStart = ref Unsafe.Add(ref searchSpace, Vector256.Count); do @@ -1056,7 +1056,7 @@ public static int LastIndexOfAny(ref byte searchSpace, int searchSpace // Process the input in chunks of 16 bytes. // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "!IsAddressLessThan". + // ">" instead of ">=" above, and why "IsAddressGreaterThan" is used instead of "IsAddressGreaterThanOrEqualTo". ref byte vectorAfterStart = ref Unsafe.Add(ref searchSpace, Vector128.Count); do diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs index 9a4e3dd0b735b2..0b09377b503e17 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs @@ -626,7 +626,7 @@ private static int LastIndexOfAnyVectorizedAvx512(ref char sea } } - if (!Unsafe.IsAddressGreaterThan(ref cur, ref lastStartVector)) + if (Unsafe.IsAddressLessThanOrEqualTo(ref cur, ref lastStartVector)) { if (Unsafe.AreSame(ref cur, ref searchSpace)) { @@ -715,7 +715,7 @@ private static int LastIndexOfAnyVectorized(ref char searchSpa } } - if (!Unsafe.IsAddressGreaterThan(ref cur, ref lastStartVectorAvx2)) + if (Unsafe.IsAddressLessThanOrEqualTo(ref cur, ref lastStartVectorAvx2)) { if (Unsafe.AreSame(ref cur, ref searchSpace)) { @@ -757,7 +757,7 @@ private static int LastIndexOfAnyVectorized(ref char searchSpa } } - if (!Unsafe.IsAddressGreaterThan(ref cur, ref lastStartVector)) + if (Unsafe.IsAddressLessThanOrEqualTo(ref cur, ref lastStartVector)) { if (Unsafe.AreSame(ref cur, ref searchSpace)) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index d8970655cb31b7..086061ffe01641 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -142,7 +142,7 @@ private readonly int IndexOfAnyCore(ReadOnlySpan span) } } - if (!Unsafe.IsAddressLessThan(ref current, ref end)) + if (Unsafe.IsAddressGreaterThanOrEqualTo(ref current, ref end)) { break; } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 52eb1b0b2bedce..70f03605d66a32 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -156,7 +156,7 @@ public static bool Contains(ref short searchSpace, short value, int length) // Process the input in chunks of 64 characters (2 * Vector512). // If the input length is a multiple of 64, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector512.Count)); do @@ -205,7 +205,7 @@ public static bool Contains(ref short searchSpace, short value, int length) // Process the input in chunks of 32 characters (2 * Vector256). // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); do @@ -263,7 +263,7 @@ public static bool Contains(ref short searchSpace, short value, int length) // Process the input in chunks of 16 characters (2 * Vector128). // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); do @@ -355,7 +355,7 @@ private static int IndexOf(ref short searchSpace, short va // Process the input in chunks of 64 characters (2 * Vector512). // If the input length is a multiple of 64, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector512.Count)); do @@ -404,7 +404,7 @@ private static int IndexOf(ref short searchSpace, short va // Process the input in chunks of 32 characters (2 * Vector256). // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); do @@ -464,7 +464,7 @@ private static int IndexOf(ref short searchSpace, short va // Process the input in chunks of 16 characters (2 * Vector128). // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); do @@ -565,7 +565,7 @@ private static int IndexOfAny(ref short searchSpace, short // Process the input in chunks of 64 characters (2 * Vector512). // If the input length is a multiple of 64, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector512.Count)); do @@ -617,7 +617,7 @@ private static int IndexOfAny(ref short searchSpace, short // Process the input in chunks of 32 characters (2 * Vector256). // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); do @@ -678,7 +678,7 @@ private static int IndexOfAny(ref short searchSpace, short // Process the input in chunks of 16 characters (2 * Vector128). // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); do @@ -781,7 +781,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho // Process the input in chunks of 64 characters (2 * Vector512). // If the input length is a multiple of 64, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector512.Count)); do @@ -834,7 +834,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho // Process the input in chunks of 32 characters (2 * Vector256). // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); do @@ -896,7 +896,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho // Process the input in chunks of 16 characters (2 * Vector128). // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); do @@ -980,7 +980,7 @@ private static int IndexOfAnyInRange(ref short searchSpace, short lowI // Process the input in chunks of 64 characters (2 * Vector512). // If the input length is a multiple of 64, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector512.Count)); do @@ -1030,7 +1030,7 @@ private static int IndexOfAnyInRange(ref short searchSpace, short lowI // Process the input in chunks of 32 characters (2 * Vector256). // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); do @@ -1091,7 +1091,7 @@ private static int IndexOfAnyInRange(ref short searchSpace, short lowI // Process the input in chunks of 16 characters (2 * Vector128). // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. // Let the fallback below handle it instead. This is why the condition is - // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "IsAddressLessThanOrEqualTo". ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); do diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index c29adea608902a..cf8e536c5fded1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1383,7 +1383,7 @@ internal static bool NonPackedContainsValueType(ref T searchSpace, T value, i currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector512.Count); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector512.Count != 0) @@ -1414,7 +1414,7 @@ internal static bool NonPackedContainsValueType(ref T searchSpace, T value, i return true; } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector256.Count != 0) @@ -1444,7 +1444,7 @@ internal static bool NonPackedContainsValueType(ref T searchSpace, T value, i return true; } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the first vector in the search space. if ((uint)length % Vector128.Count != 0) @@ -1574,7 +1574,7 @@ internal static int NonPackedIndexOfValueType(ref TValue searc currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector512.Count); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector512.Count != 0) @@ -1605,7 +1605,7 @@ internal static int NonPackedIndexOfValueType(ref TValue searc return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector256.Count != 0) @@ -1635,7 +1635,7 @@ internal static int NonPackedIndexOfValueType(ref TValue searc return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the first vector in the search space. if ((uint)length % Vector128.Count != 0) @@ -1800,7 +1800,7 @@ internal static int NonPackedIndexOfAnyValueType(ref TValue se return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector512.Count != 0) @@ -1832,7 +1832,7 @@ internal static int NonPackedIndexOfAnyValueType(ref TValue se return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector256.Count != 0) @@ -1864,7 +1864,7 @@ internal static int NonPackedIndexOfAnyValueType(ref TValue se return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the first vector in the search space. if ((uint)length % Vector128.Count != 0) @@ -2007,7 +2007,7 @@ internal static int NonPackedIndexOfAnyValueType(ref TValue se return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector512.Count != 0) @@ -2039,7 +2039,7 @@ internal static int NonPackedIndexOfAnyValueType(ref TValue se return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector256.Count != 0) @@ -2071,7 +2071,7 @@ internal static int NonPackedIndexOfAnyValueType(ref TValue se return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the first vector in the search space. if ((uint)length % Vector128.Count != 0) @@ -2164,7 +2164,7 @@ private static int IndexOfAnyValueType(ref TValue searchSpace, return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector512.Count != 0) @@ -2198,7 +2198,7 @@ private static int IndexOfAnyValueType(ref TValue searchSpace, return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector256.Count != 0) @@ -2232,7 +2232,7 @@ private static int IndexOfAnyValueType(ref TValue searchSpace, return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the first vector in the search space. if ((uint)length % Vector128.Count != 0) @@ -2328,7 +2328,7 @@ private static int IndexOfAnyValueType(ref TValue searchSpace, return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector512.Count != 0) @@ -2363,7 +2363,7 @@ private static int IndexOfAnyValueType(ref TValue searchSpace, return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the last vector in the search space. if ((uint)length % Vector256.Count != 0) @@ -2398,7 +2398,7 @@ private static int IndexOfAnyValueType(ref TValue searchSpace, return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentSearchSpace, ref oneVectorAwayFromEnd)); // If any elements remain, process the first vector in the search space. if ((uint)length % Vector128.Count != 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs index 77c3d8de92fb23..3814ba3fe622b3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs @@ -83,7 +83,7 @@ private static bool Equals(ref TLeft left, ref TRight ri currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, Vector512.Count); currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, Vector512.Count); } - while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); // If any elements remain, process the last vector in the search space. if (length % (uint)Vector512.Count != 0) @@ -113,7 +113,7 @@ private static bool Equals(ref TLeft left, ref TRight ri currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, Vector256.Count); currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, Vector256.Count); } - while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); // If any elements remain, process the last vector in the search space. if (length % (uint)Vector256.Count != 0) @@ -147,7 +147,7 @@ private static bool Equals(ref TLeft left, ref TRight ri currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector128.Count); currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, TLoader.Count128); } - while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); // If any elements remain, process the last vector in the search space. if (length % (uint)Vector128.Count != 0) @@ -274,7 +274,7 @@ private static bool EqualsIgnoreCase(ref TLeft left, ref currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector512.Count); currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, TLoader.Count512); } - while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); // If any elements remain, process the last vector in the search space. if (length % (uint)Vector512.Count != 0) @@ -346,7 +346,7 @@ private static bool EqualsIgnoreCase(ref TLeft left, ref currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector256.Count); currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, TLoader.Count256); } - while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); // If any elements remain, process the last vector in the search space. if (length % (uint)Vector256.Count != 0) @@ -419,7 +419,7 @@ private static bool EqualsIgnoreCase(ref TLeft left, ref currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector128.Count); currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, TLoader.Count128); } - while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); // If any elements remain, process the last vector in the search space. if (length % (uint)Vector128.Count != 0) diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index ee744c9e438b5a..0683865c746ed0 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -14017,7 +14017,9 @@ public static void InitBlockUnaligned(ref byte startAddress, byte value, uint by [System.CLSCompliantAttribute(false)] public unsafe static void InitBlockUnaligned(void* startAddress, byte value, uint byteCount) { } public static bool IsAddressGreaterThan([System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T left, [System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T right) where T : allows ref struct { throw null; } + public static bool IsAddressGreaterThanOrEqualTo([System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T left, [System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T right) where T : allows ref struct { throw null; } public static bool IsAddressLessThan([System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T left, [System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T right) where T : allows ref struct { throw null; } + public static bool IsAddressLessThanOrEqualTo([System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T left, [System.Diagnostics.CodeAnalysis.AllowNull] ref readonly T right) where T : allows ref struct { throw null; } public static bool IsNullRef(ref readonly T source) where T : allows ref struct { throw null; } public static ref T NullRef() where T : allows ref struct { throw null; } public static T ReadUnaligned(scoped ref readonly byte source) where T : allows ref struct { throw null; } diff --git a/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs index 3d45fe2fbe0527..b5bec7f9357f62 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs @@ -805,6 +805,25 @@ public static unsafe void RefIsAddressGreaterThan() Assert.False(Unsafe.IsAddressGreaterThan(ref Unsafe.AsRef(null), ref Unsafe.AsRef(null))); } + [Fact] + public static unsafe void RefIsAddressGreaterThanOrEqualTo() + { + int[] a = new int[2]; + + Assert.True(Unsafe.IsAddressGreaterThanOrEqualTo(ref a[0], ref a[0])); + Assert.False(Unsafe.IsAddressGreaterThanOrEqualTo(ref a[0], ref a[1])); + Assert.True(Unsafe.IsAddressGreaterThanOrEqualTo(ref a[1], ref a[0])); + Assert.True(Unsafe.IsAddressGreaterThanOrEqualTo(ref a[1], ref a[1])); + + // The following tests ensure that we're using unsigned comparison logic + + Assert.False(Unsafe.IsAddressGreaterThanOrEqualTo(ref Unsafe.AsRef((void*)(1)), ref Unsafe.AsRef((void*)(-1)))); + Assert.True(Unsafe.IsAddressGreaterThanOrEqualTo(ref Unsafe.AsRef((void*)(-1)), ref Unsafe.AsRef((void*)(1)))); + Assert.True(Unsafe.IsAddressGreaterThanOrEqualTo(ref Unsafe.AsRef((void*)(int.MinValue)), ref Unsafe.AsRef((void*)(int.MaxValue)))); + Assert.False(Unsafe.IsAddressGreaterThanOrEqualTo(ref Unsafe.AsRef((void*)(int.MaxValue)), ref Unsafe.AsRef((void*)(int.MinValue)))); + Assert.True(Unsafe.IsAddressGreaterThanOrEqualTo(ref Unsafe.AsRef(null), ref Unsafe.AsRef(null))); + } + [Fact] public static unsafe void RefIsAddressLessThan() { @@ -824,6 +843,25 @@ public static unsafe void RefIsAddressLessThan() Assert.False(Unsafe.IsAddressLessThan(ref Unsafe.AsRef(null), ref Unsafe.AsRef(null))); } + [Fact] + public static unsafe void RefIsAddressLessThanOrEqualTo() + { + int[] a = new int[2]; + + Assert.True(Unsafe.IsAddressLessThanOrEqualTo(ref a[0], ref a[0])); + Assert.True(Unsafe.IsAddressLessThanOrEqualTo(ref a[0], ref a[1])); + Assert.False(Unsafe.IsAddressLessThanOrEqualTo(ref a[1], ref a[0])); + Assert.True(Unsafe.IsAddressLessThanOrEqualTo(ref a[1], ref a[1])); + + // The following tests ensure that we're using unsigned comparison logic + + Assert.True(Unsafe.IsAddressLessThanOrEqualTo(ref Unsafe.AsRef((void*)(1)), ref Unsafe.AsRef((void*)(-1)))); + Assert.False(Unsafe.IsAddressLessThanOrEqualTo(ref Unsafe.AsRef((void*)(-1)), ref Unsafe.AsRef((void*)(1)))); + Assert.False(Unsafe.IsAddressLessThanOrEqualTo(ref Unsafe.AsRef((void*)(int.MinValue)), ref Unsafe.AsRef((void*)(int.MaxValue)))); + Assert.True(Unsafe.IsAddressLessThanOrEqualTo(ref Unsafe.AsRef((void*)(int.MaxValue)), ref Unsafe.AsRef((void*)(int.MinValue)))); + Assert.True(Unsafe.IsAddressLessThanOrEqualTo(ref Unsafe.AsRef(null), ref Unsafe.AsRef(null))); + } + [Fact] public static unsafe void ReadUnaligned_ByRef_Int32() {