Skip to content

Commit cc89f38

Browse files
MihaZupanMihaZupan
authored
Move scalar loops into IndexOfAnyAsciiSearcher (#91937)
1 parent e4df432 commit cc89f38

File tree

9 files changed

+283
-277
lines changed

9 files changed

+283
-277
lines changed

‎src/libraries/System.Private.CoreLib/src/System/SearchValues/AnyByteSearchValues.cs‎

Lines changed: 29 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -3,94 +3,56 @@
33

44
using System.Runtime.CompilerServices;
55
using System.Runtime.InteropServices;
6-
using System.Runtime.Intrinsics;
6+
using System.Runtime.Intrinsics.Arm;
7+
using System.Runtime.Intrinsics.Wasm;
8+
using System.Runtime.Intrinsics.X86;
79

810
namespace System.Buffers
911
{
1012
internal sealed class AnyByteSearchValues : SearchValues<byte>
1113
{
12-
private Vector512<byte> _bitmaps;
13-
private readonly BitVector256 _lookup;
14+
private IndexOfAnyAsciiSearcher.AnyByteState _state;
1415

15-
public AnyByteSearchValues(ReadOnlySpan<byte> values)
16-
{
17-
IndexOfAnyAsciiSearcher.ComputeBitmap256(values, out Vector256<byte> bitmap0, out Vector256<byte> bitmap1, out _lookup);
18-
_bitmaps = Vector512.Create(bitmap0, bitmap1);
19-
}
16+
public AnyByteSearchValues(ReadOnlySpan<byte> values) =>
17+
IndexOfAnyAsciiSearcher.ComputeAnyByteState(values, out _state);
2018

21-
internal override byte[] GetValues() => _lookup.GetByteValues();
19+
internal override byte[] GetValues() =>
20+
_state.Lookup.GetByteValues();
2221

2322
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2423
internal override bool ContainsCore(byte value) =>
25-
_lookup.Contains(value);
24+
_state.Lookup.Contains(value);
2625

26+
[CompExactlyDependsOn(typeof(Ssse3))]
27+
[CompExactlyDependsOn(typeof(AdvSimd))]
28+
[CompExactlyDependsOn(typeof(PackedSimd))]
2729
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2830
internal override int IndexOfAny(ReadOnlySpan<byte> span) =>
29-
IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
31+
IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.DontNegate>(
32+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
3033

34+
[CompExactlyDependsOn(typeof(Ssse3))]
35+
[CompExactlyDependsOn(typeof(AdvSimd))]
36+
[CompExactlyDependsOn(typeof(PackedSimd))]
3137
[MethodImpl(MethodImplOptions.AggressiveInlining)]
3238
internal override int IndexOfAnyExcept(ReadOnlySpan<byte> span) =>
33-
IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
39+
IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.Negate>(
40+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
3441

42+
[CompExactlyDependsOn(typeof(Ssse3))]
43+
[CompExactlyDependsOn(typeof(AdvSimd))]
44+
[CompExactlyDependsOn(typeof(PackedSimd))]
3545
[MethodImpl(MethodImplOptions.AggressiveInlining)]
3646
internal override int LastIndexOfAny(ReadOnlySpan<byte> span) =>
37-
LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
47+
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.DontNegate>(
48+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
3849

50+
[CompExactlyDependsOn(typeof(Ssse3))]
51+
[CompExactlyDependsOn(typeof(AdvSimd))]
52+
[CompExactlyDependsOn(typeof(PackedSimd))]
3953
[MethodImpl(MethodImplOptions.AggressiveInlining)]
4054
internal override int LastIndexOfAnyExcept(ReadOnlySpan<byte> span) =>
41-
LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
42-
43-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
44-
private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
45-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
46-
{
47-
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
48-
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorizedAnyByte<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmaps)
49-
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
50-
}
51-
52-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
53-
private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
54-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
55-
{
56-
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
57-
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmaps)
58-
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
59-
}
60-
61-
private int IndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
62-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
63-
{
64-
ref byte searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
65-
ref byte cur = ref searchSpace;
66-
67-
while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
68-
{
69-
byte b = cur;
70-
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
71-
{
72-
return (int)Unsafe.ByteOffset(ref searchSpace, ref cur);
73-
}
74-
75-
cur = ref Unsafe.Add(ref cur, 1);
76-
}
77-
78-
return -1;
79-
}
80-
81-
private int LastIndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
82-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
83-
{
84-
for (int i = searchSpaceLength - 1; i >= 0; i--)
85-
{
86-
byte b = Unsafe.Add(ref searchSpace, i);
87-
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
88-
{
89-
return i;
90-
}
91-
}
92-
93-
return -1;
94-
}
55+
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorizedAnyByte<IndexOfAnyAsciiSearcher.Negate>(
56+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
9557
}
9658
}

‎src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiByteSearchValues.cs‎

Lines changed: 28 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -3,91 +3,56 @@
33

44
using System.Runtime.CompilerServices;
55
using System.Runtime.InteropServices;
6-
using System.Runtime.Intrinsics;
6+
using System.Runtime.Intrinsics.Arm;
7+
using System.Runtime.Intrinsics.Wasm;
8+
using System.Runtime.Intrinsics.X86;
79

810
namespace System.Buffers
911
{
1012
internal sealed class AsciiByteSearchValues : SearchValues<byte>
1113
{
12-
private Vector256<byte> _bitmap;
13-
private readonly BitVector256 _lookup;
14+
private IndexOfAnyAsciiSearcher.AsciiState _state;
1415

1516
public AsciiByteSearchValues(ReadOnlySpan<byte> values) =>
16-
IndexOfAnyAsciiSearcher.ComputeBitmap(values, out _bitmap, out _lookup);
17+
IndexOfAnyAsciiSearcher.ComputeAsciiState(values, out _state);
1718

18-
internal override byte[] GetValues() => _lookup.GetByteValues();
19+
internal override byte[] GetValues() =>
20+
_state.Lookup.GetByteValues();
1921

2022
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2123
internal override bool ContainsCore(byte value) =>
22-
_lookup.Contains(value);
24+
_state.Lookup.Contains(value);
2325

26+
[CompExactlyDependsOn(typeof(Ssse3))]
27+
[CompExactlyDependsOn(typeof(AdvSimd))]
28+
[CompExactlyDependsOn(typeof(PackedSimd))]
2429
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2530
internal override int IndexOfAny(ReadOnlySpan<byte> span) =>
26-
IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
31+
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate>(
32+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
2733

34+
[CompExactlyDependsOn(typeof(Ssse3))]
35+
[CompExactlyDependsOn(typeof(AdvSimd))]
36+
[CompExactlyDependsOn(typeof(PackedSimd))]
2837
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2938
internal override int IndexOfAnyExcept(ReadOnlySpan<byte> span) =>
30-
IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
39+
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate>(
40+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
3141

42+
[CompExactlyDependsOn(typeof(Ssse3))]
43+
[CompExactlyDependsOn(typeof(AdvSimd))]
44+
[CompExactlyDependsOn(typeof(PackedSimd))]
3245
[MethodImpl(MethodImplOptions.AggressiveInlining)]
3346
internal override int LastIndexOfAny(ReadOnlySpan<byte> span) =>
34-
LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
47+
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate>(
48+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
3549

50+
[CompExactlyDependsOn(typeof(Ssse3))]
51+
[CompExactlyDependsOn(typeof(AdvSimd))]
52+
[CompExactlyDependsOn(typeof(PackedSimd))]
3653
[MethodImpl(MethodImplOptions.AggressiveInlining)]
3754
internal override int LastIndexOfAnyExcept(ReadOnlySpan<byte> span) =>
38-
LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
39-
40-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
41-
private int IndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
42-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
43-
{
44-
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
45-
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmap)
46-
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
47-
}
48-
49-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
50-
private int LastIndexOfAny<TNegator>(ref byte searchSpace, int searchSpaceLength)
51-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
52-
{
53-
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= sizeof(ulong)
54-
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator>(ref searchSpace, searchSpaceLength, ref _bitmap)
55-
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
56-
}
57-
58-
private int IndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
59-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
60-
{
61-
ref byte searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
62-
ref byte cur = ref searchSpace;
63-
64-
while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
65-
{
66-
byte b = cur;
67-
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
68-
{
69-
return (int)Unsafe.ByteOffset(ref searchSpace, ref cur);
70-
}
71-
72-
cur = ref Unsafe.Add(ref cur, 1);
73-
}
74-
75-
return -1;
76-
}
77-
78-
private int LastIndexOfAnyScalar<TNegator>(ref byte searchSpace, int searchSpaceLength)
79-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
80-
{
81-
for (int i = searchSpaceLength - 1; i >= 0; i--)
82-
{
83-
byte b = Unsafe.Add(ref searchSpace, i);
84-
if (TNegator.NegateIfNeeded(_lookup.Contains(b)))
85-
{
86-
return i;
87-
}
88-
}
89-
90-
return -1;
91-
}
55+
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate>(
56+
ref MemoryMarshal.GetReference(span), span.Length, ref _state);
9257
}
9358
}

‎src/libraries/System.Private.CoreLib/src/System/SearchValues/AsciiCharSearchValues.cs‎

Lines changed: 29 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -3,95 +3,57 @@
33

44
using System.Runtime.CompilerServices;
55
using System.Runtime.InteropServices;
6-
using System.Runtime.Intrinsics;
6+
using System.Runtime.Intrinsics.Arm;
7+
using System.Runtime.Intrinsics.Wasm;
8+
using System.Runtime.Intrinsics.X86;
79

810
namespace System.Buffers
911
{
1012
internal sealed class AsciiCharSearchValues<TOptimizations> : SearchValues<char>
1113
where TOptimizations : struct, IndexOfAnyAsciiSearcher.IOptimizations
1214
{
13-
private Vector256<byte> _bitmap;
14-
private readonly BitVector256 _lookup;
15+
private IndexOfAnyAsciiSearcher.AsciiState _state;
1516

16-
public AsciiCharSearchValues(Vector256<byte> bitmap, BitVector256 lookup)
17-
{
18-
_bitmap = bitmap;
19-
_lookup = lookup;
20-
}
17+
public AsciiCharSearchValues(ReadOnlySpan<char> values) =>
18+
IndexOfAnyAsciiSearcher.ComputeAsciiState(values, out _state);
2119

22-
internal override char[] GetValues() => _lookup.GetCharValues();
20+
internal override char[] GetValues() =>
21+
_state.Lookup.GetCharValues();
2322

2423
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2524
internal override bool ContainsCore(char value) =>
26-
_lookup.Contains128(value);
25+
_state.Lookup.Contains128(value);
2726

27+
[CompExactlyDependsOn(typeof(Ssse3))]
28+
[CompExactlyDependsOn(typeof(AdvSimd))]
29+
[CompExactlyDependsOn(typeof(PackedSimd))]
2830
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2931
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
30-
IndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
32+
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate, TOptimizations>(
33+
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
3134

35+
[CompExactlyDependsOn(typeof(Ssse3))]
36+
[CompExactlyDependsOn(typeof(AdvSimd))]
37+
[CompExactlyDependsOn(typeof(PackedSimd))]
3238
[MethodImpl(MethodImplOptions.AggressiveInlining)]
3339
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
34-
IndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
40+
IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate, TOptimizations>(
41+
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
3542

43+
[CompExactlyDependsOn(typeof(Ssse3))]
44+
[CompExactlyDependsOn(typeof(AdvSimd))]
45+
[CompExactlyDependsOn(typeof(PackedSimd))]
3646
[MethodImpl(MethodImplOptions.AggressiveInlining)]
3747
internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
38-
LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate>(ref MemoryMarshal.GetReference(span), span.Length);
48+
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.DontNegate, TOptimizations>(
49+
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
3950

51+
[CompExactlyDependsOn(typeof(Ssse3))]
52+
[CompExactlyDependsOn(typeof(AdvSimd))]
53+
[CompExactlyDependsOn(typeof(PackedSimd))]
4054
[MethodImpl(MethodImplOptions.AggressiveInlining)]
4155
internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
42-
LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate>(ref MemoryMarshal.GetReference(span), span.Length);
43-
44-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
45-
private int IndexOfAny<TNegator>(ref char searchSpace, int searchSpaceLength)
46-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
47-
{
48-
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= Vector128<short>.Count
49-
? IndexOfAnyAsciiSearcher.IndexOfAnyVectorized<TNegator, TOptimizations>(ref Unsafe.As<char, short>(ref searchSpace), searchSpaceLength, ref _bitmap)
50-
: IndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
51-
}
52-
53-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
54-
private int LastIndexOfAny<TNegator>(ref char searchSpace, int searchSpaceLength)
55-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
56-
{
57-
return IndexOfAnyAsciiSearcher.IsVectorizationSupported && searchSpaceLength >= Vector128<short>.Count
58-
? IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<TNegator, TOptimizations>(ref Unsafe.As<char, short>(ref searchSpace), searchSpaceLength, ref _bitmap)
59-
: LastIndexOfAnyScalar<TNegator>(ref searchSpace, searchSpaceLength);
60-
}
61-
62-
private int IndexOfAnyScalar<TNegator>(ref char searchSpace, int searchSpaceLength)
63-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
64-
{
65-
ref char searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength);
66-
ref char cur = ref searchSpace;
67-
68-
while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd))
69-
{
70-
char c = cur;
71-
if (TNegator.NegateIfNeeded(_lookup.Contains128(c)))
72-
{
73-
return (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref cur) / sizeof(char));
74-
}
75-
76-
cur = ref Unsafe.Add(ref cur, 1);
77-
}
78-
79-
return -1;
80-
}
81-
82-
private int LastIndexOfAnyScalar<TNegator>(ref char searchSpace, int searchSpaceLength)
83-
where TNegator : struct, IndexOfAnyAsciiSearcher.INegator
84-
{
85-
for (int i = searchSpaceLength - 1; i >= 0; i--)
86-
{
87-
char c = Unsafe.Add(ref searchSpace, i);
88-
if (TNegator.NegateIfNeeded(_lookup.Contains128(c)))
89-
{
90-
return i;
91-
}
92-
}
93-
94-
return -1;
95-
}
56+
IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized<IndexOfAnyAsciiSearcher.Negate, TOptimizations>(
57+
ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
9658
}
9759
}

‎src/libraries/System.Private.CoreLib/src/System/SearchValues/BitVector256.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,18 @@ internal unsafe struct BitVector256
1111
{
1212
private fixed uint _values[8];
1313

14+
public readonly BitVector256 CreateInverse()
15+
{
16+
BitVector256 inverse = default;
17+
18+
for (int i = 0; i < 8; i++)
19+
{
20+
inverse._values[i] = ~_values[i];
21+
}
22+
23+
return inverse;
24+
}
25+
1426
public void Set(int c)
1527
{
1628
Debug.Assert(c < 256);

0 commit comments

Comments
 (0)