Skip to content

Commit ce2b2c7

Browse files
sharwellsharwell
authored
Merge pull request #67999 from sharwell/incremental-hash-2
Use optimized hash creation methods on .NET 5+
2 parents ae24476 + 48e9cc5 commit ce2b2c7

File tree

2 files changed

+87
-53
lines changed

2 files changed

+87
-53
lines changed

‎src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs‎

Lines changed: 81 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,14 @@
55
using System;
66
using System.Collections.Generic;
77
using System.Collections.Immutable;
8+
using System.Diagnostics;
89
using System.IO;
9-
using System.Security.Cryptography;
1010
using System.Runtime.InteropServices;
11+
using System.Security.Cryptography;
1112
using System.Threading;
1213
using Microsoft.CodeAnalysis.PooledObjects;
1314
using Microsoft.CodeAnalysis.Serialization;
1415
using Roslyn.Utilities;
15-
using System.Diagnostics;
16-
using System.Runtime.CompilerServices;
1716

1817
namespace Microsoft.CodeAnalysis
1918
{
@@ -23,47 +22,102 @@ internal partial class Checksum
2322
// https://github.com/dotnet/runtime/blob/f2db6d6093c54e5eeb9db2d8dcbe15b2db92ad8c/src/libraries/System.Security.Cryptography.Algorithms/src/System/Security/Cryptography/SHA256.cs#L18-L19
2423
private const int SHA256HashSizeBytes = 256 / 8;
2524

25+
#if NET5_0_OR_GREATER
2626
private static readonly ObjectPool<IncrementalHash> s_incrementalHashPool =
2727
new(() => IncrementalHash.CreateHash(HashAlgorithmName.SHA256), size: 20);
28+
#else
29+
private static readonly ObjectPool<SHA256> s_incrementalHashPool =
30+
new(SHA256.Create, size: 20);
31+
#endif
2832

33+
#if !NET5_0_OR_GREATER
2934
// Dedicated pools for the byte[]s we use to create checksums from two or three existing checksums. Sized to
3035
// exactly the space needed to splat the existing checksum data into the array and then hash it.
3136

3237
private static readonly ObjectPool<byte[]> s_twoChecksumByteArrayPool = new(() => new byte[HashSize * 2]);
3338
private static readonly ObjectPool<byte[]> s_threeChecksumByteArrayPool = new(() => new byte[HashSize * 3]);
39+
#endif
3440

3541
public static Checksum Create(IEnumerable<string> values)
3642
{
43+
#if NET5_0_OR_GREATER
44+
using var pooledHash = s_incrementalHashPool.GetPooledObject();
45+
46+
foreach (var value in values)
47+
{
48+
pooledHash.Object.AppendData(MemoryMarshal.AsBytes(value.AsSpan()));
49+
pooledHash.Object.AppendData(MemoryMarshal.AsBytes("\0".AsSpan()));
50+
}
51+
52+
Span<byte> hash = stackalloc byte[SHA256HashSizeBytes];
53+
pooledHash.Object.GetHashAndReset(hash);
54+
return From(hash);
55+
#else
3756
using var pooledHash = s_incrementalHashPool.GetPooledObject();
3857
using var pooledBuffer = SharedPools.ByteArray.GetPooledObject();
3958
var hash = pooledHash.Object;
4059

60+
hash.Initialize();
4161
foreach (var value in values)
4262
{
4363
AppendData(hash, pooledBuffer.Object, value);
4464
AppendData(hash, pooledBuffer.Object, "\0");
4565
}
4666

47-
return From(hash.GetHashAndReset());
67+
hash.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
68+
return From(hash.Hash);
69+
#endif
4870
}
4971

5072
public static Checksum Create(string value)
5173
{
74+
#if NET5_0_OR_GREATER
75+
Span<byte> hash = stackalloc byte[SHA256HashSizeBytes];
76+
SHA256.HashData(MemoryMarshal.AsBytes(value.AsSpan()), hash);
77+
return From(hash);
78+
#else
5279
using var pooledHash = s_incrementalHashPool.GetPooledObject();
5380
using var pooledBuffer = SharedPools.ByteArray.GetPooledObject();
5481
var hash = pooledHash.Object;
82+
hash.Initialize();
5583

5684
AppendData(hash, pooledBuffer.Object, value);
5785

58-
return From(hash.GetHashAndReset());
86+
hash.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
87+
return From(hash.Hash);
88+
#endif
5989
}
6090

6191
public static Checksum Create(Stream stream)
6292
{
93+
#if NET7_0_OR_GREATER
94+
Span<byte> hash = stackalloc byte[SHA256HashSizeBytes];
95+
SHA256.HashData(stream, hash);
96+
return From(hash);
97+
#elif NET5_0_OR_GREATER
98+
using var pooledHash = s_incrementalHashPool.GetPooledObject();
99+
Span<byte> buffer = stackalloc byte[SharedPools.ByteBufferSize];
100+
101+
int bytesRead;
102+
do
103+
{
104+
bytesRead = stream.Read(buffer);
105+
if (bytesRead > 0)
106+
{
107+
pooledHash.Object.AppendData(buffer[..bytesRead]);
108+
}
109+
}
110+
while (bytesRead > 0);
111+
112+
Span<byte> hash = stackalloc byte[SHA256HashSizeBytes];
113+
pooledHash.Object.GetHashAndReset(hash);
114+
return From(hash);
115+
#else
63116
using var pooledHash = s_incrementalHashPool.GetPooledObject();
64117
using var pooledBuffer = SharedPools.ByteArray.GetPooledObject();
65118

66119
var hash = pooledHash.Object;
120+
hash.Initialize();
67121

68122
var buffer = pooledBuffer.Object;
69123
var bufferLength = buffer.Length;
@@ -73,12 +127,13 @@ public static Checksum Create(Stream stream)
73127
bytesRead = stream.Read(buffer, 0, bufferLength);
74128
if (bytesRead > 0)
75129
{
76-
hash.AppendData(buffer, 0, bytesRead);
130+
hash.TransformBlock(buffer, 0, bytesRead, null, 0);
77131
}
78132
}
79133
while (bytesRead > 0);
80134

81-
var bytes = hash.GetHashAndReset();
135+
hash.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
136+
var bytes = hash.Hash;
82137

83138
// if bytes array is bigger than certain size, checksum
84139
// will truncate it to predetermined size. for more detail,
@@ -91,6 +146,7 @@ public static Checksum Create(Stream stream)
91146
// hash algorithm used here should remain functionally correct even
92147
// after the truncation
93148
return From(bytes);
149+
#endif
94150
}
95151

96152
public static Checksum Create(IObjectWritable @object)
@@ -124,36 +180,44 @@ public static Checksum Create(Checksum checksum1, Checksum checksum2, Checksum c
124180
#endif
125181
}
126182

183+
#if !NET5_0_OR_GREATER
184+
127185
private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2)
128186
{
129-
using var hash = s_incrementalHashPool.GetPooledObject();
130187
using var bytes = s_twoChecksumByteArrayPool.GetPooledObject();
131188

132189
var bytesSpan = bytes.Object.AsSpan();
133190
checksum1.WriteTo(bytesSpan);
134191
checksum2.WriteTo(bytesSpan.Slice(HashSize));
135192

136-
hash.Object.AppendData(bytes.Object);
193+
using var hash = s_incrementalHashPool.GetPooledObject();
194+
hash.Object.Initialize();
195+
196+
hash.Object.TransformBlock(bytes.Object, 0, bytes.Object.Length, null, 0);
137197

138-
return From(hash.Object.GetHashAndReset());
198+
hash.Object.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
199+
return From(hash.Object.Hash);
139200
}
140201

141202
private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2, Checksum checksum3)
142203
{
143-
using var hash = s_incrementalHashPool.GetPooledObject();
144204
using var bytes = s_threeChecksumByteArrayPool.GetPooledObject();
145205

146206
var bytesSpan = bytes.Object.AsSpan();
147207
checksum1.WriteTo(bytesSpan);
148208
checksum2.WriteTo(bytesSpan.Slice(HashSize));
149209
checksum3.WriteTo(bytesSpan.Slice(2 * HashSize));
150210

151-
hash.Object.AppendData(bytes.Object);
211+
using var hash = s_incrementalHashPool.GetPooledObject();
212+
hash.Object.Initialize();
213+
214+
hash.Object.TransformBlock(bytes.Object, 0, bytes.Object.Length, null, 0);
152215

153-
return From(hash.Object.GetHashAndReset());
216+
hash.Object.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
217+
return From(hash.Object.Hash);
154218
}
155219

156-
#if NET
220+
#else
157221

158222
// Optimized helpers that do not need to allocate any arrays to combine hashes.
159223

@@ -234,7 +298,8 @@ public static Checksum Create(ParseOptions value, ISerializerService serializer)
234298
return Create(stream);
235299
}
236300

237-
private static void AppendData(IncrementalHash hash, byte[] buffer, string value)
301+
#if !NET5_0_OR_GREATER
302+
private static void AppendData(SHA256 hash, byte[] buffer, string value)
238303
{
239304
var stringBytes = MemoryMarshal.AsBytes(value.AsSpan());
240305
Debug.Assert(stringBytes.Length == value.Length * 2);
@@ -246,29 +311,11 @@ private static void AppendData(IncrementalHash hash, byte[] buffer, string value
246311
var toCopy = Math.Min(remaining, buffer.Length);
247312

248313
stringBytes.Slice(index, toCopy).CopyTo(buffer);
249-
hash.AppendData(buffer, 0, toCopy);
314+
hash.TransformBlock(buffer, 0, toCopy, null, 0);
250315

251316
index += toCopy;
252317
}
253318
}
254-
255-
public static class TestAccessor
256-
{
257-
public static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2)
258-
=> Checksum.CreateUsingByteArrays(checksum1, checksum2);
259-
260-
public static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2, Checksum checksum3)
261-
=> Checksum.CreateUsingByteArrays(checksum1, checksum2, checksum3);
262-
263-
#if NET
264-
265-
public static Checksum CreateUsingSpans(Checksum checksum1, Checksum checksum2)
266-
=> Checksum.CreateUsingSpans(checksum1, checksum2);
267-
268-
public static Checksum CreateUsingSpans(Checksum checksum1, Checksum checksum2, Checksum checksum3)
269-
=> Checksum.CreateUsingSpans(checksum1, checksum2, checksum3);
270-
271319
#endif
272-
}
273320
}
274321
}

‎src/Workspaces/CoreTest/ChecksumTests.cs‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,26 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using System;
6-
using System.Collections.Generic;
7-
using System.Linq;
8-
using System.Text;
9-
using System.Threading.Tasks;
105
using Xunit;
116

127
namespace Microsoft.CodeAnalysis.UnitTests
138
{
149
public class ChecksumTests
1510
{
16-
#if NET
1711
[Fact]
1812
public void ValidateChecksumFromSpanSameAsChecksumFromBytes1()
1913
{
2014
var checksum1 = Checksum.Create("Goo");
2115
var checksum2 = Checksum.Create("Bar");
2216

23-
var checksumA = Checksum.TestAccessor.CreateUsingByteArrays(checksum1, checksum2);
24-
var checksumB = Checksum.TestAccessor.CreateUsingSpans(checksum1, checksum2);
17+
var checksumA = Checksum.Create(checksum1, checksum2);
2518

26-
Assert.Equal(checksumA, checksumB);
19+
// Running this test on multiple target frameworks with the same expectation ensures the results match
20+
Assert.Equal(Checksum.FromBase64String("N30m5jwVeMZzWpy9cbQbtSYHoXU="), checksumA);
2721

2822
Assert.NotEqual(checksum1, checksum2);
29-
3023
Assert.NotEqual(checksum1, checksumA);
31-
Assert.NotEqual(checksum1, checksumB);
3224
Assert.NotEqual(checksum2, checksumA);
33-
Assert.NotEqual(checksum2, checksumB);
3425
}
3526

3627
[Fact]
@@ -40,22 +31,18 @@ public void ValidateChecksumFromSpanSameAsChecksumFromBytes2()
4031
var checksum2 = Checksum.Create("Bar");
4132
var checksum3 = Checksum.Create("Baz");
4233

43-
var checksumA = Checksum.TestAccessor.CreateUsingByteArrays(checksum1, checksum2, checksum3);
44-
var checksumB = Checksum.TestAccessor.CreateUsingSpans(checksum1, checksum2, checksum3);
34+
var checksumA = Checksum.Create(checksum1, checksum2, checksum3);
4535

46-
Assert.Equal(checksumA, checksumB);
36+
// Running this test on multiple target frameworks with the same expectation ensures the results match
37+
Assert.Equal(Checksum.FromBase64String("NEfIznmqkIqi4VJl12KxycWt7uo="), checksumA);
4738

4839
Assert.NotEqual(checksum1, checksum2);
4940
Assert.NotEqual(checksum2, checksum3);
5041
Assert.NotEqual(checksum3, checksum1);
5142

5243
Assert.NotEqual(checksum1, checksumA);
53-
Assert.NotEqual(checksum1, checksumB);
5444
Assert.NotEqual(checksum2, checksumA);
55-
Assert.NotEqual(checksum2, checksumB);
5645
Assert.NotEqual(checksum3, checksumA);
57-
Assert.NotEqual(checksum3, checksumB);
5846
}
59-
#endif
6047
}
6148
}

0 commit comments

Comments
 (0)