Skip to content

Commit 440dc57

Browse files
author
Martin Taillefer
committed
Redaction improvements.
- Add the HMAC redactor. - Add support for salting redacted values. The salt is derived from the name of the item being redacted, and can optionally include the current day of the year (which induces daily rotation). - Delete the xxHash3 redactor.
1 parent eb1c4f8 commit 440dc57

File tree

28 files changed

+1177
-280
lines changed

28 files changed

+1177
-280
lines changed

‎eng/Tools/ApiChief/Format/FormattingExtensions.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ namespace ApiChief.Format;
99

1010
internal static class FormattingExtensions
1111
{
12-
private static readonly HashSet<char> _numberLiterals = new() { 'l', 'L', 'u', 'U', 'f', 'F', 'd', 'D', 'm', 'M' };
13-
private static readonly HashSet<char> _secondCharInLiterals = new() { 'l', 'L', 'u', 'U' };
14-
private static readonly HashSet<char> _possibleSpecialCharactersInANumber = new() { '.', 'x', 'X', 'b', 'B' };
12+
private static readonly HashSet<char> _numberLiterals = ['l', 'L', 'u', 'U', 'f', 'F', 'd', 'D', 'm', 'M'];
13+
private static readonly HashSet<char> _secondCharInLiterals = ['l', 'L', 'u', 'U'];
14+
private static readonly HashSet<char> _possibleSpecialCharactersInANumber = ['.', 'x', 'X', 'b', 'B'];
1515

1616
/// <summary>
1717
/// Ensures a single space between parameters.

‎src/Libraries/Microsoft.AspNetCore.Diagnostics.Middleware/Microsoft.AspNetCore.Diagnostics.Middleware.json‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,24 @@
11
{
22
"Name": "Microsoft.AspNetCore.Diagnostics.Middleware, Version=8.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35",
33
"Types": [
4+
{
5+
"Type": "static class Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions",
6+
"Stage": "Experimental",
7+
"Methods": [
8+
{
9+
"Member": "static Microsoft.Extensions.DependencyInjection.IServiceCollection Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions.AddHttpLogEnricher<T>(this Microsoft.Extensions.DependencyInjection.IServiceCollection services);",
10+
"Stage": "Experimental"
11+
},
12+
{
13+
"Member": "static Microsoft.Extensions.DependencyInjection.IServiceCollection Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions.AddHttpLoggingRedaction(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action<Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions>? configure = null);",
14+
"Stage": "Experimental"
15+
},
16+
{
17+
"Member": "static Microsoft.Extensions.DependencyInjection.IServiceCollection Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions.AddHttpLoggingRedaction(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, Microsoft.Extensions.Configuration.IConfigurationSection section);",
18+
"Stage": "Experimental"
19+
}
20+
]
21+
},
422
{
523
"Type": "static class Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames",
624
"Stage": "Stable",
@@ -33,7 +51,7 @@
3351
{
3452
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.RequestHeaderPrefix",
3553
"Stage": "Stable",
36-
"Value": "RequestHeader_"
54+
"Value": "RequestHeader."
3755
},
3856
{
3957
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.ResponseBody",
@@ -43,7 +61,7 @@
4361
{
4462
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.ResponseHeaderPrefix",
4563
"Stage": "Stable",
46-
"Value": "ResponseHeader_"
64+
"Value": "ResponseHeader."
4765
},
4866
{
4967
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.StatusCode",
@@ -58,6 +76,16 @@
5876
}
5977
]
6078
},
79+
{
80+
"Type": "interface Microsoft.AspNetCore.Diagnostics.Logging.IHttpLogEnricher",
81+
"Stage": "Experimental",
82+
"Methods": [
83+
{
84+
"Member": "void Microsoft.AspNetCore.Diagnostics.Logging.IHttpLogEnricher.Enrich(Microsoft.Extensions.Diagnostics.Enrichment.IEnrichmentTagCollector collector, Microsoft.AspNetCore.Http.HttpContext httpContext);",
85+
"Stage": "Experimental"
86+
}
87+
]
88+
},
6189
{
6290
"Type": "enum Microsoft.AspNetCore.Diagnostics.Logging.IncomingPathLoggingMode",
6391
"Stage": "Stable",
@@ -80,6 +108,42 @@
80108
}
81109
]
82110
},
111+
{
112+
"Type": "class Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions",
113+
"Stage": "Experimental",
114+
"Methods": [
115+
{
116+
"Member": "Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.LoggingRedactionOptions();",
117+
"Stage": "Experimental"
118+
}
119+
],
120+
"Properties": [
121+
{
122+
"Member": "System.Collections.Generic.ISet<string> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.ExcludePathStartsWith { get; set; }",
123+
"Stage": "Experimental"
124+
},
125+
{
126+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RequestHeadersDataClasses { get; set; }",
127+
"Stage": "Experimental"
128+
},
129+
{
130+
"Member": "Microsoft.AspNetCore.Diagnostics.Logging.IncomingPathLoggingMode Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RequestPathLoggingMode { get; set; }",
131+
"Stage": "Experimental"
132+
},
133+
{
134+
"Member": "Microsoft.Extensions.Http.Diagnostics.HttpRouteParameterRedactionMode Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RequestPathParameterRedactionMode { get; set; }",
135+
"Stage": "Experimental"
136+
},
137+
{
138+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.ResponseHeadersDataClasses { get; set; }",
139+
"Stage": "Experimental"
140+
},
141+
{
142+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RouteParameterDataClasses { get; set; }",
143+
"Stage": "Experimental"
144+
}
145+
]
146+
},
83147
{
84148
"Type": "static class Microsoft.AspNetCore.Diagnostics.Latency.RequestCheckpointConstants",
85149
"Stage": "Stable",
@@ -136,7 +200,7 @@
136200
],
137201
"Properties": [
138202
{
139-
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.RequestHeadersLogEnricherOptions.Logging.HeadersDataClasses { get; set; }",
203+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.RequestHeadersLogEnricherOptions.HeadersDataClasses { get; set; }",
140204
"Stage": "Experimental"
141205
}
142206
]
@@ -194,4 +258,4 @@
194258
]
195259
}
196260
]
197-
}
261+
}

‎src/Libraries/Microsoft.Extensions.Compliance.Abstractions/Redaction/Redactor.cs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -245,30 +245,32 @@ public bool TryRedact<T>(T value, Span<char> destination, out int charsWritten,
245245
}
246246
#endif
247247

248-
string? str = null;
249248
ReadOnlySpan<char> ros = default;
250249
if (value is IFormattable)
251250
{
252251
var fmt = format.Length > 0 ? format.ToString() : string.Empty;
253-
str = ((IFormattable)value).ToString(fmt, provider);
252+
var str = ((IFormattable)value).ToString(fmt, provider);
253+
if (str != null)
254+
{
255+
ros = str.AsSpan();
256+
}
254257
}
255258
else if (value is char[])
256259
{
257-
// An attempt to call value.ToString() on a char[] will produce a string "System.Char[]" and all redaction will be attempted on it,
258-
// instead of the provided array. This will lead to incorrectly allocated buffers.
260+
// An attempt to call value.ToString() on a char[] will produce the string "System.Char[]" and redaction will be attempted on it,
261+
// instead of the provided array.
259262
//
260263
// Not using pattern matching as it is recognized by the JIT and since this is a generic type, the JIT ends up generating code
261264
// without any of those conditional statements being present. But this only happens when not using pattern matching.
262265
ros = ((char[])(object)value).AsSpan();
263266
}
264267
else
265268
{
266-
str = value?.ToString();
267-
}
268-
269-
if (str is not null)
270-
{
271-
ros = str.AsSpan();
269+
var str = value?.ToString();
270+
if (str != null)
271+
{
272+
ros = str.AsSpan();
273+
}
272274
}
273275

274276
int len = GetRedactedLength(ros);

‎src/Libraries/Microsoft.Extensions.Compliance.Redaction/HmacRedactor.cs‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Security.Cryptography;
6+
using Microsoft.Extensions.Options;
7+
using Microsoft.Shared.Diagnostics;
8+
using Microsoft.Shared.Text;
9+
10+
#if NET6_0_OR_GREATER
11+
using System.Runtime.CompilerServices;
12+
using System.Runtime.InteropServices;
13+
#else
14+
using System.Diagnostics.CodeAnalysis;
15+
using System.Text;
16+
#endif
17+
18+
namespace Microsoft.Extensions.Compliance.Redaction;
19+
20+
internal sealed class HmacRedactor : Redactor
21+
{
22+
#if NET6_0_OR_GREATER
23+
private const int SHA256HashSizeInBytes = 32;
24+
#endif
25+
private const int BytesOfHashWeUse = 16;
26+
27+
/// <remarks>
28+
/// Magic numbers are formula for calculating base64 length with padding.
29+
/// </remarks>
30+
private const int Base64HashLength = ((BytesOfHashWeUse + 2) / 3) * 4;
31+
32+
private readonly int _redactedLength;
33+
private readonly byte[] _hashKey;
34+
private readonly string _keyId;
35+
36+
public HmacRedactor(IOptions<HmacRedactorOptions> options)
37+
{
38+
var value = Throw.IfMemberNull(options, options.Value);
39+
40+
_hashKey = Convert.FromBase64String(value.Key);
41+
_keyId = value.KeyId.HasValue ? value.KeyId.Value.ToInvariantString() + ':' : string.Empty;
42+
_redactedLength = Base64HashLength + _keyId.Length;
43+
}
44+
45+
public override int GetRedactedLength(ReadOnlySpan<char> source)
46+
{
47+
if (source.IsEmpty)
48+
{
49+
return 0;
50+
}
51+
52+
return _redactedLength;
53+
}
54+
55+
#if NET6_0_OR_GREATER
56+
public override int Redact(ReadOnlySpan<char> source, Span<char> destination)
57+
{
58+
var length = GetRedactedLength(source);
59+
if (length == 0)
60+
{
61+
return 0;
62+
}
63+
64+
Throw.IfBufferTooSmall(destination.Length, length, nameof(destination));
65+
66+
_keyId.AsSpan().CopyTo(destination);
67+
return CreateSha256Hash(source, destination[_keyId.Length..], _hashKey) + _keyId.Length;
68+
}
69+
70+
[SkipLocalsInit]
71+
private static int CreateSha256Hash(ReadOnlySpan<char> source, Span<char> destination, byte[] hashKey)
72+
{
73+
Span<byte> hashBuffer = stackalloc byte[SHA256HashSizeInBytes];
74+
75+
_ = HMACSHA256.HashData(hashKey, MemoryMarshal.AsBytes(source), hashBuffer);
76+
77+
// this won't fail, we ensured the destination is big enough previously
78+
_ = Convert.TryToBase64Chars(hashBuffer.Slice(0, BytesOfHashWeUse), destination, out int charsWritten);
79+
80+
return charsWritten;
81+
}
82+
83+
#else
84+
85+
public override int Redact(ReadOnlySpan<char> source, Span<char> destination)
86+
{
87+
const int RemainingBytesToPadForBase64Hash = BytesOfHashWeUse % 3;
88+
89+
var length = GetRedactedLength(source);
90+
if (length == 0)
91+
{
92+
return 0;
93+
}
94+
95+
Throw.IfBufferTooSmall(destination.Length, length, nameof(destination));
96+
97+
_keyId.AsSpan().CopyTo(destination);
98+
return ConvertBytesToBase64(CreateSha256Hash(source, _hashKey), destination, RemainingBytesToPadForBase64Hash, _keyId.Length) + _keyId.Length;
99+
}
100+
101+
private static byte[] CreateSha256Hash(ReadOnlySpan<char> value, byte[] hashKey)
102+
{
103+
using var hmac = new HMACSHA256(hashKey);
104+
return hmac.ComputeHash(Encoding.Unicode.GetBytes(value.ToArray()));
105+
}
106+
107+
private static readonly char[] _base64CharactersTable =
108+
{
109+
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
110+
'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
111+
'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
112+
't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
113+
'8', '9', '+', '/', '=',
114+
};
115+
116+
[SuppressMessage("Code smell", "S109", Justification = "Bit operation.")]
117+
private static int ConvertBytesToBase64(byte[] hashToConvert, Span<char> destination, int remainingBytesToPad, int startOffset)
118+
{
119+
var iterations = BytesOfHashWeUse - remainingBytesToPad;
120+
var offset = startOffset;
121+
122+
unchecked
123+
{
124+
for (var i = 0; i < iterations; i += 3)
125+
{
126+
destination[offset] = _base64CharactersTable[(hashToConvert[i] & 0xfc) >> 2];
127+
destination[offset + 1] = _base64CharactersTable[((hashToConvert[i] & 0x03) << 4) | ((hashToConvert[i + 1] & 0xf0) >> 4)];
128+
destination[offset + 2] = _base64CharactersTable[((hashToConvert[i + 1] & 0x0f) << 2) | ((hashToConvert[i + 2] & 0xc0) >> 6)];
129+
destination[offset + 3] = _base64CharactersTable[hashToConvert[i + 2] & 0x3f];
130+
offset += 4;
131+
}
132+
133+
if (remainingBytesToPad == 2)
134+
{
135+
destination[offset] = _base64CharactersTable[(hashToConvert[iterations] & 0xfc) >> 2];
136+
destination[offset + 1] = _base64CharactersTable[((hashToConvert[iterations] & 0x03) << 4) | ((hashToConvert[iterations + 1] & 0xf0) >> 4)];
137+
destination[offset + 2] = _base64CharactersTable[(hashToConvert[iterations + 1] & 0x0f) << 2];
138+
destination[offset + 3] = _base64CharactersTable[64];
139+
offset += 4;
140+
}
141+
142+
if (remainingBytesToPad == 1)
143+
{
144+
destination[offset] = _base64CharactersTable[(hashToConvert[iterations] & 0xfc) >> 2];
145+
destination[offset + 1] = _base64CharactersTable[(hashToConvert[iterations] & 0x03) << 4];
146+
destination[offset + 2] = _base64CharactersTable[64];
147+
destination[offset + 3] = _base64CharactersTable[64];
148+
offset += 4;
149+
}
150+
}
151+
152+
var charsWritten = offset - startOffset;
153+
return charsWritten;
154+
}
155+
156+
#endif
157+
}

‎src/Libraries/Microsoft.Extensions.Compliance.Redaction/HmacRedactorOptions.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Diagnostics.CodeAnalysis;
5+
using Microsoft.Shared.DiagnosticIds;
6+
7+
namespace Microsoft.Extensions.Compliance.Redaction;
8+
9+
/// <summary>
10+
/// A redactor using HMACSHA256 to encode data being redacted.
11+
/// </summary>
12+
[Experimental(diagnosticId: DiagnosticIds.Experiments.Compliance, UrlFormat = DiagnosticIds.UrlFormat)]
13+
public class HmacRedactorOptions
14+
{
15+
/// <summary>
16+
/// Gets or sets the key ID.
17+
/// </summary>
18+
/// <value>
19+
/// Default set to <see langword="null"/>.
20+
/// </value>
21+
/// <remarks>
22+
/// The key id is appended to each redacted value and is intended to identity the key that was used to hash the data.
23+
/// In general, every distinct key should have a unique id associated with it. When the hashed values have different key ids,
24+
/// it means the values are unrelated and can't be used for correlation.
25+
/// </remarks>
26+
public int? KeyId { get; set; }
27+
28+
/// <summary>
29+
/// Gets or sets the hashing key.
30+
/// </summary>
31+
/// <remarks>
32+
/// The key is specified in base 64 format, and must be a minimum of 44 characters long.
33+
///
34+
/// We recommend using a distinct key for each major deployment of a service (say for each region the service is in). Additionally,
35+
/// the key material should be kept secret, and rotated on a regular basis.
36+
/// </remarks>
37+
/// <value>
38+
/// Default set to <see cref="string.Empty" />.
39+
/// </value>
40+
[StringSyntax("Base64")]
41+
#if NET8_0_OR_GREATER
42+
[System.ComponentModel.DataAnnotations.Base64String]
43+
#endif
44+
[Microsoft.Shared.Data.Validation.Length(44)]
45+
public string Key { get; set; } = string.Empty;
46+
}

‎src/Libraries/Microsoft.Extensions.Compliance.Redaction/HmacRedactorOptionsValidator.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using Microsoft.Extensions.Options;
5+
6+
namespace Microsoft.Extensions.Compliance.Redaction;
7+
8+
[OptionsValidator]
9+
internal sealed partial class HmacRedactorOptionsValidator : IValidateOptions<HmacRedactorOptions>
10+
{
11+
}

0 commit comments

Comments
 (0)