Skip to content

Commit 63b5af9

Browse files
author
Martin Taillefer
committed
Redaction improvements.
- Add the HMAC redactor. - Add support for salting redacted values. The salt is derived from the name of the item being redacted, and can optionally include the current day of the year (which induces daily rotation). - Delete the xxHash3 redactor.
1 parent 3807091 commit 63b5af9

File tree

29 files changed

+1242
-343
lines changed

29 files changed

+1242
-343
lines changed

‎eng/Tools/ApiChief/Format/FormattingExtensions.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ namespace ApiChief.Format;
99

1010
internal static class FormattingExtensions
1111
{
12-
private static readonly HashSet<char> _numberLiterals = new() { 'l', 'L', 'u', 'U', 'f', 'F', 'd', 'D', 'm', 'M' };
13-
private static readonly HashSet<char> _secondCharInLiterals = new() { 'l', 'L', 'u', 'U' };
14-
private static readonly HashSet<char> _possibleSpecialCharactersInANumber = new() { '.', 'x', 'X', 'b', 'B' };
12+
private static readonly HashSet<char> _numberLiterals = ['l', 'L', 'u', 'U', 'f', 'F', 'd', 'D', 'm', 'M'];
13+
private static readonly HashSet<char> _secondCharInLiterals = ['l', 'L', 'u', 'U'];
14+
private static readonly HashSet<char> _possibleSpecialCharactersInANumber = ['.', 'x', 'X', 'b', 'B'];
1515

1616
/// <summary>
1717
/// Ensures a single space between parameters.

‎src/Libraries/Microsoft.AspNetCore.Diagnostics.Middleware/Microsoft.AspNetCore.Diagnostics.Middleware.json‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,24 @@
11
{
22
"Name": "Microsoft.AspNetCore.Diagnostics.Middleware, Version=8.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35",
33
"Types": [
4+
{
5+
"Type": "static class Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions",
6+
"Stage": "Experimental",
7+
"Methods": [
8+
{
9+
"Member": "static Microsoft.Extensions.DependencyInjection.IServiceCollection Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions.AddHttpLogEnricher<T>(this Microsoft.Extensions.DependencyInjection.IServiceCollection services);",
10+
"Stage": "Experimental"
11+
},
12+
{
13+
"Member": "static Microsoft.Extensions.DependencyInjection.IServiceCollection Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions.AddHttpLoggingRedaction(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action<Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions>? configure = null);",
14+
"Stage": "Experimental"
15+
},
16+
{
17+
"Member": "static Microsoft.Extensions.DependencyInjection.IServiceCollection Microsoft.Extensions.DependencyInjection.HttpLoggingServiceCollectionExtensions.AddHttpLoggingRedaction(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, Microsoft.Extensions.Configuration.IConfigurationSection section);",
18+
"Stage": "Experimental"
19+
}
20+
]
21+
},
422
{
523
"Type": "static class Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames",
624
"Stage": "Stable",
@@ -33,7 +51,7 @@
3351
{
3452
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.RequestHeaderPrefix",
3553
"Stage": "Stable",
36-
"Value": "RequestHeader_"
54+
"Value": "RequestHeader."
3755
},
3856
{
3957
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.ResponseBody",
@@ -43,7 +61,7 @@
4361
{
4462
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.ResponseHeaderPrefix",
4563
"Stage": "Stable",
46-
"Value": "ResponseHeader_"
64+
"Value": "ResponseHeader."
4765
},
4866
{
4967
"Member": "const string Microsoft.AspNetCore.Diagnostics.Logging.HttpLoggingTagNames.StatusCode",
@@ -58,6 +76,16 @@
5876
}
5977
]
6078
},
79+
{
80+
"Type": "interface Microsoft.AspNetCore.Diagnostics.Logging.IHttpLogEnricher",
81+
"Stage": "Experimental",
82+
"Methods": [
83+
{
84+
"Member": "void Microsoft.AspNetCore.Diagnostics.Logging.IHttpLogEnricher.Enrich(Microsoft.Extensions.Diagnostics.Enrichment.IEnrichmentTagCollector collector, Microsoft.AspNetCore.Http.HttpContext httpContext);",
85+
"Stage": "Experimental"
86+
}
87+
]
88+
},
6189
{
6290
"Type": "enum Microsoft.AspNetCore.Diagnostics.Logging.IncomingPathLoggingMode",
6391
"Stage": "Stable",
@@ -80,6 +108,42 @@
80108
}
81109
]
82110
},
111+
{
112+
"Type": "class Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions",
113+
"Stage": "Experimental",
114+
"Methods": [
115+
{
116+
"Member": "Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.LoggingRedactionOptions();",
117+
"Stage": "Experimental"
118+
}
119+
],
120+
"Properties": [
121+
{
122+
"Member": "System.Collections.Generic.ISet<string> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.ExcludePathStartsWith { get; set; }",
123+
"Stage": "Experimental"
124+
},
125+
{
126+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RequestHeadersDataClasses { get; set; }",
127+
"Stage": "Experimental"
128+
},
129+
{
130+
"Member": "Microsoft.AspNetCore.Diagnostics.Logging.IncomingPathLoggingMode Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RequestPathLoggingMode { get; set; }",
131+
"Stage": "Experimental"
132+
},
133+
{
134+
"Member": "Microsoft.Extensions.Http.Diagnostics.HttpRouteParameterRedactionMode Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RequestPathParameterRedactionMode { get; set; }",
135+
"Stage": "Experimental"
136+
},
137+
{
138+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.ResponseHeadersDataClasses { get; set; }",
139+
"Stage": "Experimental"
140+
},
141+
{
142+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.LoggingRedactionOptions.RouteParameterDataClasses { get; set; }",
143+
"Stage": "Experimental"
144+
}
145+
]
146+
},
83147
{
84148
"Type": "static class Microsoft.AspNetCore.Diagnostics.Latency.RequestCheckpointConstants",
85149
"Stage": "Stable",
@@ -136,7 +200,7 @@
136200
],
137201
"Properties": [
138202
{
139-
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.RequestHeadersLogEnricherOptions.Logging.HeadersDataClasses { get; set; }",
203+
"Member": "System.Collections.Generic.IDictionary<string, Microsoft.Extensions.Compliance.Classification.DataClassification> Microsoft.AspNetCore.Diagnostics.Logging.RequestHeadersLogEnricherOptions.HeadersDataClasses { get; set; }",
140204
"Stage": "Experimental"
141205
}
142206
]
@@ -194,4 +258,4 @@
194258
]
195259
}
196260
]
197-
}
261+
}

‎src/Libraries/Microsoft.Extensions.Compliance.Abstractions/Redaction/Redactor.cs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -245,30 +245,32 @@ public bool TryRedact<T>(T value, Span<char> destination, out int charsWritten,
245245
}
246246
#endif
247247

248-
string? str = null;
249248
ReadOnlySpan<char> ros = default;
250249
if (value is IFormattable)
251250
{
252251
var fmt = format.Length > 0 ? format.ToString() : string.Empty;
253-
str = ((IFormattable)value).ToString(fmt, provider);
252+
var str = ((IFormattable)value).ToString(fmt, provider);
253+
if (str != null)
254+
{
255+
ros = str.AsSpan();
256+
}
254257
}
255258
else if (value is char[])
256259
{
257-
// An attempt to call value.ToString() on a char[] will produce a string "System.Char[]" and all redaction will be attempted on it,
258-
// instead of the provided array. This will lead to incorrectly allocated buffers.
260+
// An attempt to call value.ToString() on a char[] will produce the string "System.Char[]" and redaction will be attempted on it,
261+
// instead of the provided array.
259262
//
260263
// Not using pattern matching as it is recognized by the JIT and since this is a generic type, the JIT ends up generating code
261264
// without any of those conditional statements being present. But this only happens when not using pattern matching.
262265
ros = ((char[])(object)value).AsSpan();
263266
}
264267
else
265268
{
266-
str = value?.ToString();
267-
}
268-
269-
if (str is not null)
270-
{
271-
ros = str.AsSpan();
269+
var str = value?.ToString();
270+
if (str != null)
271+
{
272+
ros = str.AsSpan();
273+
}
272274
}
273275

274276
int len = GetRedactedLength(ros);

‎src/Libraries/Microsoft.Extensions.Compliance.Redaction/DeniedValuesAttribute.cs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
#if !NET8_0_OR_GREATER
5+
6+
// Stolen from dotnet/runtime/src/libraries/System.ComponentModel.Annotations/src/System/ComponentModel/DataAnnotations/DeniedValueAttribute.cs
7+
8+
using System.Diagnostics.CodeAnalysis;
9+
10+
namespace System.ComponentModel.DataAnnotations
11+
{
12+
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter,
13+
AllowMultiple = false)]
14+
[ExcludeFromCodeCoverage]
15+
internal sealed class DeniedValuesAttribute : ValidationAttribute
16+
{
17+
public DeniedValuesAttribute(params object?[] values)
18+
{
19+
Values = values;
20+
}
21+
22+
public object?[] Values { get; }
23+
24+
public override bool IsValid(object? value)
25+
{
26+
foreach (object? denied in Values)
27+
{
28+
if (denied is null ? value is null : denied.Equals(value))
29+
{
30+
return false;
31+
}
32+
}
33+
34+
return true;
35+
}
36+
}
37+
}
38+
39+
#endif

‎src/Libraries/Microsoft.Extensions.Compliance.Redaction/HmacRedactor.cs‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Security.Cryptography;
6+
using Microsoft.Extensions.Options;
7+
using Microsoft.Shared.Diagnostics;
8+
using Microsoft.Shared.Text;
9+
10+
#if NETCOREAPP3_1_OR_GREATER
11+
using System.Runtime.CompilerServices;
12+
using System.Runtime.InteropServices;
13+
#else
14+
using System.Diagnostics.CodeAnalysis;
15+
using System.Text;
16+
#endif
17+
18+
namespace Microsoft.Extensions.Compliance.Redaction;
19+
20+
internal sealed class HmacRedactor : Redactor
21+
{
22+
#if NETCOREAPP3_1_OR_GREATER
23+
private const int SHA512HashSize = 64;
24+
#endif
25+
26+
private const int BytesOfHashWeUse = 16;
27+
28+
/// <remarks>
29+
/// Magic numbers are formula for calculating base64 length with padding.
30+
/// </remarks>
31+
private const int Base64HashLength = ((BytesOfHashWeUse + 2) / 3) * 4;
32+
33+
private readonly int _redactedLength;
34+
private readonly byte[] _hashKey;
35+
private readonly string _keyId;
36+
37+
public HmacRedactor(IOptions<HmacRedactorOptions> options)
38+
{
39+
var value = Throw.IfMemberNull(options, options.Value);
40+
41+
_hashKey = Convert.FromBase64String(value.Key);
42+
_keyId = value.KeyId.ToInvariantString();
43+
_redactedLength = Base64HashLength + _keyId.Length;
44+
}
45+
46+
public override int GetRedactedLength(ReadOnlySpan<char> source)
47+
{
48+
if (source.IsEmpty)
49+
{
50+
return 0;
51+
}
52+
53+
return _redactedLength;
54+
}
55+
56+
#if NETCOREAPP3_1_OR_GREATER
57+
public override int Redact(ReadOnlySpan<char> source, Span<char> destination)
58+
{
59+
var length = GetRedactedLength(source);
60+
if (length == 0)
61+
{
62+
return 0;
63+
}
64+
65+
Throw.IfBufferTooSmall(destination.Length, length, nameof(destination));
66+
67+
_keyId.AsSpan().CopyTo(destination);
68+
return CreateSha512Hash(source, destination[_keyId.Length..], _hashKey) + _keyId.Length;
69+
}
70+
71+
[SkipLocalsInit]
72+
private static int CreateSha512Hash(ReadOnlySpan<char> source, Span<char> destination, byte[] hashKey)
73+
{
74+
Span<byte> hashBuffer = stackalloc byte[SHA512HashSize];
75+
76+
#if NET6_0_OR_GREATER
77+
_ = HMACSHA512.HashData(hashKey, MemoryMarshal.AsBytes(source), hashBuffer);
78+
#else
79+
using var hmac = new HMACSHA512(hashKey);
80+
_ = hmac.TryComputeHash(MemoryMarshal.AsBytes(source), hashBuffer, out _);
81+
#endif
82+
83+
_ = Convert.TryToBase64Chars(hashBuffer.Slice(0, BytesOfHashWeUse), destination, out int charsWritten);
84+
85+
return charsWritten;
86+
}
87+
88+
#else
89+
90+
public override int Redact(ReadOnlySpan<char> source, Span<char> destination)
91+
{
92+
const int RemainingBytesToPadForBase64Hash = BytesOfHashWeUse % 3;
93+
94+
var length = GetRedactedLength(source);
95+
if (length == 0)
96+
{
97+
return 0;
98+
}
99+
100+
Throw.IfBufferTooSmall(destination.Length, length, nameof(destination));
101+
102+
_keyId.AsSpan().CopyTo(destination);
103+
return ConvertBytesToBase64(CreateSha512Hash(source, _hashKey), destination, RemainingBytesToPadForBase64Hash, _keyId.Length) + _keyId.Length;
104+
}
105+
106+
private static byte[] CreateSha512Hash(ReadOnlySpan<char> value, byte[] hashKey)
107+
{
108+
using var hmac = new HMACSHA512(hashKey);
109+
return hmac.ComputeHash(Encoding.Unicode.GetBytes(value.ToArray()));
110+
}
111+
112+
private static readonly char[] _base64CharactersTable =
113+
{
114+
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
115+
'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
116+
'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
117+
't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
118+
'8', '9', '+', '/', '=',
119+
};
120+
121+
[SuppressMessage("Code smell", "S109", Justification = "Bit operation.")]
122+
private static int ConvertBytesToBase64(byte[] hashToConvert, Span<char> destination, int remainingBytesToPad, int startOffset)
123+
{
124+
var iterations = BytesOfHashWeUse - remainingBytesToPad;
125+
var offset = startOffset;
126+
127+
for (var i = 0; i < iterations; i += 3)
128+
{
129+
destination[offset] = _base64CharactersTable[(hashToConvert[i] & 0xfc) >> 2];
130+
destination[offset + 1] = _base64CharactersTable[((hashToConvert[i] & 0x03) << 4) | ((hashToConvert[i + 1] & 0xf0) >> 4)];
131+
destination[offset + 2] = _base64CharactersTable[((hashToConvert[i + 1] & 0x0f) << 2) | ((hashToConvert[i + 2] & 0xc0) >> 6)];
132+
destination[offset + 3] = _base64CharactersTable[hashToConvert[i + 2] & 0x3f];
133+
offset += 4;
134+
}
135+
136+
#if false
137+
// This ends up not being needed, given the current size of the hash. We comment this out so it doesn't hurt code coverage.
138+
139+
140+
if (remainingBytesToPad == 2)
141+
{
142+
destination[offset] = _base64CharactersTable[(hashToConvert[iterations] & 0xfc) >> 2];
143+
destination[offset + 1] = _base64CharactersTable[((hashToConvert[iterations] & 0x03) << 4) | ((hashToConvert[iterations + 1] & 0xf0) >> 4)];
144+
destination[offset + 2] = _base64CharactersTable[(hashToConvert[iterations + 1] & 0x0f) << 2];
145+
destination[offset + 3] = _base64CharactersTable[64];
146+
offset += 4;
147+
}
148+
#endif
149+
150+
if (remainingBytesToPad == 1)
151+
{
152+
destination[offset] = _base64CharactersTable[(hashToConvert[iterations] & 0xfc) >> 2];
153+
destination[offset + 1] = _base64CharactersTable[(hashToConvert[iterations] & 0x03) << 4];
154+
destination[offset + 2] = _base64CharactersTable[64];
155+
destination[offset + 3] = _base64CharactersTable[64];
156+
offset += 4;
157+
}
158+
159+
var charsWritten = offset - startOffset;
160+
return charsWritten;
161+
}
162+
163+
#endif
164+
}

0 commit comments

Comments
 (0)