From 7c17eca572afb29808901a46883289edca554f54 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 1 Aug 2022 17:01:53 -0700 Subject: [PATCH 1/4] add RegexEntrySanitizer and skinned out tests. just need to finish remaining tests --- .../SanitizerTests.cs | 57 ++++++++++++ .../Common/StringSanitizer.cs | 15 ++++ .../Sanitizers/RegexEntrySanitizer.cs | 86 +++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs index 483152b2957..52e22f54d68 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs @@ -1,8 +1,14 @@ using Azure.Sdk.Tools.TestProxy.Common; +using Azure.Sdk.Tools.TestProxy.Matchers; using Azure.Sdk.Tools.TestProxy.Sanitizers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Reflection; using System.Text; +using System.Threading.Tasks; using Xunit; namespace Azure.Sdk.Tools.TestProxy.Tests @@ -10,6 +16,7 @@ namespace Azure.Sdk.Tools.TestProxy.Tests public class SanitizerTests { public OAuthResponseSanitizer OAuthResponseSanitizer = new OAuthResponseSanitizer(); + private NullLoggerFactory _nullLogger = new NullLoggerFactory(); public string lookaheadReplaceRegex = @"[a-z]+(?=\.(?:table|blob|queue)\.core\.windows\.net)"; public string capturingGroupReplaceRegex = @"https\:\/\/(?[a-z]+)\.(?:table|blob|queue)\.core\.windows\.net"; @@ -48,6 +55,56 @@ public void OauthResponseSanitizerNotAggressive() Assert.Equal(expectedCount, session.Session.Entries.Count); } + [Theory] + [InlineData("")] + public void EntryOmissionSanitizerNoOpsOnNonMatch() + { + // confirm that we silently let stuff pass when not matching? + } + + [Theory] + [InlineData("input body")] + public void EntryOmissionSanitizerCorrectlySanitizes() + { + // confirm that we omit sections that match + } + + [Theory] + [InlineData("input body")] + public async Task CanCreateOverAPI(string inputObject) + { + // confirm that we can handle a request that creates the sanitizer dynamically + + //RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + //var httpContext = new DefaultHttpContext(); + //httpContext.Request.Headers["x-abstraction-identifier"] = "CustomDefaultMatcher"; + //httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody("{ \"excludedHeaders\": \"Content-Type,Content-Length\", \"ignoredHeaders\": \"Connection\", \"compareBodies\": false, \"ignoredQueryParameters\": \"api-version,location\" }"); + + //// content length must be set for the body to be parsed in SetMatcher + //httpContext.Request.ContentLength = httpContext.Request.Body.Length; + + //var controller = new Admin(testRecordingHandler, _nullLogger) + //{ + // ControllerContext = new ControllerContext() + // { + // HttpContext = httpContext + // } + //}; + //await controller.SetMatcher(); + //var matcher = testRecordingHandler.Matcher; + //Assert.True(matcher is CustomDefaultMatcher); + + //var compareBodies = (bool)typeof(RecordMatcher).GetField("_compareBodies", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(matcher); + //Assert.False(compareBodies); + + //Assert.Contains("Content-Type", matcher.ExcludeHeaders); + //Assert.Contains("Content-Length", matcher.ExcludeHeaders); + //Assert.Contains("Connection", matcher.IgnoredHeaders); + //Assert.Contains("api-version", matcher.IgnoredQueryParameters); + //Assert.Contains("location", matcher.IgnoredQueryParameters); + } + + [Fact] public void UriRegexSanitizerReplacesTableName() { diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/StringSanitizer.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/StringSanitizer.cs index a0d487f5f05..855ce62f70d 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/StringSanitizer.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/StringSanitizer.cs @@ -24,6 +24,21 @@ public static void ConfirmValidRegex(string regex) } } + /// + /// Quick and easy abstraction for checking regex validity. Passing null explicitly will result in a True return. + /// + /// A regular expression. + public static Regex GetRegex(string regex) + { + try + { + return new Regex(regex); + } + catch (Exception e) + { + throw new HttpException(HttpStatusCode.BadRequest, $"Expression of value {regex} does not successfully compile. Failure Details: {e.Message}"); + } + } /// /// General purpose string replacement. Simple abstraction of string.Replace(). diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs new file mode 100644 index 00000000000..70ecc9f4faf --- /dev/null +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs @@ -0,0 +1,86 @@ +using Azure.Sdk.Tools.TestProxy.Common; +using Azure.Sdk.Tools.TestProxy.Common.Exceptions; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Azure.Sdk.Tools.TestProxy.Sanitizers +{ + /// + /// This sanitizer applies at the session level, just before saving a recording to disk. + /// + /// It cleans out all request/response pairs that that match the defined settings. A match against URI, Header, or Body regex will result in the entire RecordEntry being recorded from the recording. + /// + public class RegexEntrySanitizer : RecordedTestSanitizer + { + private Regex rx; + private string target; + private string[] validValues = new string[] { "uri", "header", "body" }; + + public string ValidValues + { + get { return string.Join(", ", validValues.Select(x => "\"" + x + "\"")); } + } + + /// + /// During sanitization, each RecordEntry within a session is checked against a target (URI, Header, Body) and a regex. If there is any match within the request, the whole request/response pair is omitted from the recording. + /// + /// Possible values are [ "URI", "Header", "Body"]. Only requests with text-like body values will be checked when targeting "Body". The value is NOT case-sensitive. + /// During sanitization, any entry where the 'target' is matched by the regex will be fully omitted. Request/Reponse both. + public RegexEntrySanitizer(string target, string regex) + { + + target = target.ToLowerInvariant(); + if (!validValues.Contains(target)) + { + throw new HttpException(System.Net.HttpStatusCode.BadRequest, $"When defining which section of a request the regex should target, only values [ {ValidValues} ] are valid."); + } + + rx = StringSanitizer.GetRegex(regex); + } + + public bool CheckMatch(RecordEntry x) + { + bool result = false; + switch (target) + { + case "uri": + result = rx.IsMatch(x.RequestUri); + break; + case "header": + foreach (var headerKey in x.Request.Headers.Keys) + { + // Accessing 0th key safe due to the fact that we force header values in without splitting them on ;. + // We do this because letting .NET split and then reassemble header values introduces a space into the header itself + // Ex: "application/json;odata=minimalmetadata" with .NET default header parsing becomes "application/json; odata=minimalmetadata" + // Given this breaks signature verification, we have to avoid it. + var originalValue = x.Request.Headers[headerKey][0]; + + if (rx.IsMatch(originalValue)) + { + result = true; + } + } + break; + case "body": + if (x.Request.TryGetBodyAsText(out string text)) + { + result = rx.IsMatch(text); + } + else + { + result = false; + } + break; + default: + throw new HttpException(System.Net.HttpStatusCode.BadRequest, $"The RegexEntrySanitizer can only match against a target of [ {ValidValues} ]."); + } + + return result; + } + + public override void Sanitize(RecordSession session) + { + session.Entries.RemoveAll(x => CheckMatch(x)); + } + } +} From eca1c769265df66f6864e5dea1648c6dd140d589 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 1 Aug 2022 18:09:00 -0700 Subject: [PATCH 2/4] resolve most tests. need to work through the full code path for successful sanitization --- .../SanitizerTests.cs | 123 ++++++++++++------ .../Sanitizers/RegexEntrySanitizer.cs | 23 ++-- 2 files changed, 94 insertions(+), 52 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs index 52e22f54d68..29df06378dd 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs @@ -1,13 +1,15 @@ using Azure.Sdk.Tools.TestProxy.Common; -using Azure.Sdk.Tools.TestProxy.Matchers; +using Azure.Sdk.Tools.TestProxy.Common.Exceptions; using Azure.Sdk.Tools.TestProxy.Sanitizers; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging.Abstractions; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Xunit; @@ -18,6 +20,8 @@ public class SanitizerTests public OAuthResponseSanitizer OAuthResponseSanitizer = new OAuthResponseSanitizer(); private NullLoggerFactory _nullLogger = new NullLoggerFactory(); + + public string oauthRegex = "\"/oauth2(?:/v2.0)?/token\""; public string lookaheadReplaceRegex = @"[a-z]+(?=\.(?:table|blob|queue)\.core\.windows\.net)"; public string capturingGroupReplaceRegex = @"https\:\/\/(?[a-z]+)\.(?:table|blob|queue)\.core\.windows\.net"; public string scopeClean = @"scope\=(?[^&]*)"; @@ -56,52 +60,93 @@ public void OauthResponseSanitizerNotAggressive() } [Theory] - [InlineData("")] - public void EntryOmissionSanitizerNoOpsOnNonMatch() + [InlineData("uri", "\"/oauth2(?:/v2.0)?/token\"")] + [InlineData("body", "\"/oauth2(?:/v2.0)?/token\"")] + [InlineData("header", "\"/oauth2(?:/v2.0)?/token\"")] + public void RegexEntrySanitizerNoOpsOnNonMatch(string target, string regex) { - // confirm that we silently let stuff pass when not matching? + var session = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json"); + var sanitizer = new RegexEntrySanitizer(target, regex); + var expectedCount = session.Session.Entries.Count; + + session.Session.Sanitize(sanitizer); + + Assert.Equal(expectedCount, session.Session.Entries.Count); } [Theory] - [InlineData("input body")] - public void EntryOmissionSanitizerCorrectlySanitizes() + [InlineData("uri", "", 1)] + [InlineData("body", "", 1)] + [InlineData("headers", "", 1)] + public void RegexEntrySanitizerCorrectlySanitizes(string target, string regex, int endCount) { - // confirm that we omit sections that match + //var session = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json"); + //var sanitizer = new RegexEntrySanitizer(target, regex); + //var expectedCount = session.Session.Entries.Count; + + //session.Session.Sanitize(sanitizer); + + //Assert.Equal(endCount, session.Session.Entries.Count); + } + + [Fact] + public void RegexEntrySanitizerCorrectlySanitizesSpecific() + { + var session = TestHelpers.LoadRecordSession("Test.RecordEntries/response_with_xml_body.json"); + var sanitizer = new RegexEntrySanitizer("header", "b24f75a9-b830-11eb-b949-10e7c6392c5a"); + var expectedCount = session.Session.Entries.Count; + + session.Session.Sanitize(sanitizer); + + Assert.Equal(2, session.Session.Entries.Count); + Assert.Equal("b25bf92a-b830-11eb-947a-10e7c6392c5a", session.Session.Entries[0].Request.Headers["x-ms-client-request-id"][0].ToString()); + } + + [Theory] + [InlineData("wrong_name", "", "When defining which section of a request the regex should target, only values")] + [InlineData("", ".+", "When defining which section of a request the regex should target, only values")] + [InlineData("uri", "\"[\"", "Expression of value")] + public void RegexEntrySanitizerThrowsProperExceptions(string target, string regex, string exceptionMessage) + { + var assertion = Assert.Throws( + () => new RegexEntrySanitizer(target, regex) + ); + + Assert.Contains(exceptionMessage, assertion.Message); } [Theory] - [InlineData("input body")] - public async Task CanCreateOverAPI(string inputObject) - { - // confirm that we can handle a request that creates the sanitizer dynamically - - //RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); - //var httpContext = new DefaultHttpContext(); - //httpContext.Request.Headers["x-abstraction-identifier"] = "CustomDefaultMatcher"; - //httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody("{ \"excludedHeaders\": \"Content-Type,Content-Length\", \"ignoredHeaders\": \"Connection\", \"compareBodies\": false, \"ignoredQueryParameters\": \"api-version,location\" }"); - - //// content length must be set for the body to be parsed in SetMatcher - //httpContext.Request.ContentLength = httpContext.Request.Body.Length; - - //var controller = new Admin(testRecordingHandler, _nullLogger) - //{ - // ControllerContext = new ControllerContext() - // { - // HttpContext = httpContext - // } - //}; - //await controller.SetMatcher(); - //var matcher = testRecordingHandler.Matcher; - //Assert.True(matcher is CustomDefaultMatcher); - - //var compareBodies = (bool)typeof(RecordMatcher).GetField("_compareBodies", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(matcher); - //Assert.False(compareBodies); - - //Assert.Contains("Content-Type", matcher.ExcludeHeaders); - //Assert.Contains("Content-Length", matcher.ExcludeHeaders); - //Assert.Contains("Connection", matcher.IgnoredHeaders); - //Assert.Contains("api-version", matcher.IgnoredQueryParameters); - //Assert.Contains("location", matcher.IgnoredQueryParameters); + [InlineData("{ \"target\": \"URI\", \"regex\": \"/oauth2(?:/v2.0)?/token\" }")] + [InlineData("{ \"target\": \"uRi\", \"regex\": \"/login\\\\.microsoftonline.com\" }")] + [InlineData("{ \"target\": \"bodY\", \"regex\": \"/oauth2(?:/v2.0)?/token\" }")] + [InlineData("{ \"target\": \"HEADER\", \"regex\": \"/login\\\\.microsoftonline.com\" }")] + public async Task RegexEntrySanitizerCreatesOverAPI(string body) + { + + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + testRecordingHandler.Sanitizers.Clear(); + var httpContext = new DefaultHttpContext(); + httpContext.Request.Headers["x-abstraction-identifier"] = "RegexEntrySanitizer"; + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(body); + + // content length must be set for the body to be parsed in SetMatcher + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + + await controller.AddSanitizer(); + var sanitizer = testRecordingHandler.Sanitizers[0]; + Assert.True(sanitizer is RegexEntrySanitizer); + + + var sanitizerTarget = (string)typeof(RegexEntrySanitizer).GetField("section", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(sanitizer); + var regex = (Regex)typeof(RegexEntrySanitizer).GetField("rx", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(sanitizer); } diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs index 70ecc9f4faf..9d2035818a3 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs @@ -13,7 +13,7 @@ namespace Azure.Sdk.Tools.TestProxy.Sanitizers public class RegexEntrySanitizer : RecordedTestSanitizer { private Regex rx; - private string target; + private string section; private string[] validValues = new string[] { "uri", "header", "body" }; public string ValidValues @@ -28,9 +28,9 @@ public string ValidValues /// During sanitization, any entry where the 'target' is matched by the regex will be fully omitted. Request/Reponse both. public RegexEntrySanitizer(string target, string regex) { - - target = target.ToLowerInvariant(); - if (!validValues.Contains(target)) + section = target.ToLowerInvariant(); + + if (!validValues.Contains(section)) { throw new HttpException(System.Net.HttpStatusCode.BadRequest, $"When defining which section of a request the regex should target, only values [ {ValidValues} ] are valid."); } @@ -40,12 +40,10 @@ public RegexEntrySanitizer(string target, string regex) public bool CheckMatch(RecordEntry x) { - bool result = false; - switch (target) + switch (section) { case "uri": - result = rx.IsMatch(x.RequestUri); - break; + return rx.IsMatch(x.RequestUri); case "header": foreach (var headerKey in x.Request.Headers.Keys) { @@ -57,25 +55,24 @@ public bool CheckMatch(RecordEntry x) if (rx.IsMatch(originalValue)) { - result = true; + return true; } } break; case "body": if (x.Request.TryGetBodyAsText(out string text)) { - result = rx.IsMatch(text); + return rx.IsMatch(text); } else { - result = false; + return false; } - break; default: throw new HttpException(System.Net.HttpStatusCode.BadRequest, $"The RegexEntrySanitizer can only match against a target of [ {ValidValues} ]."); } - return result; + return false; } public override void Sanitize(RecordSession session) From 7ca2ab32a6ed11ab43eb9e0ce43eae35520b9825 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 2 Aug 2022 13:17:40 -0700 Subject: [PATCH 3/4] tests exercising sanitization now working --- .../SanitizerTests.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs index 29df06378dd..ca9d1688f42 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs @@ -75,18 +75,19 @@ public void RegexEntrySanitizerNoOpsOnNonMatch(string target, string regex) } [Theory] - [InlineData("uri", "", 1)] - [InlineData("body", "", 1)] - [InlineData("headers", "", 1)] + [InlineData("body", "(listtable09bf2a3d|listtable19bf2a3d)", 9)] + [InlineData("uri", "fakeazsdktestaccount", 0)] + [InlineData("body", "listtable09bf2a3d", 10)] + [InlineData("header", "a50f2f9c-b830-11eb-b8c8-10e7c6392c5a", 10)] public void RegexEntrySanitizerCorrectlySanitizes(string target, string regex, int endCount) { - //var session = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json"); - //var sanitizer = new RegexEntrySanitizer(target, regex); - //var expectedCount = session.Session.Entries.Count; + var session = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json"); + var sanitizer = new RegexEntrySanitizer(target, regex); + var expectedCount = session.Session.Entries.Count; - //session.Session.Sanitize(sanitizer); + session.Session.Sanitize(sanitizer); - //Assert.Equal(endCount, session.Session.Entries.Count); + Assert.Equal(endCount, session.Session.Entries.Count); } [Fact] From 2c6774a9ad57f7e7568dde7539cf962cc2f4bd01 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Thu, 4 Aug 2022 13:30:02 -0700 Subject: [PATCH 4/4] Update tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs --- .../Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs index 9d2035818a3..885866a31e4 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/RegexEntrySanitizer.cs @@ -8,7 +8,7 @@ namespace Azure.Sdk.Tools.TestProxy.Sanitizers /// /// This sanitizer applies at the session level, just before saving a recording to disk. /// - /// It cleans out all request/response pairs that that match the defined settings. A match against URI, Header, or Body regex will result in the entire RecordEntry being recorded from the recording. + /// It cleans out all request/response pairs that that match the defined settings. A match against URI, Header, or Body regex will result in the entire RecordEntry being omit from the recording. /// public class RegexEntrySanitizer : RecordedTestSanitizer {