Adding String Literal Sanitizers

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs

@@ -732,5 +732,64 @@ public async Task TestAddTransformWithBadUriRegexCondition(string body, string e
             Assert.Empty(testRecordingHandler.Transforms);
             Assert.Contains(errorText, assertion.Message);
         }
+
+        [Fact]
+        public async Task AddSanitizerThrowsOnMissingRequiredArgument()
+        {
+            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
+            var httpContext = new DefaultHttpContext();
+            httpContext.Request.Headers["x-abstraction-identifier"] = "GeneralStringSanitizer";
+            httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody("{\"value\":\"replacementValue\"}");
+            httpContext.Request.ContentLength = 33;
+
+            var controller = new Admin(testRecordingHandler)
+            {
+                ControllerContext = new ControllerContext()
+                {
+                    HttpContext = httpContext
+                }
+            };
+
+            testRecordingHandler.Sanitizers.Clear();
+
+            var assertion = await Assert.ThrowsAsync<HttpException>(
+               async () => await controller.AddSanitizer()
+            );
+
+            Assert.True(assertion.StatusCode.Equals(HttpStatusCode.BadRequest));
+            Assert.Contains("Required parameter key System.String target was not found in the request body.", assertion.Message);
+        }
+
+        [Fact]
+        public async Task AddSanitizerContinuesWithTwoRequiredParams()
+        {
+            var targetKey = "Content-Type";
+            var targetString = "application/javascript";
+
+            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
+            var httpContext = new DefaultHttpContext();
+            httpContext.Request.Headers["x-abstraction-identifier"] = "HeaderStringSanitizer";
+            httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody("{\"target\":\"" + targetString + "\", \"key\":\"" + targetKey + "\"}");
+            httpContext.Request.ContentLength = 79;
+
+            var controller = new Admin(testRecordingHandler)
+            {
+                ControllerContext = new ControllerContext()
+                {
+                    HttpContext = httpContext
+                }
+            };
+
+            testRecordingHandler.Sanitizers.Clear();
+            await controller.AddSanitizer();
+
+            var addedSanitizer = testRecordingHandler.Sanitizers.First();
+            Assert.True(addedSanitizer is HeaderStringSanitizer);
+
+            var actualTargetString = (string)typeof(HeaderStringSanitizer).GetField("_targetValue", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(addedSanitizer);
+            var actualTargetKey = (string)typeof(HeaderStringSanitizer).GetField("_targetKey", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(addedSanitizer);
+            Assert.Equal(targetKey, actualTargetKey);
+            Assert.Equal(targetString, actualTargetString);
+        }
     }
 }

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs

@@ -1,5 +1,6 @@
 using Azure.Sdk.Tools.TestProxy.Common;
 using Azure.Sdk.Tools.TestProxy.Sanitizers;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using Xunit;
@@ -453,5 +454,191 @@ public void ConditionalSanitizeUriRegexProperlySkips()
             session.Session.Sanitize(removeHeadersSanitizer);
             Assert.DoesNotContain<bool>(false, session.Session.Entries.Select(x => x.Request.Headers.ContainsKey(targetHeader)));
         }
+
+        [Fact]
+        public void GenStringSanitizerAppliesForMultipleComponents()
+        {
+            var session = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json");
+            var targetString = "listtable09bf2a3d";
+            var replacementString = "<thistablehasbeenreplaced!>";
+            var targetEntry = session.Session.Entries[0];
+
+            var originalRequestBody = Encoding.UTF8.GetString(targetEntry.Request.Body);
+            var originalResponseBody = Encoding.UTF8.GetString(targetEntry.Response.Body);
+            var originalLocation = targetEntry.Response.Headers["Location"].First().ToString();
+
+            var sanitizer = new GeneralStringSanitizer(targetString, replacementString);
+            session.Session.Sanitize(sanitizer);
+
+            var resultRequestBody = Encoding.UTF8.GetString(targetEntry.Request.Body);
+            var resultResponseBody = Encoding.UTF8.GetString(targetEntry.Response.Body);
+            var resultLocation = targetEntry.Response.Headers["Location"].First().ToString();
+
+
+            // request body
+            Assert.NotEqual(originalRequestBody, resultRequestBody);
+            Assert.DoesNotContain(targetString, resultRequestBody);
+            Assert.Contains(replacementString, resultRequestBody);
+
+            // result body
+            Assert.NotEqual(originalResponseBody, resultResponseBody);
+            Assert.DoesNotContain(targetString, resultResponseBody);
+            Assert.Contains(replacementString, resultResponseBody);
+
+            // uri
+            Assert.NotEqual(originalLocation, resultLocation);
+            Assert.DoesNotContain(targetString, resultLocation);
+            Assert.Contains(replacementString, resultLocation);
+        }
+
+        [Fact]
+        public void GenStringSanitizerQuietExitForAllHttpComponents()
+        {
+            var session = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json");
+            var untouchedSession = TestHelpers.LoadRecordSession("Test.RecordEntries/post_delete_get_content.json");
+
+            var targetString = ".*";
+            var replacementString = "<thistablehasbeenreplaced!>";
+            var targetEntry = session.Session.Entries[0];
+            var targetUntouchedEntry = untouchedSession.Session.Entries[0];
+            var matcher = new RecordMatcher();
+
+            var sanitizer = new GeneralStringSanitizer(targetString, replacementString);
+            session.Session.Sanitize(sanitizer);
+
+            var resultRequestBody = Encoding.UTF8.GetString(targetEntry.Request.Body);
+            var resultResponseBody = Encoding.UTF8.GetString(targetEntry.Response.Body);
+            var resultLocation = targetEntry.Response.Headers["Location"].First().ToString();
+
+            Assert.DoesNotContain(targetString, resultRequestBody);
+            Assert.DoesNotContain(targetString, resultResponseBody);
+            Assert.DoesNotContain(targetString, resultLocation);
+
+            Assert.Equal(0, matcher.CompareHeaderDictionaries(targetUntouchedEntry.Request.Headers, targetEntry.Request.Headers, new HashSet<string>(), new HashSet<string>()));
+            Assert.Equal(0, matcher.CompareHeaderDictionaries(targetUntouchedEntry.Response.Headers, targetEntry.Response.Headers, new HashSet<string>(), new HashSet<string>()));
+            Assert.Equal(0, matcher.CompareBodies(targetUntouchedEntry.Request.Body, targetEntry.Request.Body));
+            Assert.Equal(0, matcher.CompareBodies(targetUntouchedEntry.Response.Body, targetEntry.Response.Body));
+            Assert.Equal(targetUntouchedEntry.RequestUri, targetEntry.RequestUri);
+        }
+
+        [Theory]
+        [InlineData("Accept-Encoding", ",", "<comma>", "Test.RecordEntries/post_delete_get_content.json")]
+        [InlineData("Accept", "*/*", "<starslashstar>", "Test.RecordEntries/oauth_request_with_variables.json")]
+        [InlineData("User-Agent", ".19041-SP0", "<useragent>", "Test.RecordEntries/post_delete_get_content.json")]
+        public void HeaderStringSanitizerApplies(string targetKey, string targetValue, string replacementValue, string recordingFile)
+        {
+            var session = TestHelpers.LoadRecordSession(recordingFile);
+            var targetEntry = session.Session.Entries[0];
+            var originalHeaderValue = targetEntry.Request.Headers[targetKey].First().ToString();
+
+            var sanitizer = new HeaderStringSanitizer(targetKey, targetValue, value: replacementValue);
+            session.Session.Sanitize(sanitizer);
+
+            var resultHeaderValue = targetEntry.Request.Headers[targetKey].First().ToString();
+
+            Assert.NotEqual(resultHeaderValue, originalHeaderValue);
+            Assert.Contains(replacementValue, resultHeaderValue);
+            Assert.DoesNotContain(targetValue, resultHeaderValue);
+        }
+
+        [Theory]
+        [InlineData("DataServiceVersion", "application/json", "<replacedString>", "Test.RecordEntries/post_delete_get_content.json")]
+        public void HeaderStringSanitizerQuietlyExits(string targetKey, string targetValue, string replacementValue, string recordingFile)
+        {
+            var session = TestHelpers.LoadRecordSession(recordingFile);
+            var untouchedSession = TestHelpers.LoadRecordSession(recordingFile);
+            var targetUntouchedEntry = untouchedSession.Session.Entries[0];
+            var targetEntry = session.Session.Entries[0];
+            var matcher = new RecordMatcher();
+
+            var sanitizer = new HeaderStringSanitizer(targetKey, targetValue, value: replacementValue);
+            session.Session.Sanitize(sanitizer);
+
+            Assert.Equal(0, matcher.CompareHeaderDictionaries(targetUntouchedEntry.Request.Headers, targetEntry.Request.Headers, new HashSet<string>(), new HashSet<string>()));
+            Assert.Equal(0, matcher.CompareHeaderDictionaries(targetUntouchedEntry.Response.Headers, targetEntry.Response.Headers, new HashSet<string>(), new HashSet<string>()));
+        }
+
+        [Theory]
+        [InlineData("listtable09bf2a3d", "<replacedtablename>", "Test.RecordEntries/post_delete_get_content.json")]
+        [InlineData("%20profile%20offline", "<profilereplaced>", "Test.RecordEntries/oauth_request.json")]
+        [InlineData("|,&x-client-last-telemetry=2|0|", "<client>", "Test.RecordEntries/oauth_request.json")]
+        [InlineData("}", "<bracket>", "Test.RecordEntries/response_with_null_secrets.json")]
+        public void BodyStringSanitizerApplies(string targetValue, string replacementValue, string recordingFile)
+        {
+            var session = TestHelpers.LoadRecordSession(recordingFile);
+            var targetEntry = session.Session.Entries[0];
+            var originalBodyValue = Encoding.UTF8.GetString(targetEntry.Request.Body);
+
+            var sanitizer = new BodyStringSanitizer(targetValue, value: replacementValue);
+            session.Session.Sanitize(sanitizer);
+
+            var resultBodyValue = Encoding.UTF8.GetString(targetEntry.Request.Body);
+
+            Assert.NotEqual(originalBodyValue, resultBodyValue);
+            Assert.Contains(replacementValue, resultBodyValue);
+            Assert.DoesNotContain(targetValue, resultBodyValue);
+        }
+
+        [Theory]
+        [InlineData("TableNames", "<tablename>", "Test.RecordEntries/response_with_null_secrets.json")]
+        [InlineData("d2270777-c002-0072-313d-4ce19f000000", "<targetId>", "Test.RecordEntries/response_with_null_secrets.json")]
+        [InlineData(".19041-SP0", "<useragent>", "Test.RecordEntries/response_with_null_secrets.json")]
+        public void BodyStringSanitizerQuietlyExits(string targetValue, string replacementValue, string recordingFile)
+        {
+            var session = TestHelpers.LoadRecordSession(recordingFile);
+            var untouchedSession = TestHelpers.LoadRecordSession(recordingFile);
+            var targetEntry = session.Session.Entries[0];
+            var originalBodyValue = Encoding.UTF8.GetString(targetEntry.Request.Body);
+            var targetUntouchedEntry = untouchedSession.Session.Entries[0];
+            var matcher = new RecordMatcher();
+
+            var sanitizer = new BodyStringSanitizer(targetValue, value: replacementValue);
+            session.Session.Sanitize(sanitizer);
+
+            var resultBodyValue = Encoding.UTF8.GetString(targetEntry.Request.Body);
+            Assert.Equal(0, matcher.CompareBodies(targetUntouchedEntry.Request.Body, targetEntry.Request.Body));
+            Assert.Equal(0, matcher.CompareBodies(targetUntouchedEntry.Response.Body, targetEntry.Response.Body));
+        }
+
+        [Theory]
+        [InlineData("/v2.0/", "<oath-v2>", "Test.RecordEntries/oauth_request.json")]
+        [InlineData("https://management.azure.com/subscriptions/12345678-1234-1234-5678-123456789010", "<partofpath>", "Test.RecordEntries/request_with_subscriptionid.json")]
+        [InlineData("?api-version=2019-05-01", "<api-version>", "Test.RecordEntries/request_with_subscriptionid.json")]
+        public void UriStringSanitizerApplies(string targetValue, string replacementValue, string recordingFile)
+        {
+            var session = TestHelpers.LoadRecordSession(recordingFile);
+            var untouchedSession = TestHelpers.LoadRecordSession(recordingFile);
+
+            var targetEntry = session.Session.Entries[0];
+            var targetUntouchedEntry = untouchedSession.Session.Entries[0];
+            var matcher = new RecordMatcher();
+
+            var sanitizer = new UriStringSanitizer(targetValue, replacementValue);
+            session.Session.Sanitize(sanitizer);
+
+            var originalUriValue = targetUntouchedEntry.RequestUri.ToString();
+            var resultUriValue = targetEntry.RequestUri.ToString();
+
+            Assert.NotEqual(originalUriValue, resultUriValue);
+            Assert.Contains(replacementValue, resultUriValue);
+            Assert.DoesNotContain(targetValue, resultUriValue);
+        }
+
+        [Theory]
+        [InlineData("fakeazsdktestaccount2", "<replacementValue!", "Test.RecordEntries/post_delete_get_content.json")]
+        public void UriStringSanitizerQuietlyExits(string targetValue, string replacementValue, string targetFile)
+        {
+            var session = TestHelpers.LoadRecordSession(targetFile);
+            var untouchedSession = TestHelpers.LoadRecordSession(targetFile);
+
+            var targetEntry = session.Session.Entries[0];
+            var targetUntouchedEntry = untouchedSession.Session.Entries[0];
+            var matcher = new RecordMatcher();
+
+            var sanitizer = new UriStringSanitizer(targetValue, replacementValue);
+            session.Session.Sanitize(sanitizer);
+
+            Assert.Equal(targetUntouchedEntry.RequestUri, targetEntry.RequestUri);
+        }
     }
 }

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/StringSanitizer.cs

@@ -25,11 +25,23 @@ public static void ConfirmValidRegex(string regex)
         }
 
 
+        /// <summary>
+        /// General purpose string replacement. Simple abstraction of string.Replace().
+        /// </summary>
+        /// <param name="inputValue">The name of the header we're operating against.</param>
+        /// <param name="targetValue">The substitution or whole new header value, depending on "regex" setting.</param>
+        /// <param name="replacementValue">The substitution or new header value, depending on the "targetValue" setting.</param>
+        /// <returns>An updated value of the input string, with replacement operations completed if necessary.</returns>
+        public static string ReplaceValue(string inputValue, string targetValue, string replacementValue)
+        {
+            return inputValue.Replace(targetValue, replacementValue);
+        }
+
         /// <summary>
         /// General purpose string replacement/subsitution given a set of inputs. Used in many regex substitution sanitizers.
         /// </summary>
         /// <param name="inputValue">The name of the header we're operating against.</param>
-        /// <param name="replacementValue">The substitution or whole new header value, depending on "regex" setting.</param>
+        /// <param name="replacementValue">The substitution or whole input value, depending on "regex" setting.</param>
         /// <param name="regex">A regex. Can be defined as a simple regex replace OR if groupName is set, a subsitution operation.</param>
         /// <param name="groupName">The capture group that needs to be operated upon. Do not set if you're invoking a simple replacement operation. 
         /// Note that with this implementation, you can refer to a numbered group if you didn't name it, EG: '0'.</param>

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/BodyKeySanitizer.cs

@@ -7,7 +7,7 @@
 namespace Azure.Sdk.Tools.TestProxy.Sanitizers
 {
     /// <summary>
-    /// This sanitizer operates on a RecordSession entry and applies itself to the Request and Response bodies contained therein. It ONLY operates on the request/response bodies. Not header or URIs.
+    /// This sanitizer operates on a RecordSession entry and applies regex replacement to the Request and Response bodies contained therein. It ONLY operates on the request/response bodies. Not header or URIs.
     /// </summary>
     public class BodyKeySanitizer : RecordedTestSanitizer
     {

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/BodyRegexSanitizer.cs

@@ -5,7 +5,7 @@
 namespace Azure.Sdk.Tools.TestProxy.Sanitizers
 {
     /// <summary>
-    /// This sanitizer operates on a RecordSession entry and applies itself to the Request and Response bodies contained therein. It ONLY operates on the request/response bodies. Not header or URIs.
+    /// This sanitizer operates on a RecordSession entry and applies regex replacement to the Request and Response bodies contained therein. It ONLY operates on the request/response bodies. Not header or URIs.
     /// </summary>
     public class BodyRegexSanitizer : RecordedTestSanitizer
     {

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Sanitizers/BodyStringSanitizer.cs

@@ -0,0 +1,43 @@
+using Azure.Sdk.Tools.TestProxy.Common;
+using System;
+using System.Text;
+
+namespace Azure.Sdk.Tools.TestProxy.Sanitizers
+{
+    /// <summary>
+    /// This sanitizer operates on a RecordSession entry and applies value replacement to the Request and Response bodies contained therein. It ONLY operates on the request/response bodies. Not header or URIs.
+    /// </summary>
+    public class BodyStringSanitizer : RecordedTestSanitizer
+    {
+        private string _newValue;
+        private string _targetValue;
+
+        /// <summary>
+        /// This sanitizer offers regex replace within a returned body. Specifically, this means regex applying to the raw JSON. If you are attempting to simply 
+        /// replace a specific key, the BodyKeySanitizer is probably the way to go. Regardless, there are examples present in SanitizerTests.cs.
+        /// </summary>
+        /// <param name="target">A target string. This could contain special regex characters like "?()+*" but they will be treated as a literal.</param>
+        /// <param name="value">The substitution value.</param>
+        /// <param name="condition">
+        /// A condition that dictates when this sanitizer applies to a request/response pair. The content of this key should be a JSON object that contains configuration keys. 
+        /// Currently, that only includes the key "uriRegex". This translates to an object that looks like '{ "uriRegex": "when this regex matches, apply the sanitizer" }'. Defaults to "apply always."
+        /// </param>
+        public BodyStringSanitizer(string target, string value = "Sanitized", ApplyCondition condition = null)
+        {
+            _targetValue = target;
+            _newValue = value;
+            Condition = condition;
+        }
+
+        public override string SanitizeTextBody(string contentType, string body)
+        {
+            return StringSanitizer.ReplaceValue(inputValue: body, targetValue: _targetValue, replacementValue: _newValue);
+        }
+
+
+        public override byte[] SanitizeBody(string contentType, byte[] body)
+        {
+            return Encoding.UTF8.GetBytes(StringSanitizer.ReplaceValue(inputValue: Encoding.UTF8.GetString(body), targetValue: _targetValue, replacementValue: _newValue));
+        }
+    }
+}