From 8282547dd79fb1163db46ca334bbbc0e2ce96856 Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Wed, 3 Jun 2020 14:51:03 -0700 Subject: [PATCH 1/4] Roundtip response encoding --- .../src/RecordEntry.cs | 22 ++++++++++++------- .../src/RecordedTestBase.cs | 4 ++++ .../src/TestRecording.cs | 7 ++++-- .../Azure.Core/tests/RecordSessionTests.cs | 1 + 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs b/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs index 7d9c38527007..e72ea3cde186 100644 --- a/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs +++ b/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.Encodings.Web; using System.Text.Json; using Azure.Core.Pipeline; @@ -13,6 +14,11 @@ namespace Azure.Core.TestFramework { public class RecordEntry { + private static JsonWriterOptions _requestWriterOptions = new JsonWriterOptions(); + private static JsonWriterOptions _responseWriterOptions = new JsonWriterOptions() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; public RecordEntryMessage Request { get; } = new RecordEntryMessage(); public RecordEntryMessage Response { get; } = new RecordEntryMessage(); @@ -51,7 +57,7 @@ public static RecordEntry Deserialize(JsonElement element) if (element.TryGetProperty("RequestBody", out property)) { - record.Request.Body = DeserializeBody(record.Request.Headers, property); + record.Request.Body = DeserializeBody(record.Request.Headers, property, _requestWriterOptions); } if (element.TryGetProperty(nameof(StatusCode), out property) && @@ -67,13 +73,13 @@ public static RecordEntry Deserialize(JsonElement element) if (element.TryGetProperty("ResponseBody", out property)) { - record.Response.Body = DeserializeBody(record.Response.Headers, property); + record.Response.Body = DeserializeBody(record.Response.Headers, property, _responseWriterOptions); } return record; } - private static byte[] DeserializeBody(IDictionary headers, in JsonElement property) + private static byte[] DeserializeBody(IDictionary headers, in JsonElement property, JsonWriterOptions writerOptions) { if (property.ValueKind == JsonValueKind.Null) { @@ -85,7 +91,7 @@ private static byte[] DeserializeBody(IDictionary headers, in if (property.ValueKind == JsonValueKind.Object) { using var memoryStream = new MemoryStream(); - using var writer = new Utf8JsonWriter(memoryStream); + using var writer = new Utf8JsonWriter(memoryStream, writerOptions); property.WriteTo(writer); writer.Flush(); return memoryStream.ToArray(); @@ -146,7 +152,7 @@ public void Serialize(Utf8JsonWriter jsonWriter) SerializeHeaders(jsonWriter, Request.Headers); jsonWriter.WriteEndObject(); - SerializeBody(jsonWriter, "RequestBody", Request.Body, Request.Headers); + SerializeBody(jsonWriter, "RequestBody", Request.Body, Request.Headers, _requestWriterOptions); jsonWriter.WriteNumber(nameof(StatusCode), StatusCode); @@ -154,11 +160,11 @@ public void Serialize(Utf8JsonWriter jsonWriter) SerializeHeaders(jsonWriter, Response.Headers); jsonWriter.WriteEndObject(); - SerializeBody(jsonWriter, "ResponseBody", Response.Body, Response.Headers); + SerializeBody(jsonWriter, "ResponseBody", Response.Body, Response.Headers, _responseWriterOptions); jsonWriter.WriteEndObject(); } - private void SerializeBody(Utf8JsonWriter jsonWriter, string name, byte[] requestBody, IDictionary headers) + private void SerializeBody(Utf8JsonWriter jsonWriter, string name, byte[] requestBody, IDictionary headers, JsonWriterOptions writerOptions) { if (requestBody == null) { @@ -186,7 +192,7 @@ private void SerializeBody(Utf8JsonWriter jsonWriter, string name, byte[] reques // fallback to generic string writing var memoryStream = new MemoryStream(); // Settings of this writer should be in sync with the one used in deserialiation - using (var reformattedWriter = new Utf8JsonWriter(memoryStream)) + using (var reformattedWriter = new Utf8JsonWriter(memoryStream, writerOptions)) { document.RootElement.WriteTo(reformattedWriter); } diff --git a/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs b/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs index 6d54caed723f..c0072a671460 100644 --- a/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs +++ b/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; +using System.Text.Json; using NUnit.Framework; namespace Azure.Core.TestFramework @@ -29,6 +30,9 @@ public abstract class RecordedTestBase : ClientTestBase public bool SaveDebugRecordingsOnFailure { get; set; } = false; #endif + public JsonWriterOptions RequestSerializationOptions { get; set; } = new JsonWriterOptions(); + public JsonWriterOptions ResponseSerializationOptions { get; set; } = new JsonWriterOptions(); + protected RecordedTestBase(bool isAsync) : this(isAsync, RecordedTestUtilities.GetModeFromEnvironment()) { } diff --git a/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs b/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs index 118c52b2e3d1..eb84bff11a70 100644 --- a/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs +++ b/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs @@ -21,7 +21,11 @@ public class TestRecording : IDisposable private const string charsLower = "abcdefghijklmnopqrstuvwxyz0123456789"; internal const string DateTimeOffsetNowVariableKey = "DateTimeOffsetNow"; - public TestRecording(RecordedTestMode mode, string sessionFile, RecordedTestSanitizer sanitizer, RecordMatcher matcher) + public TestRecording( + RecordedTestMode mode, + string sessionFile, + RecordedTestSanitizer sanitizer, + RecordMatcher matcher) { Mode = mode; _sessionFile = sessionFile; @@ -59,7 +63,6 @@ public TestRecording(RecordedTestMode mode, string sessionFile, RecordedTestSani private readonly RecordedTestSanitizer _sanitizer; private readonly RecordMatcher _matcher; - private readonly RecordSession _session; private RecordSession _previousSession; diff --git a/sdk/core/Azure.Core/tests/RecordSessionTests.cs b/sdk/core/Azure.Core/tests/RecordSessionTests.cs index 22259a5630c6..2f1dd687be9c 100644 --- a/sdk/core/Azure.Core/tests/RecordSessionTests.cs +++ b/sdk/core/Azure.Core/tests/RecordSessionTests.cs @@ -16,6 +16,7 @@ namespace Azure.Core.Tests public class RecordSessionTests { [TestCase("{\"json\":\"value\"}", "application/json")] + [TestCase("{\"json\":\"\"value\"\"}", "application/json")] [TestCase("{\"json\":{\"json\":\"value\"}}", "application/json")] [TestCase("{\"json\"\n:\"value\"}", "application/json")] [TestCase("{\"json\" :\"value\"}", "application/json")] From 1b9de9e9ef9aa6fc06e77c1065e5e6a8c6e1fc6f Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Wed, 3 Jun 2020 15:03:38 -0700 Subject: [PATCH 2/4] more --- sdk/core/Azure.Core/tests/RecordSessionTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/Azure.Core/tests/RecordSessionTests.cs b/sdk/core/Azure.Core/tests/RecordSessionTests.cs index 2f1dd687be9c..197a95b8c8da 100644 --- a/sdk/core/Azure.Core/tests/RecordSessionTests.cs +++ b/sdk/core/Azure.Core/tests/RecordSessionTests.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.Encodings.Web; using System.Text.Json; using Azure.Core.Pipeline; using Azure.Core.TestFramework; @@ -16,7 +17,7 @@ namespace Azure.Core.Tests public class RecordSessionTests { [TestCase("{\"json\":\"value\"}", "application/json")] - [TestCase("{\"json\":\"\"value\"\"}", "application/json")] + [TestCase("{\"json\":\"\\\"value\\\"\"}", "application/json")] [TestCase("{\"json\":{\"json\":\"value\"}}", "application/json")] [TestCase("{\"json\"\n:\"value\"}", "application/json")] [TestCase("{\"json\" :\"value\"}", "application/json")] From abf0c732328b9f5230e14a0496aa81e75704d74f Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Wed, 3 Jun 2020 15:20:31 -0700 Subject: [PATCH 3/4] More --- .../Azure.Core.TestFramework/src/RecordEntry.cs | 16 ++++++++++------ .../src/TestRecording.cs | 7 ++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs b/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs index e72ea3cde186..fc3d36d3cf63 100644 --- a/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs +++ b/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs @@ -14,11 +14,15 @@ namespace Azure.Core.TestFramework { public class RecordEntry { - private static JsonWriterOptions _requestWriterOptions = new JsonWriterOptions(); - private static JsonWriterOptions _responseWriterOptions = new JsonWriterOptions() + private static readonly JsonWriterOptions RequestWriterOptions = new JsonWriterOptions(); + // Responses are usually formatted using Newtonsoft.Json that has more relaxed encoding rules + // To enable us to store more responses as JSON instead of string in Recording files use + // relaxed settings for roundrip + private static readonly JsonWriterOptions ResponseWriterOptions = new JsonWriterOptions() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; + public RecordEntryMessage Request { get; } = new RecordEntryMessage(); public RecordEntryMessage Response { get; } = new RecordEntryMessage(); @@ -57,7 +61,7 @@ public static RecordEntry Deserialize(JsonElement element) if (element.TryGetProperty("RequestBody", out property)) { - record.Request.Body = DeserializeBody(record.Request.Headers, property, _requestWriterOptions); + record.Request.Body = DeserializeBody(record.Request.Headers, property, RequestWriterOptions); } if (element.TryGetProperty(nameof(StatusCode), out property) && @@ -73,7 +77,7 @@ public static RecordEntry Deserialize(JsonElement element) if (element.TryGetProperty("ResponseBody", out property)) { - record.Response.Body = DeserializeBody(record.Response.Headers, property, _responseWriterOptions); + record.Response.Body = DeserializeBody(record.Response.Headers, property, ResponseWriterOptions); } return record; @@ -152,7 +156,7 @@ public void Serialize(Utf8JsonWriter jsonWriter) SerializeHeaders(jsonWriter, Request.Headers); jsonWriter.WriteEndObject(); - SerializeBody(jsonWriter, "RequestBody", Request.Body, Request.Headers, _requestWriterOptions); + SerializeBody(jsonWriter, "RequestBody", Request.Body, Request.Headers, RequestWriterOptions); jsonWriter.WriteNumber(nameof(StatusCode), StatusCode); @@ -160,7 +164,7 @@ public void Serialize(Utf8JsonWriter jsonWriter) SerializeHeaders(jsonWriter, Response.Headers); jsonWriter.WriteEndObject(); - SerializeBody(jsonWriter, "ResponseBody", Response.Body, Response.Headers, _responseWriterOptions); + SerializeBody(jsonWriter, "ResponseBody", Response.Body, Response.Headers, ResponseWriterOptions); jsonWriter.WriteEndObject(); } diff --git a/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs b/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs index eb84bff11a70..118c52b2e3d1 100644 --- a/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs +++ b/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs @@ -21,11 +21,7 @@ public class TestRecording : IDisposable private const string charsLower = "abcdefghijklmnopqrstuvwxyz0123456789"; internal const string DateTimeOffsetNowVariableKey = "DateTimeOffsetNow"; - public TestRecording( - RecordedTestMode mode, - string sessionFile, - RecordedTestSanitizer sanitizer, - RecordMatcher matcher) + public TestRecording(RecordedTestMode mode, string sessionFile, RecordedTestSanitizer sanitizer, RecordMatcher matcher) { Mode = mode; _sessionFile = sessionFile; @@ -63,6 +59,7 @@ public TestRecording( private readonly RecordedTestSanitizer _sanitizer; private readonly RecordMatcher _matcher; + private readonly RecordSession _session; private RecordSession _previousSession; From 58e7005862f33927613334d7289462e13fc6c01a Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Wed, 3 Jun 2020 15:28:04 -0700 Subject: [PATCH 4/4] FB --- sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs | 2 +- sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs b/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs index fc3d36d3cf63..0484c8ce6f27 100644 --- a/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs +++ b/sdk/core/Azure.Core.TestFramework/src/RecordEntry.cs @@ -194,7 +194,7 @@ private void SerializeBody(Utf8JsonWriter jsonWriter, string name, byte[] reques // Make sure we can replay JSON is exactly the same as the source // for the case where service response was pre-formatted // fallback to generic string writing - var memoryStream = new MemoryStream(); + using var memoryStream = new MemoryStream(); // Settings of this writer should be in sync with the one used in deserialiation using (var reformattedWriter = new Utf8JsonWriter(memoryStream, writerOptions)) { diff --git a/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs b/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs index c0072a671460..6d54caed723f 100644 --- a/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs +++ b/sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.IO; -using System.Text.Json; using NUnit.Framework; namespace Azure.Core.TestFramework @@ -30,9 +29,6 @@ public abstract class RecordedTestBase : ClientTestBase public bool SaveDebugRecordingsOnFailure { get; set; } = false; #endif - public JsonWriterOptions RequestSerializationOptions { get; set; } = new JsonWriterOptions(); - public JsonWriterOptions ResponseSerializationOptions { get; set; } = new JsonWriterOptions(); - protected RecordedTestBase(bool isAsync) : this(isAsync, RecordedTestUtilities.GetModeFromEnvironment()) { }