HttpRecorder optimization for large recorded files

src/SdkCommon/Auth/Az.Auth/Microsoft.Rest.ClientRuntime.Azure.Authentication.sln

@@ -0,0 +1,37 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.27703.2026
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Rest.ClientRuntime.Azure.Authentication", "Az.Authentication\Microsoft.Rest.ClientRuntime.Azure.Authentication.csproj", "{EE15BED1-9C29-4A8F-9FAC-99EEAEEF6F31}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Az.Auth.NetCore.Test", "Az.Auth.Tests\Az.Auth.NetCore.Test\Az.Auth.NetCore.Test.csproj", "{6CFB9119-B192-44D1-8249-DD85172ABB00}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Az.Auth.Net452.Test", "Az.Auth.Tests\Az.Auth.Net452.Test\Az.Auth.Net452.Test.csproj", "{ACACDA67-5693-40B4-BA2B-6099A38C1562}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{EE15BED1-9C29-4A8F-9FAC-99EEAEEF6F31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EE15BED1-9C29-4A8F-9FAC-99EEAEEF6F31}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EE15BED1-9C29-4A8F-9FAC-99EEAEEF6F31}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EE15BED1-9C29-4A8F-9FAC-99EEAEEF6F31}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6CFB9119-B192-44D1-8249-DD85172ABB00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6CFB9119-B192-44D1-8249-DD85172ABB00}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6CFB9119-B192-44D1-8249-DD85172ABB00}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6CFB9119-B192-44D1-8249-DD85172ABB00}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ACACDA67-5693-40B4-BA2B-6099A38C1562}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ACACDA67-5693-40B4-BA2B-6099A38C1562}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ACACDA67-5693-40B4-BA2B-6099A38C1562}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ACACDA67-5693-40B4-BA2B-6099A38C1562}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {B8405D09-BF11-4B8C-87CB-E79B47CFA938}
+	EndGlobalSection
+EndGlobal

src/SdkCommon/ClientRuntime.Azure/ClientRuntime.Azure/LRO/Base/AzureLRO.cs

@@ -7,6 +7,7 @@ namespace Microsoft.Rest.ClientRuntime.Azure.LRO
     using Microsoft.Rest.ClientRuntime.Azure.Properties;
     using System;
     using System.Collections.Generic;
+    using System.Diagnostics;
     using System.Linq;
     using System.Net;
     using System.Threading;
@@ -26,7 +27,7 @@ internal abstract class AzureLRO<TResourceBody, TRequestHeaders> : IAzureLRO<TRe
         #region fields
         protected AzureOperationResponse<TResourceBody, TRequestHeaders> InitialResponse;
         protected CancellationToken CancelToken;
-        protected Dictionary<string, List<string>> CustomHeaders;
+        protected Dictionary<string, List<string>> _customHeaders;
         protected LROPollState<TResourceBody, TRequestHeaders> CurrentPollingState;
         protected IAzureClient SdkClient;
         protected bool IsLROTaskCompleted { get; set; }
@@ -51,9 +52,28 @@ protected AzureLRO(IAzureClient client,
             CancelToken = cancellationToken;
             SdkClient = client;
             ValidateInitialResponse();
+#if DEBUG
+            lroPollingCount = 1;
+#endif
         }
         #endregion
 
+        protected Dictionary<string, List<string>> CustomHeaders
+        {
+            get
+            {
+                if (_customHeaders == null)
+                    _customHeaders = new Dictionary<string, List<string>>();
+
+                return _customHeaders;
+            }
+
+            set
+            {
+                _customHeaders = value;
+            }
+        }
+
         #region Public functions
         /// <summary>
         /// Begin polling
@@ -146,6 +166,9 @@ protected virtual void InitializeAsyncHeadersToUse()
             if (CurrentPollingState == null)
             {
                 CurrentPollingState = new LROPollState<TResourceBody, TRequestHeaders>(InitialResponse, SdkClient);
+#if DEBUG
+                InitLroSession();
+#endif
 
                 if (!string.IsNullOrEmpty(CurrentPollingState.AzureAsyncOperationHeaderLink))
                 {
@@ -177,14 +200,23 @@ protected virtual void InitializeAsyncHeadersToUse()
         /// <returns></returns>
         protected virtual async Task StartPollingAsync()
         {
+#if DEBUG
+            InitPollingSession();
+#endif
             while (!AzureAsyncOperation.TerminalStatuses.Any(s => s.Equals(CurrentPollingState.Status, StringComparison.OrdinalIgnoreCase)))
             {
                 await Task.Delay(CurrentPollingState.DelayBetweenPolling, CancelToken);
+#if DEBUG
+                UpdatePollingSessionIds();
+#endif
                 await CurrentPollingState.Poll(CustomHeaders, CancelToken);
                 UpdatePollingState();
                 CheckForErrors();
                 InitializeAsyncHeadersToUse();
             }
+#if DEBUG
+            RemoveLroHeaders(removePerfImapact: true);
+#endif
         }
 
         /// <summary>
@@ -342,6 +374,95 @@ protected virtual string GetValidAbsoluteUri(string url, bool throwForInvalidUri
         #endregion
 
         #region Private functions
+
+#if DEBUG
+        const string Header_LroSessionId = "LroSessionId";
+        const string Header_LroPollingId = "LroPollingId";
+        const string Header_RecordPlaybackPerfImpact = "RecordPlaybackPerfImpact";
+        const string Header_LroOperation = "LroOperation";
+        const int perfImpactCount = 3;
+
+        private long lroSessionId { get; set; }
+
+        private long lroPollingId { get; set; }
+
+        private long lroPollingCount { get; set; }
+
+        private string lroSessionPollingId { get; set; }
+
+
+        private void InitLroSession()
+        {
+            this.lroPollingCount = 1;
+            this.lroSessionId = DateTime.Now.Ticks;
+            this.CustomHeaders.Add(Header_LroOperation, new List<string>() { this.RESTOperationVerb });
+        }
+
+        private void InitPollingSession()
+        {
+            this.lroPollingId = DateTime.Now.Ticks;
+        }
+
+        private void UpdatePollingSessionIds()
+        {
+            RemoveLroHeaders();
+            this.lroSessionPollingId = string.Format("{0}.{1}.{2}", this.lroSessionId.ToString(), this.lroPollingId.ToString(), this.lroPollingCount.ToString());
+            this.CustomHeaders.Add(Header_LroSessionId, new List<string>() { this.lroSessionId.ToString() });
+            this.CustomHeaders.Add(Header_LroPollingId, new List<string>() { this.lroSessionPollingId.ToString() });
+            if(this.lroPollingCount >= perfImpactCount)
+            {
+                if(!this.CustomHeaders.TryGetValue(Header_RecordPlaybackPerfImpact, out List<string> playBackImpact))
+                {
+                    this.CustomHeaders.Add(Header_RecordPlaybackPerfImpact, new List<string>() { "true" });
+                }
+            }
+
+            this.lroPollingCount = this.lroPollingCount + 1;
+        }
+
+        protected void RemoveLroHeaders(bool removePerfImapact = false, bool removeOperation = false)
+        {
+            if(this.CustomHeaders.TryGetValue(Header_LroSessionId, out List<string> lroSessionIdList))
+            {
+                if(lroSessionIdList.Any())
+                {
+                    this.CustomHeaders.Remove(Header_LroSessionId);
+                }
+            }
+
+            if (this.CustomHeaders.TryGetValue(Header_LroPollingId, out List<string> lroPollingIdList))
+            {
+                if (lroPollingIdList.Any())
+                {
+                    this.CustomHeaders.Remove(Header_LroPollingId);
+                }
+            }
+
+            if (removePerfImapact)
+            {
+                if (this.CustomHeaders.TryGetValue(Header_RecordPlaybackPerfImpact, out List<string> recPerfImpactList))
+                {
+                    if (recPerfImpactList.Any())
+                    {
+                        this.CustomHeaders.Remove(Header_RecordPlaybackPerfImpact);
+                    }
+                }
+            }
+
+            if(removeOperation)
+            { 
+                if (this.CustomHeaders.TryGetValue(Header_LroOperation, out List<string> restOperation))
+                {
+                    if (restOperation.Any())
+                    {
+                        this.CustomHeaders.Remove(Header_LroOperation);
+                    }
+                }
+            }
+        }
+
+#endif
+
         /// <summary>
         /// Get Valid status
         /// There are cases where there is an error sent from the service and in that case, the status should be one of the valid FailedStatuses

src/SdkCommon/ClientRuntime.Azure/ClientRuntime.Azure/LRO/LROPollState.cs

@@ -11,6 +11,7 @@ namespace Microsoft.Rest.ClientRuntime.Azure.LRO
     using System;
     using System.Collections.Generic;
     using System.Globalization;
+    using System.Linq;
     using System.Net;
     using System.Net.Http;
     using System.Threading;
@@ -46,7 +47,8 @@ internal class LROPollState<TResourceBody, TRequestHeaders> : PollingState<TReso
 
         internal string LastSerializationExceptionMessage { get; set; }
 
-        internal LROPollState(HttpOperationResponse<TResourceBody, TRequestHeaders> response, IAzureClient client) : base(response, client.LongRunningOperationRetryTimeout)
+        internal LROPollState(HttpOperationResponse<TResourceBody, TRequestHeaders> response, IAzureClient client) 
+            : base(response, client.LongRunningOperationRetryTimeout)
         {
             SdkClient = client;
             InitialResponse = response;
@@ -57,7 +59,7 @@ internal LROPollState(HttpOperationResponse<TResourceBody, TRequestHeaders> resp
 
         internal async Task Poll(Dictionary<string, List<string>> customHeaders,
             CancellationToken cancellationToken)
-        {
+        {   
             await UpdateResourceFromPollingUri(customHeaders, cancellationToken);
 
             #region Error response returned while polling on AsyncOperationHeader
@@ -136,8 +138,6 @@ private TDeserializedBodyType DeserializeToObject<TDeserializedBodyType>(Func<JO
         /// <summary>
         /// Gets a resource from the specified URL.
         /// </summary>
-        /// <param name="client">IAzureClient</param>
-        /// <param name="operationUrl">URL of the resource.</param>
         /// <param name="customHeaders">Headers that will be added to request</param>
         /// <param name="cancellationToken">Cancellation token</param>
         /// <returns></returns>
@@ -193,6 +193,7 @@ internal async Task<AzureOperationResponse<JObject, JObject>> GetRawAsync(
                 ServiceClientTracing.SendRequest(invocationId, httpRequest);
             }
             cancellationToken.ThrowIfCancellationRequested();
+
             HttpResponseMessage httpResponse = await SdkClient.HttpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
             if (shouldTrace)
             {

src/SdkCommon/ClientRuntime.Azure/ClientRuntime.Azure/LRO/Operations/DeleteLro.cs

@@ -143,6 +143,9 @@ protected override async Task PostPollingAsync()
                     }
                 }
             }
+#if DEBUG
+            RemoveLroHeaders(removeOperation: true);
+#endif
         }
 
         /// <summary>

src/SdkCommon/ClientRuntime.Azure/ClientRuntime.Azure/LRO/Operations/PATCHLro.cs

@@ -126,6 +126,9 @@ protected override async Task PostPollingAsync()
                     await CurrentPollingState.UpdateResourceFromPollingUri(CustomHeaders, CancelToken);
                 }
             }
+#if DEBUG
+            RemoveLroHeaders(removeOperation: true);
+#endif
         }
     }
 }

src/SdkCommon/ClientRuntime.Azure/ClientRuntime.Azure/LRO/Operations/POSTLro.cs

@@ -120,6 +120,9 @@ protected override async Task PostPollingAsync()
                     }
                 }
             }
+#if DEBUG
+            RemoveLroHeaders(removeOperation: true);
+#endif
         }
     }
 }

src/SdkCommon/ClientRuntime.Azure/ClientRuntime.Azure/LRO/Operations/PUTLro.cs

@@ -65,6 +65,9 @@ protected override async Task PostPollingAsync()
                     await CurrentPollingState.UpdateResourceFromPollingUri(CustomHeaders, CancelToken);
                 }
             }
+#if DEBUG
+            RemoveLroHeaders(removeOperation:true);
+#endif
         }
 
         protected override void CheckForErrors()

src/SdkCommon/ClientRuntime.sln

@@ -17,8 +17,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Rest.ClientRuntim
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Rest.ClientRuntime", "ClientRuntime\ClientRuntime\Microsoft.Rest.ClientRuntime.csproj", "{C8DEE788-A658-4C1F-ACC4-866CDE9315BB}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ClientRuntime.E2E.Tests", "Test\ClientRuntime.E2E.Tests\ClientRuntime.E2E.Tests.csproj", "{87980F3C-A993-4620-B43A-07A174DB852E}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CR.Azure.NetCore.Tests", "ClientRuntime.Azure\Tests\CR.Azure.NetCore.Tests\CR.Azure.NetCore.Tests.csproj", "{99E868D7-7046-4E3C-8B4F-AFF7D9F603AD}"
 	ProjectSection(ProjectDependencies) = postProject
 		{E67D3C1A-67CA-4315-918E-ADD2108AC66D} = {E67D3C1A-67CA-4315-918E-ADD2108AC66D}
@@ -58,10 +56,6 @@ Global
 		{C8DEE788-A658-4C1F-ACC4-866CDE9315BB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C8DEE788-A658-4C1F-ACC4-866CDE9315BB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C8DEE788-A658-4C1F-ACC4-866CDE9315BB}.Release|Any CPU.Build.0 = Release|Any CPU
-		{87980F3C-A993-4620-B43A-07A174DB852E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{87980F3C-A993-4620-B43A-07A174DB852E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{87980F3C-A993-4620-B43A-07A174DB852E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{87980F3C-A993-4620-B43A-07A174DB852E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{99E868D7-7046-4E3C-8B4F-AFF7D9F603AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{99E868D7-7046-4E3C-8B4F-AFF7D9F603AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{99E868D7-7046-4E3C-8B4F-AFF7D9F603AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -81,7 +75,6 @@ Global
 	GlobalSection(NestedProjects) = preSolution
 		{05A3BD74-8C1B-457B-A85A-35C7B393449E} = {FC756A84-660A-4917-85D0-0BBFB19FF71C}
 		{582890B6-B962-4852-8420-1D7BB68E41E8} = {FC756A84-660A-4917-85D0-0BBFB19FF71C}
-		{87980F3C-A993-4620-B43A-07A174DB852E} = {FC756A84-660A-4917-85D0-0BBFB19FF71C}
 		{99E868D7-7046-4E3C-8B4F-AFF7D9F603AD} = {FC756A84-660A-4917-85D0-0BBFB19FF71C}
 		{CCE671C2-5FCC-4A5B-8D80-0636A04D2E2D} = {FC756A84-660A-4917-85D0-0BBFB19FF71C}
 	EndGlobalSection

src/SdkCommon/Test/ClientRuntime.E2E.Tests/ClientRuntime.E2E.Tests.csproj

@@ -16,15 +16,16 @@
     <PackageReference Include="Microsoft.Azure.Management.Compute" Version="[14.1.0-prerelease]" />
     <PackageReference Include="Microsoft.Azure.Management.Network" Version="[5.0.1-preview,7.0)" />
 
-    <PackageReference Include="Microsoft.Azure.Test.HttpRecorder" Version="[1.10.99-Private]" />
-    <PackageReference Include="Microsoft.Rest.ClientRuntime.Azure.TestFramework" Version="[1.6.0,2.0.0)" />
+    <!--<PackageReference Include="Microsoft.Azure.Test.HttpRecorder" Version="[1.10.99-Private]" />
+    <PackageReference Include="Microsoft.Rest.ClientRuntime.Azure.TestFramework" Version="[1.6.0,2.0.0)" />-->
 
     <!--<ProjectReference Include="..\..\ClientRuntime\ClientRuntime\Microsoft.Rest.ClientRuntime.csproj" />-->
     <ProjectReference Include="..\..\ClientRuntime.Azure\ClientRuntime.Azure\Microsoft.Rest.ClientRuntime.Azure.csproj" />
-
-    <!--
     <ProjectReference Include="..\..\TestFramework\Microsoft.Azure.Test.HttpRecorder\Microsoft.Azure.Test.HttpRecorder.csproj" />
     <ProjectReference Include="..\..\TestFramework\ClientRuntime.Azure.TestFramework\Microsoft.Rest.ClientRuntime.Azure.TestFramework.csproj" />
+
+    <!--
+
 
     <PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.6" />
     <PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.5" />

src/SdkCommon/Test/ClientRuntime.E2E.Tests/ClientRuntime.E2E.Tests.sln

@@ -13,6 +13,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Rest.ClientRuntim
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Test.HttpRecorder", "..\..\TestFramework\Microsoft.Azure.Test.HttpRecorder\Microsoft.Azure.Test.HttpRecorder.csproj", "{7BC125FA-8B13-4689-BE45-08F01C96CD51}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Rest.ClientRuntime.Azure.Authentication", "..\..\Auth\Az.Auth\Az.Authentication\Microsoft.Rest.ClientRuntime.Azure.Authentication.csproj", "{455C54DE-1122-45C4-9B53-5B2DE44B8938}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -39,8 +41,15 @@ Global
 		{7BC125FA-8B13-4689-BE45-08F01C96CD51}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7BC125FA-8B13-4689-BE45-08F01C96CD51}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7BC125FA-8B13-4689-BE45-08F01C96CD51}.Release|Any CPU.Build.0 = Release|Any CPU
+		{455C54DE-1122-45C4-9B53-5B2DE44B8938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{455C54DE-1122-45C4-9B53-5B2DE44B8938}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{455C54DE-1122-45C4-9B53-5B2DE44B8938}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{455C54DE-1122-45C4-9B53-5B2DE44B8938}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {06D4BDBC-774F-4087-8FB5-DF64CEF52FF7}
+	EndGlobalSection
 EndGlobal