Fix HTTP compression for HttpConnection HttpClient implementation (#3921)

* GzipStream was wrapping completed task handler not the writable stream
* Make sure HttpCompression is enabled randomly when running the integration tests
* Remove adding Accept-Encoding twice

This commit removes adding Accept-Encoding headers again, when constructing the requestMessage.
These headers are added by the default HttpClientHandler implementation, with AutomaticDecompression.

There is the possibility that a user overrides CreateHttpClientHandler() and does not set AutomaticDecompression, but does not override CreateRequestMessage(), and is relying on CreateHttpRequestMessage to add the Accept-Encoding headers when using HttpCompression. This commit will break that usage pattern, but I think this is acceptable to do, because we should optimize for the default behaviour i.e. we _could_ check that Accept-Encoding headers have been set when HttpCompression is enabled, inside CreateHttpRequestMessage(), and set them if not, but this is additional overhead to perform on every request. Instead, a user overriding CreateHttpClientHandler() and not setting AutomaticDecompression, and not overriding CreateRequestMessage(), when wishing to use HttpCompression, should set AutomaticDecompression inside their overridden implementation.

Fixes #3913

Author: Mpdreamz

src/Elasticsearch.Net/Connection/Content/RequestDataContent.cs

@@ -37,6 +37,8 @@ public RequestDataContent(RequestData requestData)
 
 			Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, TransportContext context)
 			{
+				if (_requestData.HttpCompression)
+					stream = new GZipStream(stream, CompressionMode.Compress, false);
 				using(stream)
 					data.Write(stream, requestData.ConnectionSettings);
 				return Task.CompletedTask;
@@ -52,6 +54,8 @@ public RequestDataContent(RequestData requestData, CancellationToken token)
 
 			async Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, TransportContext context)
 			{
+				if (_requestData.HttpCompression)
+					stream = new GZipStream(stream, CompressionMode.Compress, false);
 				using (stream)
 					await data.WriteAsync(stream, requestData.ConnectionSettings, token).ConfigureAwait(false);
 			}
@@ -75,8 +79,6 @@ protected override async Task SerializeToStreamAsync(Stream stream, TransportCon
 
 			var serializeToStreamTask = new TaskCompletionSource<bool>();
 
-			if (_requestData.HttpCompression)
-				stream = new GZipStream(stream, CompressionMode.Compress, false);
 			var wrappedStream = new CompleteTaskOnCloseStream(stream, serializeToStreamTask);
             await _onStreamAvailable(data, wrappedStream, this, context).ConfigureAwait(false);
             await serializeToStreamTask.Task.ConfigureAwait(false);

src/Elasticsearch.Net/Connection/HttpConnection.cs

@@ -241,13 +241,7 @@ protected virtual HttpRequestMessage CreateRequestMessage(RequestData requestDat
 
 			if (!requestData.RunAs.IsNullOrEmpty())
 				requestMessage.Headers.Add(RequestData.RunAsSecurityHeader, requestData.RunAs);
-
-			if (requestData.HttpCompression)
-			{
-				requestMessage.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
-				requestMessage.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("deflate"));
-			}
-
+			
 			return requestMessage;
 		}
 

src/Tests/Tests.Configuration/EnvironmentConfiguration.cs

@@ -28,6 +28,7 @@ public EnvironmentConfiguration(YamlConfiguration yamlConfiguration)
 			{
 				SourceSerializer = RandomBoolConfig("SOURCESERIALIZER", randomizer),
 				TypedKeys = RandomBoolConfig("TYPEDKEYS", randomizer),
+				HttpCompression = RandomBoolConfig("HTTPCOMPRESSION", randomizer),
 			};
 		}
 

src/Tests/Tests.Configuration/TestConfigurationBase.cs

@@ -66,5 +66,8 @@ public class RandomConfiguration
 
 		/// <summary> Randomly enable typed keys on searches (defaults to true) on NEST search requests</summary>
 		public bool TypedKeys { get; set; }
+		
+		/// <summary> Randomly enable compression on the http requests</summary>
+		public bool HttpCompression { get; set; }
 	}
 }

src/Tests/Tests.Configuration/TestConfigurationExtensions.cs

@@ -24,6 +24,7 @@ public static void DumpConfiguration(this TestConfigurationBase config)
 			Console.WriteLine($" - Random:");
 			Console.WriteLine($" \t- {nameof(config.Random.SourceSerializer)}: {config.Random.SourceSerializer}");
 			Console.WriteLine($" \t- {nameof(config.Random.TypedKeys)}: {config.Random.TypedKeys}");
+			Console.WriteLine($" \t- {nameof(config.Random.HttpCompression)}: {config.Random.HttpCompression}");
 			Console.WriteLine(new string('-', 20));
 		}
 	}

src/Tests/Tests.Configuration/YamlConfiguration.cs

@@ -34,6 +34,7 @@ public YamlConfiguration(string configurationFile)
 			{
 				SourceSerializer = RandomBool("source_serializer", randomizer),
 				TypedKeys = RandomBool("typed_keys", randomizer),
+				HttpCompression = RandomBool("http_compression", randomizer),
 			};
 		}
 

src/Tests/Tests.Configuration/tests.default.yaml

@@ -20,7 +20,8 @@ test_against_already_running_elasticsearch: true
 
 #random_source_serializer: true
 #random_old_connection: true
-seed: 74337
+#random_http_compresssion: true
+#seed: 74337
 
 # Can be helpful to speed up tests runs as setting this to true only randomly tests a single overload of the api rather than all 4.
 # Can also help keep the noise down in case of test failures

src/Tests/Tests.Core/Client/Settings/TestConnectionSettings.cs

@@ -43,8 +43,7 @@ public TestConnectionSettings(
 		private void ApplyTestSettings() => 
 			RerouteToProxyIfNeeded()
 			.EnableDebugMode()
-				//TODO make this random
-			//.EnableHttpCompression()
+			.EnableHttpCompression(TestConfiguration.Instance.Random.HttpCompression)
 #if DEBUG
 			.EnableDebugMode()
 #endif

src/Tests/Tests.Core/Xunit/NestXunitRunOptions.cs

@@ -96,6 +96,7 @@ bool runningIntegrations
 
 			AppendExplictConfig(nameof(RandomConfiguration.SourceSerializer), sb);
 			AppendExplictConfig(nameof(RandomConfiguration.TypedKeys), sb);
+			AppendExplictConfig(nameof(RandomConfiguration.HttpCompression), sb);
 
 			if (runningIntegrations)
 				sb.Append("integrate ")