Add metadata header on requests

src/Elasticsearch.Net/Api/RequestParameters/RequestParametersExtensions.cs

@@ -0,0 +1,25 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+
+namespace Elasticsearch.Net
+{
+	internal static class RequestParametersExtensions
+	{
+		internal static void SetRequestMetaData(this IRequestParameters parameters, RequestMetaData requestMetaData)
+		{
+			if (parameters is null)
+				throw new ArgumentNullException(nameof(parameters));
+
+			if (requestMetaData is null)
+				throw new ArgumentNullException(nameof(requestMetaData));
+
+			if (parameters.RequestConfiguration is null)
+				parameters.RequestConfiguration = new RequestConfiguration();
+
+			parameters.RequestConfiguration.RequestMetaData = requestMetaData;
+		}
+	}
+}

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -33,36 +33,7 @@ public class ConnectionConfiguration : ConnectionConfiguration<ConnectionConfigu
 		/// As the old curl based handler is known to bleed TCP connections:
 		/// <para>https://github.com/dotnet/runtime/issues/22366</para>
 		/// </summary>
-        private static bool UsingCurlHandler
-		{
-			get
-			{
-#if !DOTNETCORE
-				return false;
-#else
-				var curlHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.CurlHandler") != null;
-				if (!curlHandlerExists) return false;
-
-				var socketsHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.SocketsHttpHandler") != null;
-				// running on a .NET core version with CurlHandler, before the existence of SocketsHttpHandler.
-				// Must be using CurlHandler.
-				if (!socketsHandlerExists) return true;
-
-				if (AppContext.TryGetSwitch("System.Net.Http.UseSocketsHttpHandler", out var isEnabled))
-					return !isEnabled;
-
-				var environmentVariable =
-					Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER");
-
-				// SocketsHandler exists and no environment variable exists to disable it.
-				// Must be using SocketsHandler and not CurlHandler
-				if (environmentVariable == null) return false;
-
-				return environmentVariable.Equals("false", StringComparison.OrdinalIgnoreCase) ||
-					environmentVariable.Equals("0");
-#endif
-			}
-		}
+		private static bool UsingCurlHandler => ConnectionInfo.UsingCurlHandler;
 
 		/// <summary>
 		/// The default ping timeout. Defaults to 2 seconds
@@ -176,6 +147,7 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private TimeSpan? _deadTimeout;
 		private bool _disableAutomaticProxyDetection;
 		private bool _disableDirectStreaming;
+		private bool _disableMetaHeader;
 		private bool _disablePings;
 		private bool _enableHttpCompression;
 		private bool _enableHttpPipelining = true;
@@ -207,7 +179,9 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private bool _enableThreadPoolStats;
 
 		private string _userAgent = ConnectionConfiguration.DefaultUserAgent;
-		private Func<HttpMethod, int, bool> _statusCodeToResponseSuccess;
+		private readonly Func<HttpMethod, int, bool> _statusCodeToResponseSuccess;
+
+		private readonly List<IHeaderProvider> _customerHeaderProviders = new List<IHeaderProvider> { new MetaHeaderProvider<IElasticLowLevelClient>() };
 
 		protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection connection, IElasticsearchSerializer requestResponseSerializer)
 		{
@@ -248,6 +222,7 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		TimeSpan? IConnectionConfigurationValues.DeadTimeout => _deadTimeout;
 		bool IConnectionConfigurationValues.DisableAutomaticProxyDetection => _disableAutomaticProxyDetection;
 		bool IConnectionConfigurationValues.DisableDirectStreaming => _disableDirectStreaming;
+		bool IConnectionConfigurationValues.DisableMetaHeader => _disableMetaHeader;
 		bool IConnectionConfigurationValues.DisablePings => _disablePings;
 		bool IConnectionConfigurationValues.EnableHttpCompression => _enableHttpCompression;
 		NameValueCollection IConnectionConfigurationValues.Headers => _headers;
@@ -287,6 +262,8 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		bool IConnectionConfigurationValues.EnableTcpStats => _enableTcpStats;
 		bool IConnectionConfigurationValues.EnableThreadPoolStats => _enableThreadPoolStats;
 
+		IReadOnlyCollection<IHeaderProvider> IConnectionConfigurationValues.CustomHeaderProviders => _customerHeaderProviders.ToReadOnlyCollection();
+
 		void IDisposable.Dispose() => DisposeManagedResources();
 
 		private static void DefaultCompletedRequestHandler(IApiCallDetails response) { }
@@ -368,6 +345,8 @@ public T SniffOnConnectionFault(bool sniffsOnConnectionFault = true) =>
 		/// </summary>
 		public T DisableAutomaticProxyDetection(bool disable = true) => Assign(disable, (a, v) => a._disableAutomaticProxyDetection = v);
 
+		public T DisableMetaHeader(bool disable = true) => Assign(disable, (a, v) => a._disableMetaHeader = v);
+
 		/// <summary>
 		/// Instead of following a c/go like error checking on response.IsValid do throw an exception (except when <see cref="IApiCallDetails.SuccessOrKnownError"/> is false)
 		/// on the client when a call resulted in an exception on either the client or the Elasticsearch server.

src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs

@@ -77,6 +77,8 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// </summary>
 		bool DisableDirectStreaming { get; }
 
+		bool DisableMetaHeader { get; }
+
 		/// <summary>
 		/// This signals that we do not want to send initial pings to unknown/previously dead nodes
 		/// and just send the call straightaway
@@ -273,5 +275,10 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// Enable statistics about thread pools to be collected when making a request
 		/// </summary>
 		bool EnableThreadPoolStats { get; }
+
+		/// <summary>
+		/// The configured collection of <see cref="IHeaderProvider"/> instances which will be run to add additional request headers.
+		/// </summary>
+		IReadOnlyCollection<IHeaderProvider> CustomHeaderProviders { get; }
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfiguration.cs

@@ -126,6 +126,11 @@ public interface IRequestConfiguration
 
 		/// <inheritdoc cref="IConnectionConfigurationValues.EnableThreadPoolStats"/>
 		bool? EnableThreadPoolStats { get; set; }
+
+		/// <summary>
+		/// Holds additional meta data about the request.
+		/// </summary>
+		RequestMetaData RequestMetaData { get; set; }
 	}
 
 	public class RequestConfiguration : IRequestConfiguration
@@ -172,6 +177,8 @@ public class RequestConfiguration : IRequestConfiguration
 		public bool? EnableTcpStats { get; set; }
 		/// <inheritdoc />
 		public bool? EnableThreadPoolStats { get; set; }
+		/// <inheritdoc />
+		public RequestMetaData RequestMetaData { get; set; }
 	}
 
 	public class RequestConfigurationDescriptor : IRequestConfiguration
@@ -223,6 +230,7 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 		NameValueCollection IRequestConfiguration.Headers { get; set; }
 		bool? IRequestConfiguration.EnableTcpStats { get; set; }
 		bool? IRequestConfiguration.EnableThreadPoolStats { get; set; }
+		RequestMetaData IRequestConfiguration.RequestMetaData { get; set; }
 
 		/// <summary>
 		/// Submit the request on behalf in the context of a different shield user
@@ -406,5 +414,12 @@ public RequestConfigurationDescriptor EnableThreadPoolStats(bool? enableThreadPo
 			Self.EnableThreadPoolStats = enableThreadPoolStats;
 			return this;
 		}
+
+		/// <inheritdoc cref="IRequestConfiguration.RequestMetaData" />
+		internal RequestConfigurationDescriptor RequestMetaData(RequestMetaData metaData)
+		{
+			Self.RequestMetaData = metaData;
+			return this;
+		}
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfigurationExtensions.cs

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+
+namespace Elasticsearch.Net
+{
+	internal static class RequestConfigurationExtensions
+	{
+		internal static void SetRequestMetaData(this IRequestConfiguration requestConfiguration, RequestMetaData requestMetaData)
+		{
+			if (requestConfiguration is null)
+				throw new ArgumentNullException(nameof(requestConfiguration));
+
+			if (requestMetaData is null)
+				throw new ArgumentNullException(nameof(requestMetaData));
+
+			requestConfiguration.RequestMetaData = requestMetaData;
+		}
+	}
+}

src/Elasticsearch.Net/Configuration/RequestMetaData.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+
+namespace Elasticsearch.Net
+{
+	/// <summary>
+	/// Holds meta data about a client request.
+	/// </summary>
+	public sealed class RequestMetaData
+	{
+		/// <summary>
+		/// Reserved key for a meta data entry which identifies the helper which produced the request.
+		/// </summary>
+		internal const string HelperKey = "helper";
+
+		private Dictionary<string, string> _metaDataItems;
+
+		internal bool TryAddMetaData (string key, string value)
+		{
+			if (_metaDataItems is null)
+				_metaDataItems = new Dictionary<string, string>();
+
+#if NETSTANDARD2_1
+			return _metaDataItems.TryAdd(key, value);
+#else
+			if (_metaDataItems.ContainsKey(key))
+				return false;
+
+			_metaDataItems.Add(key, value);
+			return true;
+#endif
+		}		
+
+		public IReadOnlyDictionary<string, string> Items => _metaDataItems is null ? EmptyReadOnly<string, string>.Dictionary : _metaDataItems;
+	}
+}

src/Elasticsearch.Net/Connection/ConnectionInfo.cs

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+#if DOTNETCORE
+using System.Net.Http;
+#endif
+
+namespace Elasticsearch.Net
+{
+	public static class ConnectionInfo
+	{
+		public static bool UsingCurlHandler
+		{
+			get
+			{
+#if !DOTNETCORE
+				return false;
+#else
+				var curlHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.CurlHandler") != null;
+				if (!curlHandlerExists)
+					return false;
+
+				var socketsHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.SocketsHttpHandler") != null;
+				// running on a .NET core version with CurlHandler, before the existence of SocketsHttpHandler.
+				// Must be using CurlHandler.
+				if (!socketsHandlerExists)
+					return true;
+
+				if (AppContext.TryGetSwitch("System.Net.Http.UseSocketsHttpHandler", out var isEnabled))
+					return !isEnabled;
+
+				var environmentVariable =
+					Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER");
+
+				// SocketsHandler exists and no environment variable exists to disable it.
+				// Must be using SocketsHandler and not CurlHandler
+				if (environmentVariable == null)
+					return false;
+
+				return environmentVariable.Equals("false", StringComparison.OrdinalIgnoreCase) ||
+					environmentVariable.Equals("0");
+#endif
+			}
+		}
+	}
+}

src/Elasticsearch.Net/Connection/HttpConnection.cs

@@ -37,7 +37,6 @@ internal class WebProxy : IWebProxy
 		public bool IsBypassed(Uri host) => host.IsLoopback;
 	}
 
-
 	/// <summary> The default IConnection implementation. Uses <see cref="HttpClient" />.</summary>
 	public class HttpConnection : IConnection
 	{
@@ -139,6 +138,7 @@ public virtual async Task<TResponse> RequestAsync<TResponse>(RequestData request
 			IDisposable receive = DiagnosticSources.SingletonDisposable;
 			ReadOnlyDictionary<TcpState, int> tcpStats = null;
 			ReadOnlyDictionary<string, ThreadPoolStatistics> threadPoolStats = null;
+			requestData.IsAsync = true;
 
 			try
 			{
@@ -248,6 +248,8 @@ protected virtual HttpMessageHandler CreateHttpClientHandler(RequestData request
 
 		protected virtual HttpRequestMessage CreateHttpRequestMessage(RequestData requestData)
 		{
+			requestData.HttpClientIdentifier = ConnectionInfo.UsingCurlHandler ? "c" : "s";
+
 			var request = CreateRequestMessage(requestData);
 			SetAuthenticationIfNeeded(request, requestData);
 			return request;
@@ -333,6 +335,14 @@ protected virtual HttpRequestMessage CreateRequestMessage(RequestData requestDat
 			if (!requestData.RunAs.IsNullOrEmpty())
 				requestMessage.Headers.Add(RequestData.RunAsSecurityHeader, requestData.RunAs);
 
+			foreach (var customHeaderProvider in requestData.CustomHeaderProviders)
+			{
+				var value = customHeaderProvider.ProduceHeaderValue(requestData);
+
+				if (!string.IsNullOrEmpty(value))
+						requestMessage.Headers.TryAddWithoutValidation(customHeaderProvider.HeaderName, value);
+			}
+
 			return requestMessage;
 		}
 

src/Elasticsearch.Net/Connection/HttpWebRequestConnection.cs

@@ -111,6 +111,7 @@ CancellationToken cancellationToken
 
 			try
 			{
+				requestData.IsAsync = true;
 				var data = requestData.PostData;
 				var request = CreateHttpWebRequest(requestData);
 				using (cancellationToken.Register(() => request.Abort()))
@@ -179,6 +180,8 @@ CancellationToken cancellationToken
 
 		protected virtual HttpWebRequest CreateHttpWebRequest(RequestData requestData)
 		{
+			requestData.HttpClientIdentifier = "w";
+
 			var request = CreateWebRequest(requestData);
 			SetAuthenticationIfNeeded(requestData, request);
 			SetProxyIfNeeded(request, requestData);
@@ -238,6 +241,14 @@ protected virtual HttpWebRequest CreateWebRequest(RequestData requestData)
 			if (requestData.Headers != null && requestData.Headers.HasKeys())
 				request.Headers.Add(requestData.Headers);
 
+			foreach (var customHeaderProvider in requestData.CustomHeaderProviders)
+			{
+				var value = customHeaderProvider.ProduceHeaderValue(requestData);
+
+				if (!string.IsNullOrEmpty(value))
+					request.Headers.Add(customHeaderProvider.HeaderName, customHeaderProvider.ProduceHeaderValue(requestData));
+			}
+
 			var timeout = (int)requestData.RequestTimeout.TotalMilliseconds;
 			request.Timeout = timeout;
 			request.ReadWriteTimeout = timeout;

src/Elasticsearch.Net/Connection/IHeaderProvider.cs

@@ -0,0 +1,24 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elasticsearch.Net
+{
+	/// <summary>
+	/// A provider for additional HTTP request headers.
+	/// </summary>
+	public interface IHeaderProvider
+	{
+		/// <summary>
+		/// The name of the header produced by this provider.
+		/// </summary>
+		string HeaderName { get; }
+
+		/// <summary>
+		/// Produces the value for the header using information from the <see cref="RequestData"/>.
+		/// </summary>
+		/// <param name="requestData">Data about the request which may be used to produce the header value.</param>
+		/// <returns></returns>
+		string ProduceHeaderValue(RequestData requestData);
+	}
+}