Date Nanos implementation

docs/common-options/date-math/date-math-expressions.asciidoc

@@ -83,14 +83,28 @@ anchor will be an actual `DateTime`, even after a serialization/deserialization
 [source,csharp]
 ----
 var date = new DateTime(2015, 05, 05);
-Expect("2015-05-05T00:00:00")
-    .WhenSerializing<Nest.DateMath>(date)
-    .AssertSubject(dateMath => ((IDateMath)dateMath)
-        .Anchor.Match(
-            d => d.Should().Be(date),
-            s => s.Should().BeNull()
-        )
-    );
+----
+
+will serialize to
+
+[source,javascript]
+----
+"2015-05-05T00:00:00"
+----
+
+When the `DateTime` is local or UTC, the time zone information is included.
+For example, for a UTC `DateTime`
+
+[source,csharp]
+----
+var utcDate = new DateTime(2015, 05, 05, 0, 0, 0, DateTimeKind.Utc);
+----
+
+will serialize to
+
+[source,javascript]
+----
+"2015-05-05T00:00:00Z"
 ----
 
 ==== Complex expressions

src/CodeGeneration/DocGenerator/StringExtensions.cs

@@ -213,7 +213,7 @@ public static string RemoveNumberOfLeadingTabsOrSpacesAfterNewline(this string i
 
 		public static string[] SplitOnNewLines(this string input, StringSplitOptions options) => input.Split(new[] { "\r\n", "\n" }, options);
 
-		public static bool TryGetJsonForAnonymousType(this string anonymousTypeString, out string json)
+		public static bool TryGetJsonForExpressionSyntax(this string anonymousTypeString, out string json)
 		{
 			json = null;
 

src/CodeGeneration/DocGenerator/SyntaxNodeExtensions.cs

@@ -72,11 +72,10 @@ public static bool TryGetJsonForSyntaxNode(this SyntaxNode node, out string json
 			json = null;
 
 			// find the first anonymous object or new object expression
-			var creationExpressionSyntax = node.DescendantNodes()
-				.FirstOrDefault(n => n is AnonymousObjectCreationExpressionSyntax || n is ObjectCreationExpressionSyntax);
+			var syntax = node.DescendantNodes()
+				.FirstOrDefault(n => n is AnonymousObjectCreationExpressionSyntax || n is ObjectCreationExpressionSyntax || n is LiteralExpressionSyntax);
 
-			return creationExpressionSyntax != null &&
-				creationExpressionSyntax.ToFullString().TryGetJsonForAnonymousType(out json);
+			return syntax != null && syntax.ToFullString().TryGetJsonForExpressionSyntax(out json);
 		}
 
 		/// <summary>

src/Elasticsearch.Net/Connection/Content/RequestDataContent.cs

@@ -25,8 +25,7 @@ namespace Elasticsearch.Net
 	internal class RequestDataContent : HttpContent
 	{
 		private readonly RequestData _requestData;
-		private readonly Func<PostData, CompleteTaskOnCloseStream, RequestDataContent, TransportContext, Task> _onStreamAvailable;
-
+		private readonly Func<RequestData, CompleteTaskOnCloseStream, RequestDataContent, TransportContext, Task> _onStreamAvailable;
 
 		public RequestDataContent(RequestData requestData)
 		{
@@ -35,12 +34,17 @@ public RequestDataContent(RequestData requestData)
 			if (requestData.HttpCompression)
 				Headers.ContentEncoding.Add("gzip");
 
-			Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, TransportContext context)
+			Task OnStreamAvailable(RequestData data, Stream stream, HttpContent content, TransportContext context)
 			{
+				if (data.HttpCompression)
+					stream = new GZipStream(stream, CompressionMode.Compress, false);
+
 				using(stream)
-					data.Write(stream, requestData.ConnectionSettings);
+					data.PostData.Write(stream, data.ConnectionSettings);
+
 				return Task.CompletedTask;
 			}
+
 			_onStreamAvailable = OnStreamAvailable;
 		}
 		public RequestDataContent(RequestData requestData, CancellationToken token)
@@ -50,11 +54,15 @@ public RequestDataContent(RequestData requestData, CancellationToken token)
 			if (requestData.HttpCompression)
 				Headers.ContentEncoding.Add("gzip");
 
-			async Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, TransportContext context)
+			async Task OnStreamAvailable(RequestData data, Stream stream, HttpContent content, TransportContext context)
 			{
+				if (data.HttpCompression)
+					stream = new GZipStream(stream, CompressionMode.Compress, false);
+
 				using (stream)
-					await data.WriteAsync(stream, requestData.ConnectionSettings, token).ConfigureAwait(false);
+					await data.PostData.WriteAsync(stream, data.ConnectionSettings, token).ConfigureAwait(false);
 			}
+
 			_onStreamAvailable = OnStreamAvailable;
 		}
 
@@ -69,16 +77,9 @@ async Task OnStreamAvailable(PostData data, Stream stream, HttpContent content,
 		[SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Exception is passed as task result.")]
 		protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context)
 		{
-
-			var data = _requestData.PostData;
-			if (data == null) return;
-
 			var serializeToStreamTask = new TaskCompletionSource<bool>();
-
-			if (_requestData.HttpCompression)
-				stream = new GZipStream(stream, CompressionMode.Compress, false);
 			var wrappedStream = new CompleteTaskOnCloseStream(stream, serializeToStreamTask);
-            await _onStreamAvailable(data, wrappedStream, this, context).ConfigureAwait(false);
+            await _onStreamAvailable(_requestData, wrappedStream, this, context).ConfigureAwait(false);
             await serializeToStreamTask.Task.ConfigureAwait(false);
 		}
 
@@ -111,7 +112,6 @@ protected override void Dispose(bool disposing)
 				base.Dispose();
 			}
 
-
 			public override void Close() => _serializeToStreamTask.TrySetResult(true);
 		}
 
@@ -193,6 +193,8 @@ public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, As
 			public override void EndWrite(IAsyncResult asyncResult) => _innerStream.EndWrite(asyncResult);
 
 			public override void WriteByte(byte value) => _innerStream.WriteByte(value);
+
+			public override void Close() => _innerStream.Close();
 		}
 	}
 }

src/Elasticsearch.Net/Connection/HttpConnection.cs

@@ -57,7 +57,10 @@ public virtual TResponse Request<TResponse>(RequestData requestData)
 			try
 			{
 				var requestMessage = CreateHttpRequestMessage(requestData);
-				SetContent(requestMessage, requestData);
+
+				if (requestData.PostData != null)
+					SetContent(requestMessage, requestData);
+
 				using(requestMessage?.Content ?? (IDisposable)Stream.Null)
 				using (var d = DiagnosticSource.Diagnose<RequestData, int?>(DiagnosticSources.HttpConnection.SendAndReceiveHeaders, requestData))
 				{
@@ -107,8 +110,11 @@ public virtual async Task<TResponse> RequestAsync<TResponse>(RequestData request
 			try
 			{
 				var requestMessage = CreateHttpRequestMessage(requestData);
-				SetAsyncContent(requestMessage, requestData, cancellationToken);
-				using(requestMessage?.Content ?? (IDisposable)Stream.Null) 
+
+				if (requestData.PostData != null)
+					SetAsyncContent(requestMessage, requestData, cancellationToken);
+
+				using(requestMessage?.Content ?? (IDisposable)Stream.Null)
 				using (var d = DiagnosticSource.Diagnose<RequestData, int?>(DiagnosticSources.HttpConnection.SendAndReceiveHeaders, requestData))
 				{
 					responseMessage = await client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);

src/Elasticsearch.Net/Extensions/StringBuilderCache.cs

@@ -0,0 +1,64 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Text;
+
+namespace Elasticsearch.Net.Extensions
+{
+	/// <summary>Provide a cached reusable instance of stringbuilder per thread.</summary>
+	internal static class StringBuilderCache
+	{
+		private const int DefaultCapacity = 16; // == StringBuilder.DefaultCapacity
+
+		// The value 360 was chosen in discussion with performance experts as a compromise between using
+		// as little memory per thread as possible and still covering a large part of short-lived
+		// StringBuilder creations on the startup path of VS designers.
+		private const int MaxBuilderSize = 360;
+
+		// WARNING: We allow diagnostic tools to directly inspect this member (t_cachedInstance).
+		// See https://github.com/dotnet/corert/blob/master/Documentation/design-docs/diagnostics/diagnostics-tools-contract.md for more details.
+		// Please do not change the type, the name, or the semantic usage of this member without understanding the implication for tools.
+		// Get in touch with the diagnostics team if you have questions.
+		[ThreadStatic]
+		private static StringBuilder _cachedInstance;
+
+		/// <summary>Get a StringBuilder for the specified capacity.</summary>
+		/// <remarks>If a StringBuilder of an appropriate size is cached, it will be returned and the cache emptied.</remarks>
+		public static StringBuilder Acquire(int capacity = DefaultCapacity)
+		{
+			if (capacity <= MaxBuilderSize)
+			{
+				var sb = _cachedInstance;
+				if (sb != null)
+				{
+					// Avoid stringbuilder block fragmentation by getting a new StringBuilder
+					// when the requested size is larger than the current capacity
+					if (capacity <= sb.Capacity)
+					{
+						_cachedInstance = null;
+						sb.Clear();
+						return sb;
+					}
+				}
+			}
+
+			return new StringBuilder(capacity);
+		}
+
+		/// <summary>Place the specified builder in the cache if it is not too big.</summary>
+		public static void Release(StringBuilder sb)
+		{
+			if (sb.Capacity <= MaxBuilderSize) _cachedInstance = sb;
+		}
+
+		/// <summary>ToString() the stringbuilder, Release it to the cache, and return the resulting string.</summary>
+		public static string GetStringAndRelease(StringBuilder sb)
+		{
+			var result = sb.ToString();
+			Release(sb);
+			return result;
+		}
+	}
+}

src/Nest/Aggregations/Metric/WeightedAverage/WeightedAverageValue.cs

@@ -7,7 +7,7 @@
 namespace Nest
 {
 	/// <summary>
-	/// The configuration for a field or scrip that provides a value or weight
+	/// The configuration for a field or script that provides a value or weight
 	/// for <see cref="WeightedAverageAggregation" />
 	/// </summary>
 	[InterfaceDataContract]
@@ -104,6 +104,9 @@ public enum ValueType
 		/// <summary>A date value</summary>
 		[EnumMember(Value = "date")] Date,
 
+		/// <summary>A date nanos value</summary>
+		[EnumMember(Value = "date_nanos")] DateNanos,
+
 		/// <summary>An IP value</summary>
 		[EnumMember(Value = "ip")] Ip,
 

src/Nest/CommonOptions/DateMath/DateMath.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Text;
 using System.Text.RegularExpressions;
 using Elasticsearch.Net.Extensions;
@@ -109,21 +110,50 @@ public override string ToString()
 		}
 
 		/// <summary>
-		/// Formats a <see cref="DateTime"/> to have a minimum of 3 decimal places if there
-		/// are sub second values
+		/// Formats a <see cref="DateTime"/> to have a minimum of 3 decimal places if there are sub second values
 		/// </summary>
-		/// Fixes bug in Elasticsearch: https://github.com/elastic/elasticsearch/pull/41871
 		private static string ToMinThreeDecimalPlaces(DateTime dateTime)
 		{
-			var format = dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFF");
+			var builder = StringBuilderCache.Acquire(33);
+			var format = dateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss.FFFFFFF", CultureInfo.InvariantCulture);
+			builder.Append(format);
 
+			// Fixes bug in Elasticsearch: https://github.com/elastic/elasticsearch/pull/41871
 			if (format.Length > 20 && format.Length < 23)
 			{
 				var diff = 23 - format.Length;
-				return $"{format}{new string('0', diff)}";
+				for (int i = 0; i < diff; i++)
+					builder.Append('0');
 			}
 
-			return format;
+			switch (dateTime.Kind)
+			{
+				case DateTimeKind.Local:
+					var offset = TimeZoneInfo.Local.GetUtcOffset(dateTime);
+					if (offset >= TimeSpan.Zero)
+						builder.Append('+');
+					else
+					{
+						builder.Append('-');
+						offset = offset.Negate();
+					}
+
+					AppendTwoDigitNumber(builder, offset.Hours);
+					builder.Append(':');
+					AppendTwoDigitNumber(builder, offset.Minutes);
+					break;
+				case DateTimeKind.Utc:
+					builder.Append('Z');
+					break;
+			}
+
+			return StringBuilderCache.GetStringAndRelease(builder);
+		}
+
+		private static void AppendTwoDigitNumber(StringBuilder result, int val)
+		{
+			result.Append((char)('0' + (val / 10)));
+			result.Append((char)('0' + (val % 10)));
 		}
 	}
 

src/Nest/Mapping/DynamicTemplate/SingleMapping.cs

@@ -18,6 +18,9 @@ public IProperty Completion(Func<CompletionPropertyDescriptor<T>, ICompletionPro
 		public IProperty Date(Func<DatePropertyDescriptor<T>, IDateProperty> selector) =>
 			selector?.Invoke(new DatePropertyDescriptor<T>());
 
+		public IProperty DateNanos(Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector) =>
+			selector?.Invoke(new DateNanosPropertyDescriptor<T>());
+
 		public IProperty DateRange(Func<DateRangePropertyDescriptor<T>, IDateRangeProperty> selector) =>
 			selector?.Invoke(new DateRangePropertyDescriptor<T>());
 
@@ -247,6 +250,30 @@ public IProperty Scalar(Expression<Func<T, IEnumerable<DateTimeOffset?>>> field,
 		) =>
 			selector.InvokeOrDefault(new DatePropertyDescriptor<T>().Name(field));
 
+		public IProperty ScalarNanos(Expression<Func<T, DateTime>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null) =>
+			selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T, DateTime?>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null) =>
+			selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T, IEnumerable<DateTime>>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null) =>
+			selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T, IEnumerable<DateTime?>>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null) =>
+			selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T,DateTimeOffset>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null) =>
+			selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T, DateTimeOffset?>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null) =>
+			selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T, IEnumerable<DateTimeOffset>>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null
+		) => selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
+		public IProperty ScalarNanos(Expression<Func<T, IEnumerable<DateTimeOffset?>>> field, Func<DateNanosPropertyDescriptor<T>, IDateNanosProperty> selector = null
+		) => selector.InvokeOrDefault(new DateNanosPropertyDescriptor<T>().Name(field));
+
 		public IProperty Scalar(Expression<Func<T, bool>> field, Func<BooleanPropertyDescriptor<T>, IBooleanProperty> selector = null) =>
 			selector.InvokeOrDefault(new BooleanPropertyDescriptor<T>().Name(field));