Add hard_bounds for histograms (#5098)

* Add hard_bounds for histograms
* Apply date_optional_time format for hard_bounds
* Add new hard_bounds tests

As with extended_bounds, we append the date_optional_time format
to the format starting on the aggregation. This avoids parsing errors on
the server.

Includes an update to the documentation which we be generated and 
added in a subsequent PR.

Author: stevejgordon

src/Nest/Aggregations/Bucket/DateHistogram/DateHistogramAggregation.cs

@@ -24,6 +24,13 @@ public interface IDateHistogramAggregation : IBucketAggregation
 		[DataMember(Name ="extended_bounds")]
 		ExtendedBounds<DateMath> ExtendedBounds { get; set; }
 
+		/// <summary>
+		/// The hard_bounds is a counterpart of extended_bounds and can limit the range of buckets in the histogram.
+		/// It is particularly useful in the case of open data ranges that can result in a very large number of buckets.
+		/// </summary>
+		[DataMember(Name = "hard_bounds")]
+		HardBounds<DateMath> HardBounds { get; set; }
+
 		/// <summary>
 		/// The field to target
 		/// </summary>
@@ -104,20 +111,21 @@ public DateHistogramAggregation(string name) : base(name) { }
 		/// <inheritdoc />
 		public ExtendedBounds<DateMath> ExtendedBounds { get; set; }
 		/// <inheritdoc />
+		public HardBounds<DateMath> HardBounds { get; set; }
+		/// <inheritdoc />
 		public Field Field { get; set; }
 
 		/// <inheritdoc />
 		public string Format
 		{
 			get => !string.IsNullOrEmpty(_format) &&
 				!_format.Contains("date_optional_time") &&
-				(ExtendedBounds != null || Missing.HasValue)
+				(ExtendedBounds != null || HardBounds != null || Missing.HasValue)
 					? _format + "||date_optional_time"
 					: _format;
 			set => _format = value;
 		}
 
-
 		[Obsolete("Deprecated in version 7.2.0, use CalendarInterval or FixedInterval instead")]
 		public Union<DateInterval, Time> Interval { get; set; }
 		/// <inheritdoc />
@@ -148,14 +156,15 @@ public class DateHistogramAggregationDescriptor<T>
 		private string _format;
 
 		ExtendedBounds<DateMath> IDateHistogramAggregation.ExtendedBounds { get; set; }
+		HardBounds<DateMath> IDateHistogramAggregation.HardBounds { get; set; }
 		Field IDateHistogramAggregation.Field { get; set; }
 
 		//see: https://github.com/elastic/elasticsearch/issues/9725
 		string IDateHistogramAggregation.Format
 		{
 			get => !string.IsNullOrEmpty(_format) &&
 				!_format.Contains("date_optional_time") &&
-				(Self.ExtendedBounds != null || Self.Missing.HasValue)
+				(Self.ExtendedBounds != null || Self.HardBounds != null || Self.Missing.HasValue)
 					? _format + "||date_optional_time"
 					: _format;
 			set => _format = value;
@@ -229,6 +238,10 @@ public DateHistogramAggregationDescriptor<T> OrderDescending(string key) =>
 		public DateHistogramAggregationDescriptor<T> ExtendedBounds(DateMath min, DateMath max) =>
 			Assign(new ExtendedBounds<DateMath> { Minimum = min, Maximum = max }, (a, v) => a.ExtendedBounds = v);
 
+		/// <inheritdoc cref="IDateHistogramAggregation.HardBounds" />
+		public DateHistogramAggregationDescriptor<T> HardBounds(DateMath min, DateMath max) =>
+			Assign(new HardBounds<DateMath> { Minimum = min, Maximum = max }, (a, v) => a.HardBounds = v);
+
 		/// <inheritdoc cref="IDateHistogramAggregation.Missing" />
 		public DateHistogramAggregationDescriptor<T> Missing(DateTime? missing) => Assign(missing, (a, v) => a.Missing = v);
 	}

src/Nest/Aggregations/Bucket/Histogram/HardBounds.cs

@@ -0,0 +1,17 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Runtime.Serialization;
+
+namespace Nest
+{
+	public class HardBounds<T>
+	{
+		[DataMember(Name = "max")]
+		public T Maximum { get; set; }
+
+		[DataMember(Name = "min")]
+		public T Minimum { get; set; }
+	}
+}

src/Nest/Aggregations/Bucket/Histogram/HistogramAggregation.cs

@@ -16,6 +16,9 @@ public interface IHistogramAggregation : IBucketAggregation
 		[DataMember(Name ="extended_bounds")]
 		ExtendedBounds<double> ExtendedBounds { get; set; }
 
+		[DataMember(Name = "hard_bounds")]
+		HardBounds<double> HardBounds { get; set; }
+
 		[DataMember(Name ="field")]
 		Field Field { get; set; }
 
@@ -45,6 +48,7 @@ internal HistogramAggregation() { }
 		public HistogramAggregation(string name) : base(name) { }
 
 		public ExtendedBounds<double> ExtendedBounds { get; set; }
+		public HardBounds<double> HardBounds { get; set; }
 		public Field Field { get; set; }
 		public double? Interval { get; set; }
 		public int? MinimumDocumentCount { get; set; }
@@ -61,6 +65,7 @@ public class HistogramAggregationDescriptor<T>
 		where T : class
 	{
 		ExtendedBounds<double> IHistogramAggregation.ExtendedBounds { get; set; }
+		HardBounds<double> IHistogramAggregation.HardBounds { get; set; }
 		Field IHistogramAggregation.Field { get; set; }
 
 		double? IHistogramAggregation.Interval { get; set; }
@@ -100,6 +105,9 @@ public HistogramAggregationDescriptor<T> OrderDescending(string key) =>
 		public HistogramAggregationDescriptor<T> ExtendedBounds(double min, double max) =>
 			Assign(new ExtendedBounds<double> { Minimum = min, Maximum = max }, (a, v) => a.ExtendedBounds = v);
 
+		public HistogramAggregationDescriptor<T> HardBounds(double min, double max) =>
+			Assign(new HardBounds<double> { Minimum = min, Maximum = max }, (a, v) => a.HardBounds = v);
+
 		public HistogramAggregationDescriptor<T> Offset(double? offset) => Assign(offset, (a, v) => a.Offset = v);
 
 		public HistogramAggregationDescriptor<T> Missing(double? missing) => Assign(missing, (a, v) => a.Missing = v);

tests/Tests/Aggregations/Bucket/DateHistogram/DateHistogramAggregationUsageTests.cs

@@ -21,7 +21,7 @@ namespace Tests.Aggregations.Bucket.DateHistogram
 	 * From a functionality perspective, this histogram supports the same features as the normal histogram.
 	 * The main difference is that the interval can be specified by date/time expressions.
 	 *
-	 * NOTE: When specifying a `format` **and** `extended_bounds` or `missing`, in order for Elasticsearch to be able to parse
+	 * NOTE: When specifying a `format` **and** `extended_bounds`, `hard_bounds` or `missing`, in order for Elasticsearch to be able to parse
 	 * the serialized `DateTime` of `extended_bounds` or `missing` correctly, the `date_optional_time` format is included
 	 * as part of the `format` value.
 	 *
@@ -41,7 +41,7 @@ public DateHistogramAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage
 					field = "startedOn",
 					calendar_interval = "month",
 					min_doc_count = 2,
-					format = "yyyy-MM-dd'T'HH:mm:ss||date_optional_time", //<1> Note the inclusion of `date_optional_time` to `format`
+					format = "yyyy-MM-dd'T'HH:mm:ss||date_optional_time", // <1> Note the inclusion of `date_optional_time` to `format`
 					order = new { _count = "asc" },
 					extended_bounds = new
 					{
@@ -208,4 +208,82 @@ protected override void ExpectResponse(ISearchResponse<Project> response)
 			}
 		}
 	}
+
+	// hide
+	[SkipVersion("<7.10.0", "hard_bounds introduced in 7.10.0")]
+	public class DateHistogramAggregationWithHardBoundsUsageTests : ProjectsOnlyAggregationUsageTestBase
+	{
+		private readonly DateTime _hardBoundsMinimum;
+		private readonly DateTime _hardBoundsMaximum;
+		
+		public DateHistogramAggregationWithHardBoundsUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage)
+		{
+			// Note: If these tests are run against an existing node, and seeding is not forced, it's possible the
+			// dates used will not appear in the index and result in no buckets being returned. The test will still
+			// pass if this is the case. For best results locally, force a reseed. This is not an issue in CI.
+
+			var projects = Project.Projects.OrderBy(p => p.StartedOn).Skip(2).Take(5).ToArray();
+			
+			_hardBoundsMinimum = projects.Min(p => p.StartedOn.Date);
+			_hardBoundsMaximum = projects.Max(p => p.StartedOn.Date);
+		}
+
+		protected override object AggregationJson => new
+		{
+			projects_started_per_day = new
+			{
+				date_histogram = new
+				{
+					field = "startedOn",
+					calendar_interval = "day",
+					format = "yyyy-MM-dd'T'HH:mm:ss||date_optional_time",
+					min_doc_count = 1,
+					hard_bounds = new
+					{
+						min = _hardBoundsMinimum,
+						max = _hardBoundsMaximum
+					},
+					order = new { _key = "asc" },
+				}
+			}
+		};
+
+#pragma warning disable 618, 612
+		protected override Func<AggregationContainerDescriptor<Project>, IAggregationContainer> FluentAggs => a => a
+			.DateHistogram("projects_started_per_day", date => date
+				.Field(p => p.StartedOn)
+				.Format("yyyy-MM-dd'T'HH:mm:ss")
+				.CalendarInterval(DateInterval.Day)
+				.HardBounds(_hardBoundsMinimum, _hardBoundsMaximum)
+				.MinimumDocumentCount(1)
+				.Order(HistogramOrder.KeyAscending)
+			);
+
+		protected override AggregationDictionary InitializerAggs =>
+			new DateHistogramAggregation("projects_started_per_day")
+			{
+				Field = Field<Project>(p => p.StartedOn),
+				Format = "yyyy-MM-dd'T'HH:mm:ss",
+				CalendarInterval = DateInterval.Day,
+				HardBounds = new HardBounds<DateMath>
+				{
+					Minimum = _hardBoundsMinimum,
+					Maximum = _hardBoundsMaximum
+				},
+				MinimumDocumentCount = 1,
+				Order = HistogramOrder.KeyAscending
+			};
+#pragma warning restore 618, 612
+
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			response.ShouldBeValid();
+			var dateHistogram = response.Aggregations.DateHistogram("projects_started_per_day");
+			dateHistogram.Should().NotBeNull();
+			dateHistogram.Buckets.Should().NotBeNull();
+
+			foreach (var date in dateHistogram.Buckets.Select(b => DateTime.Parse(b.KeyAsString)))
+				date.Should().BeOnOrAfter(_hardBoundsMinimum).And.BeOnOrBefore(_hardBoundsMaximum);
+		}
+	}
 }

tests/Tests/Aggregations/Bucket/Histogram/HistogramAggregationUsageTests.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System;
+using Elastic.Elasticsearch.Xunit.XunitPlumbing;
 using FluentAssertions;
 using Nest;
 using Tests.Core.Extensions;
@@ -31,6 +32,7 @@ public HistogramAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) :
 						_key = "desc"
 					},
 					offset = 1.1
+
 				}
 			}
 		};
@@ -65,4 +67,66 @@ protected override void ExpectResponse(ISearchResponse<Project> response)
 				item.DocCount.Should().BeGreaterThan(0);
 		}
 	}
+
+	// hide
+	[SkipVersion("<7.10.0", "hard_bounds introduced in 7.10.0")]
+	public class HistogramAggregationWithHardBoundsUsageTests : AggregationUsageTestBase
+	{
+		private const double HardBoundsMinimum = 100;
+		private const double HardBoundsMaximum = 300;
+
+		public HistogramAggregationWithHardBoundsUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { }
+
+		protected override object AggregationJson => new
+		{
+			commits = new
+			{
+				histogram = new
+				{
+					field = "numberOfCommits",
+					hard_bounds = new { min = HardBoundsMinimum, max = HardBoundsMaximum },
+					interval = 100.0,
+					min_doc_count = 1,
+					order = new
+					{
+						_key = "desc"
+					}
+				}
+			}
+		};
+
+		protected override Func<AggregationContainerDescriptor<Project>, IAggregationContainer> FluentAggs => a => a
+			.Histogram("commits", h => h
+				.Field(p => p.NumberOfCommits)
+				.Interval(100)
+				.MinimumDocumentCount(1)
+				.Order(HistogramOrder.KeyDescending)
+				.HardBounds(HardBoundsMinimum, HardBoundsMaximum)
+			);
+
+		protected override AggregationDictionary InitializerAggs =>
+			new HistogramAggregation("commits")
+			{
+				Field = Field<Project>(p => p.NumberOfCommits),
+				Interval = 100,
+				MinimumDocumentCount = 1,
+				Order = HistogramOrder.KeyDescending,
+				HardBounds = new HardBounds<double>
+				{
+					Minimum = HardBoundsMinimum,
+					Maximum = HardBoundsMaximum
+				}
+			};
+
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			response.ShouldBeValid();
+			var commits = response.Aggregations.Histogram("commits");
+			commits.Should().NotBeNull();
+			commits.Buckets.Should().NotBeNull();
+
+			foreach (var bucket in commits.Buckets)
+				bucket.Key.Should().BeGreaterOrEqualTo(HardBoundsMinimum).And.BeLessOrEqualTo(HardBoundsMaximum);
+		}
+	}
 }