Skip to content

Commit bd35d28

Browse files
stevejgordonstevejgordon
authored
Implement runtime fields on searches (#5259) (#5267)
* Implement runtime fields on searches * Add runtime fields documentation (cherry picked from commit caa6920)
1 parent f6e0cc0 commit bd35d28

File tree

12 files changed

+629
-31
lines changed

12 files changed

+629
-31
lines changed

‎docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,3 +371,59 @@ As demonstrated, by calling `.AutoMap()` inside of the `.Nested<Employee>` mappi
371371
`Employee` nested properties and again, override any inferred mapping from the automapping process,
372372
through manual mapping
373373

374+
[[mapping-runtime-fields]]
375+
==== Mapping runtime fields
376+
377+
A {ref_current}/runtime.html[runtime field] is a field that is evaluated at query time. Runtime fields may
378+
be defined in the mapping of an index.
379+
380+
In this example, we'll define a `CompanyRuntimeFields` class with a single property which we may then use in
381+
the strongly-typed runtime field mapping.
382+
383+
[source,csharp]
384+
----
385+
public class CompanyRuntimeFields
386+
{
387+
public string BirthDayOfWeek { get; set; }
388+
}
389+
390+
var createIndexResponse = _client.Indices.Create("myindex", c => c
391+
.Map<Company>(m => m
392+
.RuntimeFields<CompanyRuntimeFields>(rtf => rtf <1>
393+
.RuntimeField(f => f.BirthDayOfWeek, FieldType.Keyword, f => f.Script("emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"))) <2>
394+
)
395+
);
396+
----
397+
<1> Use the `CompanyRuntimeFields` class as the generic argument
398+
<2> Use the `BirthDayOfWeek` property as the runtime field name
399+
400+
[source,javascript]
401+
----
402+
{
403+
"mappings": {
404+
"runtime": {
405+
"birthDayOfWeek": {
406+
"type": "keyword",
407+
"script": {
408+
"lang": "painless",
409+
"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
410+
}
411+
}
412+
}
413+
}
414+
}
415+
----
416+
417+
It's not necessary to define a type for the runtime field mapping. Runtime fields can optionally be defined
418+
by providing a `string` name.
419+
420+
[source,csharp]
421+
----
422+
createIndexResponse = _client.Indices.Create("myindex", c => c
423+
.Map<Company>(m => m
424+
.RuntimeFields(rtf => rtf
425+
.RuntimeField("birthDayOfWeek", FieldType.Keyword, f => f.Script("emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))")))
426+
)
427+
);
428+
----
429+

‎docs/high-level.asciidoc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ include::search/scrolling-documents.asciidoc[]
211211

212212
include::{output-dir}/covariant-hits/covariant-search-results.asciidoc[]
213213

214+
include::search/searching-runtime-fields.asciidoc[]
215+
214216
[[aggregations]]
215217
== Aggregations
216218

‎docs/search/searching-runtime-fields.asciidoc‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/7.11
2+
3+
:github: https://github.com/elastic/elasticsearch-net
4+
5+
:nuget: https://www.nuget.org/packages
6+
7+
////
8+
IMPORTANT NOTE
9+
==============
10+
This file has been generated from https://github.com/elastic/elasticsearch-net/tree/7.x/src/Tests/Tests/Search/SearchingRuntimeFields.doc.cs.
11+
If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
12+
please modify the original csharp file found at the link and submit the PR with that change. Thanks!
13+
////
14+
15+
[[searching-runtime-fields]]
16+
=== Searching runtime fields
17+
18+
Runtime fields can be returned with search requests by specifying the fields using `.Fields`
19+
on the search request.
20+
21+
[WARNING]
22+
--
23+
This functionality is in beta and is subject to change. The design and code is less mature
24+
than official GA features and is being provided as-is with no warranties. Beta features
25+
are not subject to the support SLA of official GA features.
26+
27+
--
28+
29+
[source,csharp]
30+
----
31+
var searchResponse = _client.Search<Project>(s => s
32+
.Query(q => q
33+
.MatchAll()
34+
)
35+
.Fields<ProjectRuntimeFields>(fs => fs
36+
.Field(f => f.StartedOnDayOfWeek)
37+
.Field(f => f.ThirtyDaysFromStarted, format: DateFormat.basic_date)
38+
)
39+
);
40+
----
41+
42+
which serializes to the following JSON
43+
44+
[source,javascript]
45+
----
46+
{
47+
"query": {
48+
"match_all": {}
49+
},
50+
"fields": [
51+
"runtime_started_on_day_of_week",
52+
{
53+
"field": "runtime_thirty_days_after_started",
54+
"format": "basic_date"
55+
}
56+
]
57+
}
58+
----
59+
60+
The previous example used the Fluent API to express the query. NEST also exposes an
61+
Object Initializer syntax to compose queries
62+
63+
[source,csharp]
64+
----
65+
var searchRequest = new SearchRequest<Project>
66+
{
67+
Query = new MatchAllQuery(),
68+
Fields = Infer.Field<ProjectRuntimeFields>(p => p.StartedOnDayOfWeek) <1>
69+
.And<ProjectRuntimeFields>(p => p.ThirtyDaysFromStarted, format: DateFormat.basic_date) <2>
70+
};
71+
72+
searchResponse = _client.Search<Project>(searchRequest);
73+
----
74+
<1> Here we infer the field name from a property on a POCO class
75+
<2> For runtime fields which return a date, a format may be specified.
76+
77+
==== Defining runtime fields
78+
79+
You may define runtime fields that exist only as part of a query by specifying `.RuntimeFields` on
80+
the search request. You may return this field using `.Fields` or use it for an aggregation.
81+
82+
[source,csharp]
83+
----
84+
var searchResponse = _client.Search<Project>(s => s
85+
.Query(q => q
86+
.MatchAll()
87+
)
88+
.Fields<ProjectRuntimeFields>(fs => fs
89+
.Field(f => f.StartedOnDayOfWeek)
90+
.Field(f => f.ThirtyDaysFromStarted, format: DateFormat.basic_date)
91+
.Field("search_runtime_field")
92+
)
93+
.RuntimeFields(rtf => rtf.RuntimeField("search_runtime_field", FieldType.Keyword, r => r
94+
.Script("if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}")))
95+
);
96+
----
97+
98+
which yields the following query JSON
99+
100+
[source,javascript]
101+
----
102+
{
103+
"query": {
104+
"match_all": {}
105+
},
106+
"fields": [
107+
"runtime_started_on_day_of_week",
108+
{
109+
"field": "runtime_thirty_days_after_started",
110+
"format": "basic_date"
111+
},
112+
"search_runtime_field"
113+
],
114+
"runtime_mappings": {
115+
"search_runtime_field": {
116+
"script": {
117+
"lang": "painless",
118+
"source": "if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}"
119+
},
120+
"type": "keyword"
121+
}
122+
}
123+
}
124+
----
125+
126+
The previous example used the Fluent API to express the query. Here is the same query using the
127+
Object Initializer syntax.
128+
129+
[source,csharp]
130+
----
131+
var searchRequest = new SearchRequest<Project>
132+
{
133+
Query = new MatchAllQuery(),
134+
Fields = Infer.Field<ProjectRuntimeFields>(p => p.StartedOnDayOfWeek)
135+
.And<ProjectRuntimeFields>(p => p.ThirtyDaysFromStarted, format: DateFormat.basic_date)
136+
.And("search_runtime_field"),
137+
RuntimeFields = new RuntimeFields
138+
{
139+
{ "search_runtime_field", new RuntimeField
140+
{
141+
Type = FieldType.Keyword,
142+
Script = new PainlessScript("if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}")
143+
}
144+
}
145+
}
146+
};
147+
148+
searchResponse = _client.Search<Project>(searchRequest);
149+
----
150+

‎src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,13 @@ public PutMappingDescriptor<TDocument> RoutingField(Func<RoutingFieldDescriptor<
155155
Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor<TDocument>()));
156156

157157
/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
158-
public PutMappingDescriptor<TDocument> RuntimeFields(Func<RuntimeFieldsDescriptor, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
159-
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor())?.Value);
158+
public PutMappingDescriptor<TDocument> RuntimeFields(Func<RuntimeFieldsDescriptor<TDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
159+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TDocument>())?.Value);
160160

161+
/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
162+
public PutMappingDescriptor<TDocument> RuntimeFields<TSource>(Func<RuntimeFieldsDescriptor<TSource>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TSource : class =>
163+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TSource>())?.Value);
164+
161165
/// <inheritdoc cref="ITypeMapping.FieldNamesField" />
162166
public PutMappingDescriptor<TDocument> FieldNamesField(Func<FieldNamesFieldDescriptor<TDocument>, IFieldNamesField> fieldNamesFieldSelector) =>
163167
Assign(fieldNamesFieldSelector, (a, v) => a.FieldNamesField = v.Invoke(new FieldNamesFieldDescriptor<TDocument>()));

‎src/Nest/Mapping/RuntimeFields/RuntimeFields.cs‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,40 @@
44

55
using System;
66
using System.Collections.Generic;
7+
using System.Linq.Expressions;
78
using Nest.Utf8Json;
89

910
namespace Nest
1011
{
11-
[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<RuntimeFields, IRuntimeFields, string, IRuntimeField>))]
12-
public interface IRuntimeFields : IIsADictionary<string, IRuntimeField> { }
12+
[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<RuntimeFields, IRuntimeFields, Field, IRuntimeField>))]
13+
public interface IRuntimeFields : IIsADictionary<Field, IRuntimeField> { }
1314

14-
public class RuntimeFields : IsADictionaryBase<string, IRuntimeField>, IRuntimeFields
15+
public class RuntimeFields : IsADictionaryBase<Field, IRuntimeField>, IRuntimeFields
1516
{
1617
public RuntimeFields() { }
1718

18-
public RuntimeFields(IDictionary<string, IRuntimeField> container) : base(container) { }
19+
public RuntimeFields(IDictionary<Field, IRuntimeField> container) : base(container) { }
1920

20-
public RuntimeFields(Dictionary<string, IRuntimeField> container) : base(container) { }
21+
public RuntimeFields(Dictionary<Field, IRuntimeField> container) : base(container) { }
2122

22-
public void Add(string name, IRuntimeField runtimeField) => BackingDictionary.Add(name, runtimeField);
23+
public void Add(Field name, IRuntimeField runtimeField) => BackingDictionary.Add(name, runtimeField);
2324
}
2425

25-
public class RuntimeFieldsDescriptor
26-
: IsADictionaryDescriptorBase<RuntimeFieldsDescriptor, RuntimeFields, string, IRuntimeField>
26+
public class RuntimeFieldsDescriptor<TDocument>
27+
: IsADictionaryDescriptorBase<RuntimeFieldsDescriptor<TDocument>, RuntimeFields, Field, IRuntimeField> where TDocument : class
2728
{
2829
public RuntimeFieldsDescriptor() : base(new RuntimeFields()) { }
2930

30-
public RuntimeFieldsDescriptor RuntimeField(string name, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
31+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(string name, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
3132
Assign(name, selector?.Invoke(new RuntimeFieldDescriptor(type)));
3233

33-
public RuntimeFieldsDescriptor RuntimeField(string name, FieldType type) =>
34+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(Expression<Func<TDocument, Field>> field, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
35+
Assign(field, selector?.Invoke(new RuntimeFieldDescriptor(type)));
36+
37+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(string name, FieldType type) =>
3438
Assign(name, new RuntimeFieldDescriptor(type));
39+
40+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(Expression<Func<TDocument, Field>> field, FieldType type) =>
41+
Assign(field, new RuntimeFieldDescriptor(type));
3542
}
3643
}

‎src/Nest/Mapping/TypeMapping.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -269,9 +269,12 @@ public TypeMappingDescriptor<T> DisableIndexField(bool? disabled = true) =>
269269
public TypeMappingDescriptor<T> RoutingField(Func<RoutingFieldDescriptor<T>, IRoutingField> routingFieldSelector) =>
270270
Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor<T>()));
271271

272+
public TypeMappingDescriptor<T> RuntimeFields(Func<RuntimeFieldsDescriptor<T>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
273+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<T>())?.Value);
274+
272275
/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
273-
public TypeMappingDescriptor<T> RuntimeFields(Func<RuntimeFieldsDescriptor, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
274-
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor())?.Value);
276+
public TypeMappingDescriptor<T> RuntimeFields<TDocument>(Func<RuntimeFieldsDescriptor<TDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TDocument : class =>
277+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TDocument>())?.Value);
275278

276279
/// <inheritdoc cref="ITypeMapping.FieldNamesField" />
277280
public TypeMappingDescriptor<T> FieldNamesField(Func<FieldNamesFieldDescriptor<T>, IFieldNamesField> fieldNamesFieldSelector) =>

‎src/Nest/Search/Search/SearchRequest.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ public partial interface ISearchRequest : ITypedSearchRequest
3939
[DataMember(Name = "explain")]
4040
bool? Explain { get; set; }
4141

42+
/// <summary>
43+
/// BETA: Allows for retrieving a list of document fields in the search response.
44+
/// <para>This functionality is in beta and is subject to change. </para>
45+
/// </summary>
46+
[DataMember(Name = "fields")]
47+
Fields Fields { get; set; }
48+
4249
/// <summary>
4350
/// The starting from index of the hits to return. Defaults to 0.
4451
/// </summary>
@@ -173,6 +180,12 @@ public partial interface ISearchRequest : ITypedSearchRequest
173180
/// </summary>
174181
[DataMember(Name = "pit")]
175182
IPointInTime PointInTime { get; set; }
183+
184+
/// <summary>
185+
/// Specifies runtime fields which exist only as part of the query.
186+
/// </summary>
187+
[DataMember(Name = "runtime_mappings")]
188+
IRuntimeFields RuntimeFields { get; set; }
176189
}
177190

178191
[ReadAs(typeof(SearchRequest<>))]
@@ -199,6 +212,8 @@ public partial class SearchRequest
199212
/// <inheritdoc />
200213
public bool? Explain { get; set; }
201214
/// <inheritdoc />
215+
public Fields Fields { get; set; }
216+
/// <inheritdoc />
202217
public int? From { get; set; }
203218
/// <inheritdoc />
204219
public IHighlight Highlight { get; set; }
@@ -243,6 +258,8 @@ public partial class SearchRequest
243258
public bool? Version { get; set; }
244259
/// <inheritdoc />
245260
public IPointInTime PointInTime { get; set; }
261+
/// <inheritdoc />
262+
public IRuntimeFields RuntimeFields { get; set; }
246263

247264
protected override HttpMethod? DynamicHttpMethod =>
248265
RequestState.RequestParameters?.ContainsQueryString("source") == true
@@ -286,6 +303,7 @@ public partial class SearchDescriptor<TInferDocument> where TInferDocument : cla
286303
IFieldCollapse ISearchRequest.Collapse { get; set; }
287304
Fields ISearchRequest.DocValueFields { get; set; }
288305
bool? ISearchRequest.Explain { get; set; }
306+
Fields ISearchRequest.Fields { get; set; }
289307
int? ISearchRequest.From { get; set; }
290308
IHighlight ISearchRequest.Highlight { get; set; }
291309
IDictionary<IndexName, double> ISearchRequest.IndicesBoost { get; set; }
@@ -308,6 +326,7 @@ public partial class SearchDescriptor<TInferDocument> where TInferDocument : cla
308326
bool? ISearchRequest.TrackTotalHits { get; set; }
309327
bool? ISearchRequest.Version { get; set; }
310328
IPointInTime ISearchRequest.PointInTime { get; set; }
329+
IRuntimeFields ISearchRequest.RuntimeFields { get; set; }
311330

312331
protected sealed override void RequestDefaults(SearchRequestParameters parameters) => TypedKeys();
313332

@@ -336,6 +355,17 @@ public SearchDescriptor<TInferDocument> Source(Func<SourceFilterDescriptor<TInfe
336355
/// <inheritdoc cref="ISearchRequest.Size" />
337356
public SearchDescriptor<TInferDocument> Take(int? take) => Size(take);
338357

358+
/// <inheritdoc cref="ISearchRequest.Fields" />
359+
public SearchDescriptor<TInferDocument> Fields(Func<FieldsDescriptor<TInferDocument>, IPromise<Fields>> fields) =>
360+
Assign(fields, (a, v) => a.Fields = v?.Invoke(new FieldsDescriptor<TInferDocument>())?.Value);
361+
362+
/// <inheritdoc cref="ISearchRequest.Fields" />
363+
public SearchDescriptor<TInferDocument> Fields<TSource>(Func<FieldsDescriptor<TSource>, IPromise<Fields>> fields) where TSource : class =>
364+
Assign(fields, (a, v) => a.Fields = v?.Invoke(new FieldsDescriptor<TSource>())?.Value);
365+
366+
/// <inheritdoc cref="ISearchRequest.Fields" />
367+
public SearchDescriptor<TInferDocument> Fields(Fields fields) => Assign(fields, (a, v) => a.DocValueFields = v);
368+
339369
/// <inheritdoc cref="ISearchRequest.From" />
340370
public SearchDescriptor<TInferDocument> From(int? from) => Assign(from, (a, v) => a.From = v);
341371

@@ -477,6 +507,14 @@ public SearchDescriptor<TInferDocument> PointInTime(string pitId)
477507
public SearchDescriptor<TInferDocument> PointInTime(string pitId, Func<PointInTimeDescriptor, IPointInTime> pit) =>
478508
Assign(pit, (a, v) => a.PointInTime = v?.Invoke(new PointInTimeDescriptor(pitId)));
479509

510+
/// <inheritdoc cref="ISearchRequest.RuntimeFields" />
511+
public SearchDescriptor<TInferDocument> RuntimeFields(Func<RuntimeFieldsDescriptor<TInferDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
512+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TInferDocument>())?.Value);
513+
514+
/// <inheritdoc cref="ISearchRequest.RuntimeFields" />
515+
public SearchDescriptor<TInferDocument> RuntimeFields<TSource>(Func<RuntimeFieldsDescriptor<TSource>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TSource : class =>
516+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TSource>())?.Value);
517+
480518
protected override string ResolveUrl(RouteValues routeValues, IConnectionSettingsValues settings)
481519
{
482520
if (Self.PointInTime is object && !string.IsNullOrEmpty(Self.PointInTime.Id) && routeValues.ContainsKey("index"))

0 commit comments

Comments
 (0)