diff --git a/docs/changelog/89440.yaml b/docs/changelog/89440.yaml new file mode 100644 index 0000000000000..816c43467375a --- /dev/null +++ b/docs/changelog/89440.yaml @@ -0,0 +1,5 @@ +pr: 89440 +summary: Add source fallback support for date and `date_nanos` mapped types +area: Mapping +type: enhancement +issues: [] diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 979f0a1cdf7df..053a12ae3ba72 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -14,8 +14,14 @@ setup: doc_values: false date: type: date + date_no_doc_values: + type: date + doc_values: false nanos: type: date_nanos + nanos_no_doc_values: + type: date_nanos + doc_values: false geo_point: type: geo_point geo_point_no_doc_values: @@ -93,7 +99,9 @@ setup: boolean: true boolean_no_doc_values: true date: 2017-01-01T12:11:12 + date_no_doc_values: 2017-01-01T12:11:12 nanos: 2015-01-01T12:10:30.123456789Z + nanos_no_doc_values: 2015-01-01T12:10:30.123456789Z geo_point: 41.12,-71.34 geo_point_no_doc_values: 41.12,-71.34 ip: 192.168.0.19 @@ -136,7 +144,9 @@ setup: boolean_no_doc_values: [true, false, true] ip: ["10.1.2.3", "2001:db8::2:1"] date: [2017-01-01T12:11:12, 2018-01-01T12:11:12] + date_no_doc_values: [2017-01-01T12:11:12, 2018-01-01T12:11:12] nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z] + nanos_no_doc_values: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z] geo_point: [[-71.34,41.12],[60.32,21.25]] geo_point_no_doc_values: [[60.32,21.25],[-71.34,41.12]] keyword: ["one string", "another string"] @@ -692,6 +702,244 @@ setup: source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('nanos')) times.add(zdt); times" - match: { hits.hits.0.fields.field: ["2015-01-01T12:10:30.123456789Z", "2015-01-01T12:10:30.987654321Z"] } +--- +"date_no_doc_values": + - skip: + features: "warnings" + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc.date_no_doc_values.get(0)" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc.date_no_doc_values.value" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "field('date_no_doc_values').get(null)" + - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('date_no_doc_values', null)" + - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: 1 } } + script_fields: + field: + script: + source: "field('date_no_doc_values').get(null).getMillis()" + - match: { hits.hits.0.fields.field.0: 1483272672000 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: 1 } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('date_no_doc_values', null).getMillis()" + - match: { hits.hits.0.fields.field.0: 1483272672000 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: 1 } } + script_fields: + field: + script: + source: "field('date_no_doc_values').get(null).millis" + - match: { hits.hits.0.fields.field.0: 1483272672000 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: 1 } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('date_no_doc_values', null).millis" + - match: { hits.hits.0.fields.field.0: 1483272672000 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "field('date_no_doc_values').get(ZonedDateTime.parse('2018-01-01T12:11:12.000Z'))" + - match: { hits.hits.0.fields.field.0: '2018-01-01T12:11:12.000Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('date_no_doc_values', ZonedDateTime.parse('2018-01-01T12:11:12.000Z'))" + - match: { hits.hits.0.fields.field.0: '2018-01-01T12:11:12.000Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "field('nanos_no_doc_values').get(null)" + - match: { hits.hits.0.fields.field.0: '2015-01-01T12:10:30.123456789Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('nanos_no_doc_values', null)" + - match: { hits.hits.0.fields.field.0: '2015-01-01T12:10:30.123456789Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "field('nanos_no_doc_values').get(ZonedDateTime.parse('2016-01-01T12:10:30.123Z'))" + - match: { hits.hits.0.fields.field.0: '2016-01-01T12:10:30.123Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('nanos_no_doc_values', ZonedDateTime.parse('2016-01-01T12:10:30.123Z'))" + - match: { hits.hits.0.fields.field.0: '2016-01-01T12:10:30.123Z' } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "field('nanos_no_doc_values').get(null).getNano()" + - match: { hits.hits.0.fields.field.0: 123456789 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('nanos_no_doc_values', null).getNano()" + - match: { hits.hits.0.fields.field.0: 123456789 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "field('nanos_no_doc_values').get(ZonedDateTime.parse('2016-01-01T12:10:30.123Z')).getNano()" + - match: { hits.hits.0.fields.field.0: 123000000 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "field('date_no_doc_values').get(1, null)" + - match: { hits.hits.0.fields.field.0: "2018-01-01T12:11:12.000Z" } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "field('nanos_no_doc_values').get(1, null)" + - match: { hits.hits.0.fields.field.0: "2015-01-01T12:10:30.987654321Z" } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('date_no_doc_values')) times.add(zdt); times" + - match: { hits.hits.0.fields.field: ["2017-01-01T12:11:12.000Z", "2018-01-01T12:11:12.000Z"] } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('nanos_no_doc_values')) times.add(zdt); times" + - match: { hits.hits.0.fields.field: ["2015-01-01T12:10:30.123456789Z", "2015-01-01T12:10:30.987654321Z"] } + --- "geo_point": - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index c3728b8205025..f61cca78ec2ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.DateRangeIncludingNowQuery; import org.elasticsearch.index.query.QueryRewriteContext; @@ -64,6 +65,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -519,6 +521,17 @@ public String parseSourceValue(Object value) { }; } + // returns a Long to support source fallback which emulates numeric doc values for dates + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { + return new SourceValueFetcher(sourcePaths, nullValue) { + @Override + public Long parseSourceValue(Object value) { + String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString(); + return parse(date); + } + }; + } + private String format(long timestamp, DateFormatter formatter) { ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); return formatter.format(dateTime); @@ -750,8 +763,34 @@ public Function pointReaderIfPossible() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - failIfNoDocValues(); - return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType(), resolution.getDefaultToScriptFieldFactory()); + FielddataOperation operation = fieldDataContext.fielddataOperation(); + + if (operation == FielddataOperation.SEARCH) { + failIfNoDocValues(); + } + + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { + return new SortedNumericIndexFieldData.Builder( + name(), + resolution.numericType(), + resolution.getDefaultToScriptFieldFactory() + ); + } + + if (operation == FielddataOperation.SCRIPT) { + SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); + Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + + return new SourceValueFetcherSortedNumericIndexFieldData.Builder( + name(), + resolution.numericType().getValuesSourceType(), + sourceValueFetcher(sourcePaths), + searchLookup.source(), + resolution.getDefaultToScriptFieldFactory() + ); + } + + throw new IllegalStateException("unknown field data operation [" + operation.name() + "]"); } @Override