Skip to content

Commit 058ea45

Browse files
jdconradjdconrad
authored
Add source fallback support for date and date_nanos mapped types (#89440)
This change adds source fallback support for date and date_nanos by using the existing SourceValueFetcherSortedNumericIndexFieldData to emulate doc values.
1 parent 22e1150 commit 058ea45

File tree

3 files changed

+294
-2
lines changed

3 files changed

+294
-2
lines changed

‎docs/changelog/89440.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 89440
2+
summary: Add source fallback support for date and `date_nanos` mapped types
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,14 @@ setup:
1414
doc_values: false
1515
date:
1616
type: date
17+
date_no_doc_values:
18+
type: date
19+
doc_values: false
1720
nanos:
1821
type: date_nanos
22+
nanos_no_doc_values:
23+
type: date_nanos
24+
doc_values: false
1925
geo_point:
2026
type: geo_point
2127
geo_point_no_doc_values:
@@ -93,7 +99,9 @@ setup:
9399
boolean: true
94100
boolean_no_doc_values: true
95101
date: 2017-01-01T12:11:12
102+
date_no_doc_values: 2017-01-01T12:11:12
96103
nanos: 2015-01-01T12:10:30.123456789Z
104+
nanos_no_doc_values: 2015-01-01T12:10:30.123456789Z
97105
geo_point: 41.12,-71.34
98106
geo_point_no_doc_values: 41.12,-71.34
99107
ip: 192.168.0.19
@@ -136,7 +144,9 @@ setup:
136144
boolean_no_doc_values: [true, false, true]
137145
ip: ["10.1.2.3", "2001:db8::2:1"]
138146
date: [2017-01-01T12:11:12, 2018-01-01T12:11:12]
147+
date_no_doc_values: [2017-01-01T12:11:12, 2018-01-01T12:11:12]
139148
nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z]
149+
nanos_no_doc_values: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z]
140150
geo_point: [[-71.34,41.12],[60.32,21.25]]
141151
geo_point_no_doc_values: [[60.32,21.25],[-71.34,41.12]]
142152
keyword: ["one string", "another string"]
@@ -692,6 +702,244 @@ setup:
692702
source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('nanos')) times.add(zdt); times"
693703
- match: { hits.hits.0.fields.field: ["2015-01-01T12:10:30.123456789Z", "2015-01-01T12:10:30.987654321Z"] }
694704

705+
---
706+
"date_no_doc_values":
707+
- skip:
708+
features: "warnings"
709+
710+
- do:
711+
catch: bad_request
712+
search:
713+
rest_total_hits_as_int: true
714+
body:
715+
query: { term: { _id: "1" } }
716+
script_fields:
717+
field:
718+
script:
719+
source: "doc.date_no_doc_values.get(0)"
720+
- match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" }
721+
722+
- do:
723+
catch: bad_request
724+
search:
725+
rest_total_hits_as_int: true
726+
body:
727+
query: { term: { _id: "1" } }
728+
script_fields:
729+
field:
730+
script:
731+
source: "doc.date_no_doc_values.value"
732+
- match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" }
733+
734+
- do:
735+
search:
736+
rest_total_hits_as_int: true
737+
body:
738+
query: { term: { _id: "1" } }
739+
script_fields:
740+
field:
741+
script:
742+
source: "field('date_no_doc_values').get(null)"
743+
- match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
744+
745+
- do:
746+
search:
747+
rest_total_hits_as_int: true
748+
body:
749+
query: { term: { _id: "1" } }
750+
script_fields:
751+
field:
752+
script:
753+
source: "/* avoid yaml stash */ $('date_no_doc_values', null)"
754+
- match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
755+
756+
- do:
757+
search:
758+
rest_total_hits_as_int: true
759+
body:
760+
query: { term: { _id: 1 } }
761+
script_fields:
762+
field:
763+
script:
764+
source: "field('date_no_doc_values').get(null).getMillis()"
765+
- match: { hits.hits.0.fields.field.0: 1483272672000 }
766+
767+
- do:
768+
search:
769+
rest_total_hits_as_int: true
770+
body:
771+
query: { term: { _id: 1 } }
772+
script_fields:
773+
field:
774+
script:
775+
source: "/* avoid yaml stash */ $('date_no_doc_values', null).getMillis()"
776+
- match: { hits.hits.0.fields.field.0: 1483272672000 }
777+
778+
- do:
779+
search:
780+
rest_total_hits_as_int: true
781+
body:
782+
query: { term: { _id: 1 } }
783+
script_fields:
784+
field:
785+
script:
786+
source: "field('date_no_doc_values').get(null).millis"
787+
- match: { hits.hits.0.fields.field.0: 1483272672000 }
788+
789+
- do:
790+
search:
791+
rest_total_hits_as_int: true
792+
body:
793+
query: { term: { _id: 1 } }
794+
script_fields:
795+
field:
796+
script:
797+
source: "/* avoid yaml stash */ $('date_no_doc_values', null).millis"
798+
- match: { hits.hits.0.fields.field.0: 1483272672000 }
799+
800+
- do:
801+
search:
802+
rest_total_hits_as_int: true
803+
body:
804+
query: { term: { _id: "2" } }
805+
script_fields:
806+
field:
807+
script:
808+
source: "field('date_no_doc_values').get(ZonedDateTime.parse('2018-01-01T12:11:12.000Z'))"
809+
- match: { hits.hits.0.fields.field.0: '2018-01-01T12:11:12.000Z' }
810+
811+
- do:
812+
search:
813+
rest_total_hits_as_int: true
814+
body:
815+
query: { term: { _id: "2" } }
816+
script_fields:
817+
field:
818+
script:
819+
source: "/* avoid yaml stash */ $('date_no_doc_values', ZonedDateTime.parse('2018-01-01T12:11:12.000Z'))"
820+
- match: { hits.hits.0.fields.field.0: '2018-01-01T12:11:12.000Z' }
821+
822+
- do:
823+
search:
824+
rest_total_hits_as_int: true
825+
body:
826+
query: { term: { _id: "1" } }
827+
script_fields:
828+
field:
829+
script:
830+
source: "field('nanos_no_doc_values').get(null)"
831+
- match: { hits.hits.0.fields.field.0: '2015-01-01T12:10:30.123456789Z' }
832+
833+
- do:
834+
search:
835+
rest_total_hits_as_int: true
836+
body:
837+
query: { term: { _id: "1" } }
838+
script_fields:
839+
field:
840+
script:
841+
source: "/* avoid yaml stash */ $('nanos_no_doc_values', null)"
842+
- match: { hits.hits.0.fields.field.0: '2015-01-01T12:10:30.123456789Z' }
843+
844+
- do:
845+
search:
846+
rest_total_hits_as_int: true
847+
body:
848+
query: { term: { _id: "2" } }
849+
script_fields:
850+
field:
851+
script:
852+
source: "field('nanos_no_doc_values').get(ZonedDateTime.parse('2016-01-01T12:10:30.123Z'))"
853+
- match: { hits.hits.0.fields.field.0: '2016-01-01T12:10:30.123Z' }
854+
855+
- do:
856+
search:
857+
rest_total_hits_as_int: true
858+
body:
859+
query: { term: { _id: "2" } }
860+
script_fields:
861+
field:
862+
script:
863+
source: "/* avoid yaml stash */ $('nanos_no_doc_values', ZonedDateTime.parse('2016-01-01T12:10:30.123Z'))"
864+
- match: { hits.hits.0.fields.field.0: '2016-01-01T12:10:30.123Z' }
865+
866+
- do:
867+
search:
868+
rest_total_hits_as_int: true
869+
body:
870+
query: { term: { _id: "1" } }
871+
script_fields:
872+
field:
873+
script:
874+
source: "field('nanos_no_doc_values').get(null).getNano()"
875+
- match: { hits.hits.0.fields.field.0: 123456789 }
876+
877+
- do:
878+
search:
879+
rest_total_hits_as_int: true
880+
body:
881+
query: { term: { _id: "1" } }
882+
script_fields:
883+
field:
884+
script:
885+
source: "/* avoid yaml stash */ $('nanos_no_doc_values', null).getNano()"
886+
- match: { hits.hits.0.fields.field.0: 123456789 }
887+
888+
- do:
889+
search:
890+
rest_total_hits_as_int: true
891+
body:
892+
query: { term: { _id: "2" } }
893+
script_fields:
894+
field:
895+
script:
896+
source: "field('nanos_no_doc_values').get(ZonedDateTime.parse('2016-01-01T12:10:30.123Z')).getNano()"
897+
- match: { hits.hits.0.fields.field.0: 123000000 }
898+
899+
- do:
900+
search:
901+
rest_total_hits_as_int: true
902+
body:
903+
query: { term: { _id: "3" } }
904+
script_fields:
905+
field:
906+
script:
907+
source: "field('date_no_doc_values').get(1, null)"
908+
- match: { hits.hits.0.fields.field.0: "2018-01-01T12:11:12.000Z" }
909+
910+
- do:
911+
search:
912+
rest_total_hits_as_int: true
913+
body:
914+
query: { term: { _id: "3" } }
915+
script_fields:
916+
field:
917+
script:
918+
source: "field('nanos_no_doc_values').get(1, null)"
919+
- match: { hits.hits.0.fields.field.0: "2015-01-01T12:10:30.987654321Z" }
920+
921+
- do:
922+
search:
923+
rest_total_hits_as_int: true
924+
body:
925+
query: { term: { _id: "3" } }
926+
script_fields:
927+
field:
928+
script:
929+
source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('date_no_doc_values')) times.add(zdt); times"
930+
- match: { hits.hits.0.fields.field: ["2017-01-01T12:11:12.000Z", "2018-01-01T12:11:12.000Z"] }
931+
932+
- do:
933+
search:
934+
rest_total_hits_as_int: true
935+
body:
936+
query: { term: { _id: "3" } }
937+
script_fields:
938+
field:
939+
script:
940+
source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('nanos_no_doc_values')) times.add(zdt); times"
941+
- match: { hits.hits.0.fields.field: ["2015-01-01T12:10:30.123456789Z", "2015-01-01T12:10:30.987654321Z"] }
942+
695943
---
696944
"geo_point":
697945
- do:

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.elasticsearch.index.fielddata.FieldDataContext;
3737
import org.elasticsearch.index.fielddata.IndexFieldData;
3838
import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
39+
import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData;
3940
import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
4041
import org.elasticsearch.index.query.DateRangeIncludingNowQuery;
4142
import org.elasticsearch.index.query.QueryRewriteContext;
@@ -64,6 +65,7 @@
6465
import java.util.Locale;
6566
import java.util.Map;
6667
import java.util.Objects;
68+
import java.util.Set;
6769
import java.util.function.BiFunction;
6870
import java.util.function.Function;
6971
import java.util.function.LongSupplier;
@@ -519,6 +521,17 @@ public String parseSourceValue(Object value) {
519521
};
520522
}
521523

524+
// returns a Long to support source fallback which emulates numeric doc values for dates
525+
private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
526+
return new SourceValueFetcher(sourcePaths, nullValue) {
527+
@Override
528+
public Long parseSourceValue(Object value) {
529+
String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString();
530+
return parse(date);
531+
}
532+
};
533+
}
534+
522535
private String format(long timestamp, DateFormatter formatter) {
523536
ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
524537
return formatter.format(dateTime);
@@ -750,8 +763,34 @@ public Function<byte[], Number> pointReaderIfPossible() {
750763

751764
@Override
752765
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
753-
failIfNoDocValues();
754-
return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType(), resolution.getDefaultToScriptFieldFactory());
766+
FielddataOperation operation = fieldDataContext.fielddataOperation();
767+
768+
if (operation == FielddataOperation.SEARCH) {
769+
failIfNoDocValues();
770+
}
771+
772+
if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) {
773+
return new SortedNumericIndexFieldData.Builder(
774+
name(),
775+
resolution.numericType(),
776+
resolution.getDefaultToScriptFieldFactory()
777+
);
778+
}
779+
780+
if (operation == FielddataOperation.SCRIPT) {
781+
SearchLookup searchLookup = fieldDataContext.lookupSupplier().get();
782+
Set<String> sourcePaths = fieldDataContext.sourcePathsLookup().apply(name());
783+
784+
return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
785+
name(),
786+
resolution.numericType().getValuesSourceType(),
787+
sourceValueFetcher(sourcePaths),
788+
searchLookup.source(),
789+
resolution.getDefaultToScriptFieldFactory()
790+
);
791+
}
792+
793+
throw new IllegalStateException("unknown field data operation [" + operation.name() + "]");
755794
}
756795

757796
@Override

0 commit comments

Comments
 (0)