From d556c6aaa3fa45e1b57fcebc04b2510689c5e0df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 11 Oct 2021 15:07:48 +0200 Subject: [PATCH 1/6] adding _ignore fetching to fields api --- .../test/search/330_fetch_fields.yml | 13 ++++ .../index/mapper/IgnoredFieldMapper.java | 2 +- .../index/mapper/IgnoredFieldMapperTests.java | 68 +++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index cb7335ad78e7d..d7ec835dd1eff 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1024,6 +1024,9 @@ test fetching metadata fields: properties: field: type: keyword + short_field: + type: keyword + ignore_above: 5 idAlias: type: alias path: _id @@ -1035,6 +1038,7 @@ test fetching metadata fields: refresh: true body: field: foo + short_field: foo123 - do: search: @@ -1062,3 +1066,12 @@ test fetching metadata fields: - length: { hits.hits.0.fields : 1 } - match: { hits.hits.0.fields._id.0: "1" } + + - do: + search: + index: test + body: + fields: [ "_ignored" ] + + - length: { hits.hits.0.fields : 1 } + - match: { hits.hits.0.fields._ignored.0: "short_field" } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index f4c44d4227f67..d65c5897e54fd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -68,7 +68,7 @@ public Query existsQuery(SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + return new StoredValueFetcher(context.lookup(), NAME); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java new file mode 100644 index 0000000000000..83342ebf538b8 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IgnoredFieldMapperTests extends MapperServiceTestCase { + + public void testIncludeInObjectNotAllowed() throws Exception { + DocumentMapper docMapper = createDocumentMapper(mapping(b -> {})); + + Exception e = expectThrows(MapperParsingException.class, + () -> docMapper.parse(source(b -> b.field("_ignored", 1)))); + + assertThat(e.getCause().getMessage(), + containsString("Field [_ignored] is a metadata field and cannot be added inside a document")); + } + + public void testDefaults() throws IOException { + DocumentMapper mapper = createDocumentMapper( + mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 3).endObject()) + ); + ParsedDocument document = mapper.parse(source(b -> b.field("field", "value"))); + IndexableField[] fields = document.rootDoc().getFields(IgnoredFieldMapper.NAME); + assertEquals(1, fields.length); + assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions()); + assertTrue(fields[0].fieldType().stored()); + } + + public void testFetchIgnoredFieldValue() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "keyword").field("ignore_above", 3)) + ); + withLuceneIndex(mapperService, iw -> { + iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc()); + }, iw -> { + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.lookup()).thenReturn(lookup); + IgnoredFieldMapper.IgnoredFieldType ft = (IgnoredFieldMapper.IgnoredFieldType) mapperService.fieldType("_ignored"); + ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); + IndexSearcher searcher = newSearcher(iw); + LeafReaderContext context = searcher.getIndexReader().leaves().get(0); + lookup.source().setSegmentAndDocument(context, 0); + valueFetcher.setNextReader(context); + assertEquals(List.of("field"), valueFetcher.fetchValues(lookup.source())); + }); + } + +} From 158c04bca5344e743741440a74875071c9f33a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 11 Oct 2021 16:05:44 +0200 Subject: [PATCH 2/6] adding _routing fetching to fields api --- .../test/search/330_fetch_fields.yml | 21 ++++++++++++++++ .../index/mapper/RoutingFieldMapper.java | 2 +- .../index/mapper/RoutingFieldMapperTests.java | 25 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index d7ec835dd1eff..ca288ec54b80f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1035,11 +1035,20 @@ test fetching metadata fields: index: index: test id: 1 + routing: abcd refresh: true body: field: foo short_field: foo123 + - do: + index: + index: test + id: 2 + refresh: true + body: + field: bar + - do: search: index: test @@ -1066,6 +1075,8 @@ test fetching metadata fields: - length: { hits.hits.0.fields : 1 } - match: { hits.hits.0.fields._id.0: "1" } + - length: { hits.hits.1.fields : 1 } + - match: { hits.hits.1.fields._id.0: "2" } - do: search: @@ -1075,3 +1086,13 @@ test fetching metadata fields: - length: { hits.hits.0.fields : 1 } - match: { hits.hits.0.fields._ignored.0: "short_field" } + + - do: + search: + index: test + body: + fields: [ "_routing" ] + + - length: { hits.hits.0.fields : 1 } + - match: { hits.hits.0.fields._routing.0: "abcd" } + - is_false: hits.hits.1.fields diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index e7a676098d9e4..c8548a4cd4c1a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -84,7 +84,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + return new StoredValueFetcher(context.lookup(), NAME); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index e855843e7cc03..dfac015dc0587 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -8,15 +8,22 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class RoutingFieldMapperTests extends MetadataMapperTestCase { @@ -52,4 +59,22 @@ public void testIncludeInObjectNotAllowed() throws Exception { assertThat(e.getCause().getMessage(), containsString("Field [_routing] is a metadata field and cannot be added inside a document")); } + + public void testFetchRoutingFieldValue() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> {})); + withLuceneIndex(mapperService, iw -> { + iw.addDocument(mapperService.documentMapper().parse(source("1", b -> {}, "abcd")).rootDoc()); + }, iw -> { + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.lookup()).thenReturn(lookup); + RoutingFieldMapper.RoutingFieldType ft = (RoutingFieldMapper.RoutingFieldType) mapperService.fieldType("_routing"); + ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); + IndexSearcher searcher = newSearcher(iw); + LeafReaderContext context = searcher.getIndexReader().leaves().get(0); + lookup.source().setSegmentAndDocument(context, 0); + valueFetcher.setNextReader(context); + assertEquals(List.of("abcd"), valueFetcher.fetchValues(lookup.source())); + }); + } } From 8964620c146618e7ca814d67e83e2399a22b4394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 11 Oct 2021 17:37:21 +0200 Subject: [PATCH 3/6] wip _source --- .../test/search/330_fetch_fields.yml | 7 +++++ .../index/mapper/SourceFieldMapper.java | 22 +++++++++++--- .../index/mapper/StoredValueFetcher.java | 2 +- .../index/mapper/SourceFieldMapperTests.java | 30 ++++++++++++++++++- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index ca288ec54b80f..370f00b273b28 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1096,3 +1096,10 @@ test fetching metadata fields: - length: { hits.hits.0.fields : 1 } - match: { hits.hits.0.fields._routing.0: "abcd" } - is_false: hits.hits.1.fields + + - do: + search: + index: test + body: + fields: [ "_source" ] + diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 20859cb557aa7..3d8bcc6999731 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,15 +18,16 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Arrays; @@ -34,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; public class SourceFieldMapper extends MetadataFieldMapper { @@ -106,7 +108,19 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + if (isStored()) { + return new StoredValueFetcher(context.lookup(), NAME) { + + @Override + public List fetchValues(SourceLookup lookup) throws IOException { + List values = super.fetchValues(lookup); + // _source is stored as BytesRef, to make it useful convert it to String + return values.stream().map(o -> ((BytesRef) o).utf8ToString()).collect(Collectors.toList()); + } + }; + } else { + return lookup -> List.of(); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java index 77ceae54e6297..0329ce864a0ea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java @@ -19,7 +19,7 @@ /** * Value fetcher that loads from stored values. */ -public final class StoredValueFetcher implements ValueFetcher { +public class StoredValueFetcher implements ValueFetcher { private final SearchLookup lookup; private LeafSearchLookup leafSearchLookup; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index fcc80ced2fe15..e5809f4d67f17 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -9,19 +9,26 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SourceFieldMapperTests extends MetadataMapperTestCase { @@ -121,4 +128,25 @@ public void testSourceObjectContainsExtraTokens() throws Exception { assertThat(exception.getRootCause().getMessage(), containsString("Unexpected close marker '}'")); } + public void testFetchSourceFieldValue() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "keyword")) + ); + withLuceneIndex(mapperService, iw -> { + iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc()); + }, iw -> { + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.lookup()).thenReturn(lookup); + SourceFieldMapper.SourceFieldType ft = (SourceFieldMapper.SourceFieldType) mapperService.fieldType("_source"); + ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); + IndexSearcher searcher = newSearcher(iw); + LeafReaderContext context = searcher.getIndexReader().leaves().get(0); + lookup.source().setSegmentAndDocument(context, 0); + valueFetcher.setNextReader(context); + assertEquals(List.of("{\"field\":\"value\"}"), valueFetcher.fetchValues(lookup.source())); + }); + } + + } From 283375f2ca38389d1f15f7ba18ef1e7531b80a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 12 Oct 2021 13:18:51 +0200 Subject: [PATCH 4/6] Revert "wip _source" This reverts commit 8964620c146618e7ca814d67e83e2399a22b4394. --- .../test/search/330_fetch_fields.yml | 7 ----- .../index/mapper/SourceFieldMapper.java | 22 +++----------- .../index/mapper/StoredValueFetcher.java | 2 +- .../index/mapper/SourceFieldMapperTests.java | 30 +------------------ 4 files changed, 6 insertions(+), 55 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index 370f00b273b28..ca288ec54b80f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1096,10 +1096,3 @@ test fetching metadata fields: - length: { hits.hits.0.fields : 1 } - match: { hits.hits.0.fields._routing.0: "abcd" } - is_false: hits.hits.1.fields - - - do: - search: - index: test - body: - fields: [ "_source" ] - diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3d8bcc6999731..20859cb557aa7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,16 +18,15 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SourceLookup; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Arrays; @@ -35,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.stream.Collectors; public class SourceFieldMapper extends MetadataFieldMapper { @@ -108,19 +106,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - if (isStored()) { - return new StoredValueFetcher(context.lookup(), NAME) { - - @Override - public List fetchValues(SourceLookup lookup) throws IOException { - List values = super.fetchValues(lookup); - // _source is stored as BytesRef, to make it useful convert it to String - return values.stream().map(o -> ((BytesRef) o).utf8ToString()).collect(Collectors.toList()); - } - }; - } else { - return lookup -> List.of(); - } + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java index 0329ce864a0ea..77ceae54e6297 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java @@ -19,7 +19,7 @@ /** * Value fetcher that loads from stored values. */ -public class StoredValueFetcher implements ValueFetcher { +public final class StoredValueFetcher implements ValueFetcher { private final SearchLookup lookup; private LeafSearchLookup leafSearchLookup; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index e5809f4d67f17..fcc80ced2fe15 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -9,26 +9,19 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; -import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class SourceFieldMapperTests extends MetadataMapperTestCase { @@ -128,25 +121,4 @@ public void testSourceObjectContainsExtraTokens() throws Exception { assertThat(exception.getRootCause().getMessage(), containsString("Unexpected close marker '}'")); } - public void testFetchSourceFieldValue() throws IOException { - MapperService mapperService = createMapperService( - fieldMapping(b -> b.field("type", "keyword")) - ); - withLuceneIndex(mapperService, iw -> { - iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc()); - }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); - SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); - when(searchExecutionContext.lookup()).thenReturn(lookup); - SourceFieldMapper.SourceFieldType ft = (SourceFieldMapper.SourceFieldType) mapperService.fieldType("_source"); - ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); - IndexSearcher searcher = newSearcher(iw); - LeafReaderContext context = searcher.getIndexReader().leaves().get(0); - lookup.source().setSegmentAndDocument(context, 0); - valueFetcher.setNextReader(context); - assertEquals(List.of("{\"field\":\"value\"}"), valueFetcher.fetchValues(lookup.source())); - }); - } - - } From 60e085abdfae4528fee88b5d059f5ba1113371af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 12 Oct 2021 13:21:50 +0200 Subject: [PATCH 5/6] Fix FieldFetcherTests --- .../search/fetch/subphase/FieldFetcherTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 87cdfd8d1de92..fe2a0023bc5d9 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -15,14 +15,14 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.util.ArrayList; @@ -195,7 +195,7 @@ public void testMetadataFields() throws IOException { assertTrue(fields.isEmpty()); // several other metadata fields throw exceptions via their value fetchers when trying to get them - for (String fieldname : List.of("_index", "_seq_no", "_routing", "_ignored")) { + for (String fieldname : List.of("_index", "_seq_no")) { expectThrows(UnsupportedOperationException.class, () -> fetchFields(mapperService, source, fieldname)); } } From fa74052572c5c4f66d60e4bf204a9d14c0d1bf57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 12 Oct 2021 19:08:20 +0200 Subject: [PATCH 6/6] Add to FieldFetcherTests --- .../test/search/330_fetch_fields.yml | 34 -------------- .../fetch/subphase/FieldFetcherTests.java | 46 ++++++++++++++++++- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index ca288ec54b80f..cb7335ad78e7d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1024,9 +1024,6 @@ test fetching metadata fields: properties: field: type: keyword - short_field: - type: keyword - ignore_above: 5 idAlias: type: alias path: _id @@ -1035,19 +1032,9 @@ test fetching metadata fields: index: index: test id: 1 - routing: abcd refresh: true body: field: foo - short_field: foo123 - - - do: - index: - index: test - id: 2 - refresh: true - body: - field: bar - do: search: @@ -1075,24 +1062,3 @@ test fetching metadata fields: - length: { hits.hits.0.fields : 1 } - match: { hits.hits.0.fields._id.0: "1" } - - length: { hits.hits.1.fields : 1 } - - match: { hits.hits.1.fields._id.0: "2" } - - - do: - search: - index: test - body: - fields: [ "_ignored" ] - - - length: { hits.hits.0.fields : 1 } - - match: { hits.hits.0.fields._ignored.0: "short_field" } - - - do: - search: - index: test - body: - fields: [ "_routing" ] - - - length: { hits.hits.0.fields : 1 } - - match: { hits.hits.0.fields._routing.0: "abcd" } - - is_false: hits.hits.1.fields diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index fe2a0023bc5d9..6ea57c6860fda 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -8,18 +8,24 @@ package org.elasticsearch.search.fetch.subphase; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -29,6 +35,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static java.util.Collections.emptyMap; import static org.elasticsearch.xcontent.ObjectPath.eval; @@ -198,6 +205,34 @@ public void testMetadataFields() throws IOException { for (String fieldname : List.of("_index", "_seq_no")) { expectThrows(UnsupportedOperationException.class, () -> fetchFields(mapperService, source, fieldname)); } + + String docId = randomAlphaOfLength(12); + String routing = randomAlphaOfLength(12); + withLuceneIndex(mapperService, iw -> { + iw.addDocument(mapperService.documentMapper().parse(source(docId, b -> b.field("integer_field", "value"), routing)).rootDoc()); + }, iw -> { + List fieldList = List.of( + new FieldAndFormat("_id", null), + new FieldAndFormat("_routing", null), + new FieldAndFormat("_ignored", null) + ); + FieldFetcher fieldFetcher = FieldFetcher.create( + newSearchExecutionContext(mapperService, (ft, index, sl) -> fieldDataLookup().apply(ft, sl)), + fieldList + ); + IndexSearcher searcher = newSearcher(iw); + LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); + fieldFetcher.setNextReader(readerContext); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSegmentAndDocument(readerContext, 0); + + Map fetchedFields = fieldFetcher.fetch(sourceLookup); + assertThat(fetchedFields.size(), equalTo(3)); + assertEquals(docId, fetchedFields.get("_id").getValue()); + assertEquals(routing, fetchedFields.get("_routing").getValue()); + assertEquals("integer_field", fetchedFields.get("_ignored").getValue()); + }); } public void testFetchAllFields() throws IOException { @@ -946,7 +981,7 @@ public MapperService createMapperService() throws IOException { .startObject("_doc") .startObject("properties") .startObject("field").field("type", "keyword").endObject() - .startObject("integer_field").field("type", "integer").endObject() + .startObject("integer_field").field("type", "integer").field("ignore_malformed", "true").endObject() .startObject("date_field").field("type", "date").endObject() .startObject("geo_point").field("type", "geo_point").endObject() .startObject("float_range").field("type", "float_range").endObject() @@ -965,6 +1000,13 @@ public MapperService createMapperService() throws IOException { } private static SearchExecutionContext newSearchExecutionContext(MapperService mapperService) { + return newSearchExecutionContext(mapperService, null); + } + + private static SearchExecutionContext newSearchExecutionContext( + MapperService mapperService, + TriFunction, IndexFieldData> indexFieldDataLookup + ) { Settings settings = Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) @@ -976,7 +1018,7 @@ private static SearchExecutionContext newSearchExecutionContext(MapperService ma 0, indexSettings, null, - null, + indexFieldDataLookup, mapperService, mapperService.mappingLookup(), null,