diff --git a/docs/changelog/70065.yaml b/docs/changelog/70065.yaml new file mode 100644 index 0000000000000..db885e1498d81 --- /dev/null +++ b/docs/changelog/70065.yaml @@ -0,0 +1,5 @@ +pr: 70065 +summary: Add aliases as runtime fields +area: Search +type: feature +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java index d4298784521ee..26c131d7df3f4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java @@ -20,6 +20,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class IndexSortIT extends ESIntegTestCase { private static final XContentBuilder TEST_MAPPING = createTestMapping(); @@ -28,6 +29,9 @@ private static XContentBuilder createTestMapping() { try { return jsonBuilder() .startObject() + .startObject("runtime") + .startObject("alias").field("type", "alias").field("path", "numeric").endObject() + .endObject() .startObject("properties") .startObject("date") .field("type", "date") @@ -79,6 +83,18 @@ public void testIndexSort() { flushAndRefresh(); ensureYellow(); assertSortedSegments("test", indexSort); + + Exception e = expectThrows(Exception.class, () -> + prepareCreate("test_with_runtime_sort") + .setSettings(Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.sort.field", "alias") + ) + .setMapping(TEST_MAPPING) + .get()); + assertThat(e.getMessage(), equalTo("Cannot use alias [alias] as an index sort field")); } public void testInvalidIndexSort() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index 0f2038b6e1e0b..c1ea665f4417d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -58,7 +58,7 @@ abstract class AbstractScriptFieldType extends MappedFieldType impl } @Override - public final MappedFieldType asMappedFieldType() { + public final MappedFieldType asMappedFieldType(Function lookup) { return this; } @@ -257,8 +257,4 @@ private static Script parseScript(String name, Mapper.TypeParser.ParserContext p return script; } } - - static Function initializerNotSupported() { - return mapper -> { throw new UnsupportedOperationException(); }; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AliasRuntimeField.java b/server/src/main/java/org/elasticsearch/index/mapper/AliasRuntimeField.java new file mode 100644 index 0000000000000..d961c1c3ea70a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/AliasRuntimeField.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +public class AliasRuntimeField implements RuntimeField { + + public static final String CONTENT_TYPE = "alias"; + + private static class Builder extends RuntimeField.Builder { + + final FieldMapper.Parameter path = FieldMapper.Parameter.stringParam( + "path", + true, + initializerNotSupported(), + null + ).setValidator( + s -> { + if (s == null) { + throw new MapperParsingException("Missing required parameter [path]"); + } + } + ); + + protected Builder(String name) { + super(name); + } + + @Override + protected List> getParameters() { + return List.of(path); + } + + @Override + protected RuntimeField createRuntimeField(Mapper.TypeParser.ParserContext parserContext) { + return new AliasRuntimeField(name, path.get()); + } + } + + public static final RuntimeField.Parser PARSER = new RuntimeField.Parser(Builder::new); + + private final String name; + private final String path; + + public AliasRuntimeField(String name, String path) { + this.name = name; + this.path = path; + if (Objects.equals(name, path)) { + throw new MapperParsingException("Invalid path [" + path + "] for alias [" + path + "]: an alias cannot refer to itself"); + } + } + + @Override + public void doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field("path", path); + } + + @Override + public MappedFieldType asMappedFieldType(Function lookup) { + MappedFieldType ft = lookup.apply(path); + if (ft == null) { + throw new MapperParsingException("Cannot resolve alias [" + name + "]: path [" + path + "] does not exist in mappings"); + } + return ft; + } + + @Override + public String name() { + return name; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 87f30ad10ede3..8fc265c81c621 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -875,7 +875,7 @@ private static Mapper getLeafMapper(final ParseContext context, String fieldPath = context.path().pathAsText(fieldName); RuntimeField runtimeField = context.root().getRuntimeField(fieldPath); if (runtimeField != null) { - return new NoOpFieldMapper(subfields[subfields.length - 1], runtimeField.asMappedFieldType().name()); + return new NoOpFieldMapper(subfields[subfields.length - 1], runtimeField.name()); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 8e32180d188c9..828561f09192c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -68,21 +68,29 @@ final class FieldTypeLookup { } this.maxParentPathDots = maxParentPathDots; + Map resolvedRuntimeMappers = new HashMap<>(); + Map resolvedDynamicMappers = new HashMap<>(); for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) { String aliasName = fieldAliasMapper.name(); String path = fieldAliasMapper.path(); - MappedFieldType fieldType = fullNameToFieldType.get(path); - fullNameToFieldType.put(aliasName, fieldType); - if (fieldType instanceof DynamicFieldType) { - dynamicFieldTypes.put(aliasName, (DynamicFieldType) fieldType); + RuntimeField aliasField = new AliasRuntimeField(aliasName, path); + MappedFieldType resolved = aliasField.asMappedFieldType(fullNameToFieldType::get); + resolvedRuntimeMappers.put(aliasName, resolved); + if (resolved instanceof DynamicFieldType) { + resolvedDynamicMappers.put(aliasName, (DynamicFieldType) resolved); } } for (RuntimeField runtimeField : runtimeFields) { - MappedFieldType runtimeFieldType = runtimeField.asMappedFieldType(); - //this will override concrete fields with runtime fields that have the same name - fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType); + MappedFieldType resolved = runtimeField.asMappedFieldType(fullNameToFieldType::get); + resolvedRuntimeMappers.put(runtimeField.name(), resolved); + if (resolved instanceof DynamicFieldType) { + resolvedDynamicMappers.put(runtimeField.name(), (DynamicFieldType) resolved); + } } + + this.fullNameToFieldType.putAll(resolvedRuntimeMappers); + this.dynamicFieldTypes.putAll(resolvedDynamicMappers); } private static int dotCount(String path) { @@ -103,6 +111,9 @@ MappedFieldType get(String field) { if (fieldType != null) { return fieldType; } + + // If the mapping contains fields that support dynamic sub-key lookup, check + // if this could correspond to a keyed field of the form 'path_to_field.path_to_key'. return getDynamicField(field); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java index b589069dae304..5b28b8d17216d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java @@ -54,7 +54,7 @@ default XContentBuilder toXContent(XContentBuilder builder, Params params) throw * Exposes the {@link MappedFieldType} backing this runtime field, used to execute queries, run aggs etc. * @return the {@link MappedFieldType} backing this runtime field */ - MappedFieldType asMappedFieldType(); + MappedFieldType asMappedFieldType(Function lookup); /** * For runtime fields the {@link RuntimeField.Parser} returns directly the {@link MappedFieldType}. @@ -103,6 +103,10 @@ private void validate() { throw new IllegalArgumentException("runtime field [" + name + "] does not support [copy_to]"); } } + + protected static Function initializerNotSupported() { + return mapper -> { throw new UnsupportedOperationException(); }; + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index a6d97aec332b7..caa9b9e89a299 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -394,8 +394,10 @@ public boolean isFieldMapped(String name) { } private MappedFieldType fieldType(String name) { - MappedFieldType fieldType = runtimeMappings.get(name); - return fieldType == null ? mappingLookup.getFieldType(name) : fieldType; + if (runtimeMappings.containsKey(name)) { + return runtimeMappings.get(name); + } + return mappingLookup.getFieldType(name); } public ObjectMapper getObjectMapper(String name) { @@ -664,14 +666,15 @@ private static Map parseRuntimeMappings(Map runtimeFields = RuntimeField.parseRuntimeFields(new HashMap<>(runtimeMappings), - mapperService.parserContext(), false); - Map runtimeFieldTypes = new HashMap<>(); + Map runtimeFields + = RuntimeField.parseRuntimeFields(new HashMap<>(runtimeMappings), mapperService.parserContext(), false); + MappingLookup lookup = mapperService.mappingLookup(); + Map resolvedFields = new HashMap<>(); + for (RuntimeField runtimeField : runtimeFields.values()) { - MappedFieldType fieldType = runtimeField.asMappedFieldType(); - runtimeFieldTypes.put(fieldType.name(), fieldType); + resolvedFields.put(runtimeField.name(), runtimeField.asMappedFieldType(lookup::getFieldType)); } - return Collections.unmodifiableMap(runtimeFieldTypes); + return resolvedFields; } /** diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 2d9553b086de7..56c0471447812 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.index.mapper.AliasRuntimeField; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper; @@ -149,6 +150,7 @@ private static Map getRuntimeFields(List entry : mapperPlugin.getRuntimeFields().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 2b151c18332ae..f3502da79cf58 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -62,26 +62,20 @@ public void testAliasThatRefersToAlias() { FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "field"); FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "alias"); - MappingLookup mappers = createMappingLookup( + Exception e = expectThrows(MapperParsingException.class, () -> createMappingLookup( singletonList(field), emptyList(), Arrays.asList(alias, invalidAlias), emptyList() - ); - alias.validate(mappers); + )); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> { - invalidAlias.validate(mappers); - }); - - assertEquals("Invalid [path] value [alias] for field alias [invalid-alias]: an alias" + - " cannot refer to another alias.", e.getMessage()); + assertEquals("Cannot resolve alias [invalid-alias]: path [alias] does not exist in mappings", e.getMessage()); } public void testAliasThatRefersToItself() { FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "invalid-alias"); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> { + Exception e = expectThrows(MapperParsingException.class, () -> { MappingLookup mappers = createMappingLookup( emptyList(), emptyList(), @@ -91,8 +85,8 @@ public void testAliasThatRefersToItself() { invalidAlias.validate(mappers); }); - assertEquals("Invalid [path] value [invalid-alias] for field alias [invalid-alias]: an alias" + - " cannot refer to itself.", e.getMessage()); + assertEquals("Invalid path [invalid-alias] for alias [invalid-alias]: an alias" + + " cannot refer to itself", e.getMessage()); } public void testAliasWithNonExistentPath() { @@ -108,8 +102,7 @@ public void testAliasWithNonExistentPath() { invalidAlias.validate(mappers); }); - assertEquals("Invalid [path] value [non-existent] for field alias [invalid-alias]: an alias" + - " must refer to an existing field in the mappings.", e.getMessage()); + assertEquals("Cannot resolve alias [invalid-alias]: path [non-existent] does not exist in mappings", e.getMessage()); } public void testFieldAliasWithNestedScope() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RuntimeAliasTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RuntimeAliasTests.java new file mode 100644 index 0000000000000..89c7e36c9e453 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/RuntimeAliasTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; + +import java.io.IOException; + +public class RuntimeAliasTests extends MapperServiceTestCase { + + public void testSimpleAlias() throws IOException { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("runtime"); + { + b.startObject("alias-to-field").field("type", "alias").field("path", "field").endObject(); + } + b.endObject(); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + })); + + MappedFieldType aliased = mapperService.mappingLookup().getFieldType("alias-to-field"); + assertEquals("field", aliased.name()); + assertEquals(KeywordFieldMapper.KeywordFieldType.class, aliased.getClass()); + } + + public void testInvalidAlias() { + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(topMapping(b -> { + b.startObject("runtime"); + { + b.startObject("alias-to-field").field("type", "alias").field("path", "field").endObject(); + } + b.endObject(); + }))); + assertEquals("Cannot resolve alias [alias-to-field]: path [field] does not exist in mappings", e.getMessage()); + } + + public void testDynamicLookup() throws IOException { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startObject("runtime"); + { + b.startObject("dynamic-alias").field("type", "alias").field("path", "flattened").endObject(); + } + b.endObject(); + b.startObject("properties"); + { + b.startObject("flattened").field("type", "flattened").endObject(); + } + b.endObject(); + })); + + MappedFieldType dynamic = mapperService.fieldType("flattened.key"); + assertEquals("flattened._keyed", dynamic.name()); + MappedFieldType aliased = mapperService.fieldType("dynamic-alias.key"); + assertNotNull(aliased); + assertEquals("flattened._keyed", aliased.name()); + FlattenedFieldMapper.KeyedFlattenedFieldType keyed = (FlattenedFieldMapper.KeyedFlattenedFieldType) aliased; + assertEquals("key", keyed.key()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java index 3ae88f81b72b7..72a2b99ac1be2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.function.Function; public class TestRuntimeField extends MappedFieldType implements RuntimeField { @@ -30,7 +31,7 @@ public String typeName() { } @Override - public MappedFieldType asMappedFieldType() { + public MappedFieldType asMappedFieldType(Function lookup) { return this; } diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index df9789e07ce34..4f4f6d1f225c2 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.AliasRuntimeField; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -324,6 +325,17 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { ); } + public void testAliases() throws IOException { + RuntimeField runtime = new AliasRuntimeField("alias", "field"); + MappedFieldType field = new MockFieldMapper.FakeFieldType("field"); + SearchExecutionContext sec = createSearchExecutionContext( + "uuid", + null, + createMappingLookup(List.of(field), List.of(runtime)), + Map.of()); + assertEquals("field", sec.getFieldType("alias").name()); + } + private static MappingLookup createMappingLookup(List concreteFields, List runtimeFields) { List mappers = concreteFields.stream().map(MockFieldMapper::new).collect(Collectors.toList()); RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc", Version.CURRENT); @@ -368,6 +380,20 @@ public void testSearchRequestRuntimeFields() { assertThat(matches, hasSize(3)); } + public void testSearchRequestAliasLoops() { + Map runtimeMappings = Map.ofEntries( + Map.entry("alias-loop-1", Map.of("type", "alias", "path", "alias-loop-2")), + Map.entry("alias-loop-2", Map.of("type", "alias", "path", "alias-loop-1")) + ); + Exception e = expectThrows(MapperParsingException.class, () -> createSearchExecutionContext( + "uuid", + null, + createMappingLookup(Collections.emptyList(), Collections.emptyList()), + runtimeMappings + )); + assertEquals("Cannot resolve alias [alias-loop-2]: path [alias-loop-1] does not exist in mappings", e.getMessage()); + } + public void testSearchRequestRuntimeFieldsWrongFormat() { Map runtimeMappings = new HashMap<>(); runtimeMappings.put("field", Arrays.asList("test1", "test2")); @@ -417,7 +443,7 @@ private static SearchExecutionContext createSearchExecutionContext( ); IndexMetadata indexMetadata = indexMetadataBuilder.build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - MapperService mapperService = createMapperService(indexSettings); + MapperService mapperService = createMapperService(indexSettings, mappingLookup); final long nowInMillis = randomNonNegativeLong(); return new SearchExecutionContext( 0, @@ -443,7 +469,8 @@ private static SearchExecutionContext createSearchExecutionContext( } private static MapperService createMapperService( - IndexSettings indexSettings + IndexSettings indexSettings, + MappingLookup mappingLookup ) { IndexAnalyzers indexAnalyzers = new IndexAnalyzers( singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, null)), @@ -467,6 +494,7 @@ private static MapperService createMapperService( indexSettings, () -> true )); + when(mapperService.mappingLookup()).thenReturn(mappingLookup); return mapperService; }