From c3b5d0f8c9f8b6c9ac7955765c2e1641e7b8ede9 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 23 Oct 2025 13:31:55 -0600 Subject: [PATCH 01/24] Use the a mod number of shard function to route Using hash % number of shards to routing documents. --- .../elasticsearch/cluster/routing/IndexRouting.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 12d45898bfba5..fe44bebedee49 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -70,6 +70,7 @@ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { } protected final String indexName; + private final int numberOfShards; private final int routingNumShards; private final int routingFactor; @Nullable @@ -77,6 +78,7 @@ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { private IndexRouting(IndexMetadata metadata) { this.indexName = metadata.getIndex().getName(); + this.numberOfShards = metadata.getNumberOfShards(); this.routingNumShards = metadata.getRoutingNumShards(); this.routingFactor = metadata.getRoutingFactor(); this.indexReshardingMetadata = metadata.getReshardingMetadata(); @@ -142,6 +144,15 @@ public void postProcess(IndexRequest indexRequest) {} * shard id. */ protected final int hashToShardId(int hash) { + return Math.floorMod(hash, numberOfShards); + } + + + /** + * Convert a hash generated from an {@code (id, routing}) pair into a + * shard id using the old routingNumShards mechanism. + */ + protected final int hashToShardIdOld(int hash) { return Math.floorMod(hash, routingNumShards) / routingFactor; } From cb339d6c40763e3772d9f21e9d2dd789bc0efb2c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 23 Oct 2025 19:40:55 +0000 Subject: [PATCH 02/24] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/cluster/routing/IndexRouting.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index fe44bebedee49..6457d5c27ccff 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -147,7 +147,6 @@ protected final int hashToShardId(int hash) { return Math.floorMod(hash, numberOfShards); } - /** * Convert a hash generated from an {@code (id, routing}) pair into a * shard id using the old routingNumShards mechanism. From feb0029730600f43f3f8fbf393dcc3efb14880fa Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 24 Oct 2025 16:36:59 -0600 Subject: [PATCH 03/24] Split shrink without bwc --- .../elasticsearch/cluster/metadata/IndexMetadata.java | 9 +++++++-- .../elasticsearch/cluster/routing/IndexRoutingTests.java | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 38d0fada7d866..edcead5531656 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -3077,7 +3077,8 @@ public static ShardId selectSplitShard(int shardId, IndexMetadata sourceIndexMet } final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); - return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); + return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards())); + // return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); } /** @@ -3167,9 +3168,13 @@ public static Set selectShrinkShards(int shardId, IndexMetadata sourceI } int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = Sets.newHashSetWithExpectedSize(routingFactor); - for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) { + for (int i = shardId; i < sourceIndexMetadata.getNumberOfShards(); i += numTargetShards) { + assert Math.floorMod(i, numTargetShards) == shardId; shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); } + for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) { + // shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + } return shards; } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 122a6190c06fa..c8bb4f2c58c27 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -206,6 +206,7 @@ public void testPartitionedIndex() { } } + @AwaitsFix(bugUrl = "I believe it is valid that these change but need to check.") public void testPartitionedIndexShrunk() { Map> routingIdToShard = new HashMap<>(); From 88dd3ffa144ab7aa0828f38f9d85f5d7dca806f9 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 27 Oct 2025 12:41:09 -0600 Subject: [PATCH 04/24] Change --- .../shrink/TransportResizeActionTests.java | 2 +- .../cluster/metadata/IndexMetadataTests.java | 24 +++++++++---------- .../DiskThresholdDeciderUnitTests.java | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index 66fb8a58bba1c..c5042c5aeee45 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -118,7 +118,7 @@ public void testErrorCondition() { "target", new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator( new StoreStats(between(1, 100), between(0, 100), between(1, 100)), - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + (i) -> i == 1 || i == 5 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null ) ); }).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index d0aae463dd193..ab3bfeedc5c49 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -311,28 +311,28 @@ public void testSelectShrinkShards() { shardIds, Sets.newHashSet( new ShardId(metadata.getIndex(), 0), - new ShardId(metadata.getIndex(), 1), - new ShardId(metadata.getIndex(), 2), - new ShardId(metadata.getIndex(), 3) + new ShardId(metadata.getIndex(), 8), + new ShardId(metadata.getIndex(), 16), + new ShardId(metadata.getIndex(), 24) ) ); shardIds = IndexMetadata.selectShrinkShards(1, metadata, 8); assertEquals( shardIds, Sets.newHashSet( - new ShardId(metadata.getIndex(), 4), - new ShardId(metadata.getIndex(), 5), - new ShardId(metadata.getIndex(), 6), - new ShardId(metadata.getIndex(), 7) + new ShardId(metadata.getIndex(), 1), + new ShardId(metadata.getIndex(), 9), + new ShardId(metadata.getIndex(), 17), + new ShardId(metadata.getIndex(), 25) ) ); shardIds = IndexMetadata.selectShrinkShards(7, metadata, 8); assertEquals( shardIds, Sets.newHashSet( - new ShardId(metadata.getIndex(), 28), - new ShardId(metadata.getIndex(), 29), - new ShardId(metadata.getIndex(), 30), + new ShardId(metadata.getIndex(), 7), + new ShardId(metadata.getIndex(), 15), + new ShardId(metadata.getIndex(), 23), new ShardId(metadata.getIndex(), 31) ) ); @@ -381,9 +381,9 @@ public void testSelectSplitShard() { ShardId shardId = IndexMetadata.selectSplitShard(0, metadata, 4); assertEquals(0, shardId.getId()); shardId = IndexMetadata.selectSplitShard(1, metadata, 4); - assertEquals(0, shardId.getId()); - shardId = IndexMetadata.selectSplitShard(2, metadata, 4); assertEquals(1, shardId.getId()); + shardId = IndexMetadata.selectSplitShard(2, metadata, 4); + assertEquals(0, shardId.getId()); shardId = IndexMetadata.selectSplitShard(3, metadata, 4); assertEquals(1, shardId.getId()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 37510f0774312..4e8bcaeb857fb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -725,7 +725,7 @@ public void testSizeShrinkIndex() { new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"), ShardRouting.Role.DEFAULT ); - assertEquals(110L, getExpectedShardSize(target2, 0L, allocation)); + assertEquals(510L, getExpectedShardSize(target2, 0L, allocation)); target2 = ShardRouting.newUnassigned( new ShardId(new Index("target2", "9101112"), 1), @@ -734,7 +734,7 @@ public void testSizeShrinkIndex() { new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"), ShardRouting.Role.DEFAULT ); - assertEquals(1000L, getExpectedShardSize(target2, 0L, allocation)); + assertEquals(600L, getExpectedShardSize(target2, 0L, allocation)); // check that the DiskThresholdDecider still works even if the source index has been deleted ClusterState clusterStateWithMissingSourceIndex = ClusterState.builder(clusterState) From 4f1b8ca1ee0399a777cc0a4eda6cffa03229d748 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 27 Oct 2025 14:52:07 -0600 Subject: [PATCH 05/24] Fixes --- .../test/reindex/35_search_failures.yml | 2 +- .../update_by_query/35_search_failure.yml | 2 +- .../action/bulk/BulkIntegrationIT.java | 2 +- .../routing/PartitionedRoutingIT.java | 22 +++++++------------ .../elasticsearch/index/IndexVersions.java | 1 + .../test/rrf/300_rrf_retriever.yml | 7 +++--- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml index fb9fb0302de6c..64e5533d2135c 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml @@ -33,7 +33,7 @@ - match: {error.phase: query} - match: {error.root_cause.0.type: script_exception} - match: {error.root_cause.0.reason: runtime error} - - match: {error.failed_shards.0.shard: 0} + - match: {error.failed_shards.0.shard: 1} - match: {error.failed_shards.0.index: source} - is_true: error.failed_shards.0.node - match: {error.failed_shards.0.reason.type: script_exception} diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml index 1a28ec8b183b5..70f6c538dae46 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml @@ -30,7 +30,7 @@ - match: {error.phase: query} - match: {error.root_cause.0.type: script_exception} - match: {error.root_cause.0.reason: runtime error} - - match: {error.failed_shards.0.shard: 0} + - match: {error.failed_shards.0.shard: 1} - match: {error.failed_shards.0.index: source} - is_true: error.failed_shards.0.node - match: {error.failed_shards.0.reason.type: script_exception} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 2cd319d148321..35235c310d166 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -83,7 +83,7 @@ public void testBulkWithWriteIndexAndRouting() { indexRequestWithAlias.source(Collections.singletonMap("foo", "baz")); BulkResponse bulkResponse = client().prepareBulk().add(indexRequestWithAlias).get(); assertThat(bulkResponse.getItems()[0].getResponse().getIndex(), equalTo("index3")); - assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(0)); + assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(1)); assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); assertThat(bulkResponse.getItems()[0].getResponse().status(), equalTo(RestStatus.CREATED)); assertThat(client().prepareGet("index3", "id").setRouting("1").get().getSource().get("foo"), equalTo("baz")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 68bc6656cec7f..341e87ea1e515 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.mockito.internal.util.collections.Sets; import java.util.HashMap; import java.util.HashSet; @@ -25,7 +24,9 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +// TODO: Double check logic public class PartitionedRoutingIT extends ESIntegTestCase { public void testVariousPartitionSizes() throws Exception { @@ -48,7 +49,7 @@ public void testVariousPartitionSizes() throws Exception { verifyGets(index, routingToDocumentIds); verifyBroadSearches(index, routingToDocumentIds, shards); - verifyRoutedSearches(index, routingToDocumentIds, Sets.newSet(partitionSize)); + verifyRoutedSearches(index, routingToDocumentIds, partitionSize); } } } @@ -75,20 +76,12 @@ public void testShrinking() throws Exception { Map> routingToDocumentIds = generateRoutedDocumentIds(index); while (true) { - int factor = originalShards / currentShards; - verifyGets(index, routingToDocumentIds); verifyBroadSearches(index, routingToDocumentIds, currentShards); // we need the floor and ceiling of the routing_partition_size / factor since the partition size of the shrunken // index will be one of those, depending on the routing value - verifyRoutedSearches( - index, - routingToDocumentIds, - Math.floorDiv(partitionSize, factor) == 0 - ? Sets.newSet(1, 2) - : Sets.newSet(Math.floorDiv(partitionSize, factor), -Math.floorDiv(-partitionSize, factor)) - ); + verifyRoutedSearches(index, routingToDocumentIds, Math.min(partitionSize, currentShards)); updateIndexSettings( Settings.builder() @@ -144,7 +137,7 @@ public void testUnableToUpdateIndexRoutingPartitionSizes() throws Exception { assertThat(exc.getMessage(), containsString("final indexMetadata setting [index.routing_partition_size]")); } - private void verifyRoutedSearches(String index, Map> routingToDocumentIds, Set expectedShards) { + private void verifyRoutedSearches(String index, Map> routingToDocumentIds, int expectedShards) { for (Map.Entry> routingEntry : routingToDocumentIds.entrySet()) { String routing = routingEntry.getKey(); int expectedDocuments = routingEntry.getValue().size(); @@ -164,9 +157,10 @@ private void verifyRoutedSearches(String index, Map> routing + "]" ); - assertTrue( + assertThat( response.getTotalShards() + " was not in " + expectedShards + " for " + index, - expectedShards.contains(response.getTotalShards()) + expectedShards, + equalTo(response.getTotalShards()) ); assertEquals(expectedDocuments, response.getHits().getTotalHits().value()); diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index e63b655e2ce8d..dd704ef2fb9a0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -192,6 +192,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1); public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1); + public static final IndexVersion MOD_ROUTING_FUNCTION = def(9_044_0_00, Version.LUCENE_10_3_1); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml index ac328967d9fcf..5cc0a9b5608ef 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml @@ -254,9 +254,10 @@ setup: - match: { hits.total.value : 3 } - length: { hits.hits: 1 } - - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0.fields.text.0: "term" } - - match: { hits.hits.0.fields.keyword.0: "keyword" } +# TODO: Maybe changed ordering of hits? + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.fields.text.0: "term term" } + - match: { hits.hits.0.fields.keyword.0: "other" } --- "rrf retriever with multiple standard retrievers and multiple knn retriever and a filter": From c21f9308d4cf3e0c3a3882e240bf024a436754f4 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 28 Oct 2025 14:23:09 -0600 Subject: [PATCH 06/24] BWC --- .../cluster/routing/IndexRouting.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 6457d5c27ccff..f7eeccc037706 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -73,6 +73,7 @@ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { private final int numberOfShards; private final int routingNumShards; private final int routingFactor; + protected final IndexVersion creationVersion; @Nullable private final IndexReshardingMetadata indexReshardingMetadata; @@ -81,6 +82,7 @@ private IndexRouting(IndexMetadata metadata) { this.numberOfShards = metadata.getNumberOfShards(); this.routingNumShards = metadata.getRoutingNumShards(); this.routingFactor = metadata.getRoutingFactor(); + this.creationVersion = metadata.getCreationVersion(); this.indexReshardingMetadata = metadata.getReshardingMetadata(); } @@ -139,12 +141,20 @@ public void postProcess(IndexRequest indexRequest) {} */ public abstract void collectSearchShards(String routing, IntConsumer consumer); + private static boolean shouldUseShardCountModRouting(final IndexVersion creationVersion) { + return creationVersion.onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION); + } + /** * Convert a hash generated from an {@code (id, routing}) pair into a * shard id. */ protected final int hashToShardId(int hash) { - return Math.floorMod(hash, numberOfShards); + if (shouldUseShardCountModRouting(creationVersion)) { + return Math.floorMod(hash, numberOfShards); + } else { + return hashToShardIdOld(hash); + } } /** @@ -194,12 +204,10 @@ private int rerouteFromSplitTargetShard(int shardId, IndexReshardingState.Split. private abstract static class IdAndRoutingOnly extends IndexRouting { private final boolean routingRequired; - private final IndexVersion creationVersion; private final IndexMode indexMode; IdAndRoutingOnly(IndexMetadata metadata) { super(metadata); - this.creationVersion = metadata.getCreationVersion(); MappingMetadata mapping = metadata.mapping(); this.routingRequired = mapping == null ? false : mapping.routingRequired(); this.indexMode = metadata.getIndexMode(); From 10d2a22354b2b7e888a992a0fd177c8c50e9b113 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 28 Oct 2025 15:17:17 -0600 Subject: [PATCH 07/24] More --- .../action/admin/indices/shrink/TransportResizeAction.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 2f8765fc20b8c..04df71ecf6c89 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -239,6 +239,8 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest( targetIndex.cause(cause); Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); settingsBuilder.put("index.number_of_shards", targetNumberOfShards); + // TODO: Maybe not this approach + settingsBuilder.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), sourceMetadata.getCreationVersion().toString()); targetIndex.settings(settingsBuilder); return new CreateIndexClusterStateUpdateRequest(cause, projectId, targetIndex.index(), targetIndexName) From 57e2d6a3b72f1cb024a70544a41edc648cfe0790 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 28 Oct 2025 15:47:53 -0600 Subject: [PATCH 08/24] Change --- .../indices/shrink/TransportResizeAction.java | 2 -- .../cluster/metadata/IndexMetadata.java | 27 ++++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 04df71ecf6c89..2f8765fc20b8c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -239,8 +239,6 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest( targetIndex.cause(cause); Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); settingsBuilder.put("index.number_of_shards", targetNumberOfShards); - // TODO: Maybe not this approach - settingsBuilder.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), sourceMetadata.getCreationVersion().toString()); targetIndex.settings(settingsBuilder); return new CreateIndexClusterStateUpdateRequest(cause, projectId, targetIndex.index(), targetIndexName) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index edcead5531656..edb3fd8fbe3e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -3075,10 +3075,14 @@ public static ShardId selectSplitShard(int shardId, IndexMetadata sourceIndexMet "the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId ); } - final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); - assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); - return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards())); - // return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); + if (sourceIndexMetadata.getCreationVersion().onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION)) { + return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards())); + + } else { + final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); + return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); + } } /** @@ -3168,12 +3172,15 @@ public static Set selectShrinkShards(int shardId, IndexMetadata sourceI } int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = Sets.newHashSetWithExpectedSize(routingFactor); - for (int i = shardId; i < sourceIndexMetadata.getNumberOfShards(); i += numTargetShards) { - assert Math.floorMod(i, numTargetShards) == shardId; - shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); - } - for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) { - // shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + if (sourceIndexMetadata.getCreationVersion().onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION)) { + for (int i = shardId; i < sourceIndexMetadata.getNumberOfShards(); i += numTargetShards) { + assert Math.floorMod(i, numTargetShards) == shardId; + shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + } + } else { + for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) { + shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + } } return shards; } From ff8bbb17aa30a1431106bb9a495ed91137339a08 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 28 Oct 2025 16:24:13 -0600 Subject: [PATCH 09/24] Change --- .../org/elasticsearch/cluster/metadata/IndexMetadata.java | 5 +++-- .../elasticsearch/cluster/metadata/IndexMetadataTests.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index edb3fd8fbe3e4..feaba9b137875 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -3075,12 +3075,13 @@ public static ShardId selectSplitShard(int shardId, IndexMetadata sourceIndexMet "the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId ); } + // TODO: Simplify validation for new version + final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); if (sourceIndexMetadata.getCreationVersion().onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION)) { return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards())); } else { - final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); - assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index ab3bfeedc5c49..90f295613fa09 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -303,7 +303,7 @@ public void testGetRoutingFactor() { public void testSelectShrinkShards() { int numberOfReplicas = randomIntBetween(0, 10); IndexMetadata metadata = IndexMetadata.builder("foo") - .settings(indexSettings(32, numberOfReplicas).put("index.version.created", 1)) + .settings(indexSettings(32, numberOfReplicas).put("index.version.created", IndexVersions.MOD_ROUTING_FUNCTION.id())) .creationDate(randomLong()) .build(); Set shardIds = IndexMetadata.selectShrinkShards(0, metadata, 8); @@ -374,7 +374,7 @@ public void testSelectResizeShards() { public void testSelectSplitShard() { IndexMetadata metadata = IndexMetadata.builder("foo") - .settings(indexSettings(2, 0).put("index.version.created", 1)) + .settings(indexSettings(2, 0).put("index.version.created", IndexVersions.MOD_ROUTING_FUNCTION.id())) .creationDate(randomLong()) .setRoutingNumShards(4) .build(); From 1db460c17c80728bad705922735f7ef1dcfd5d4e Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 17 Nov 2025 12:52:40 -0700 Subject: [PATCH 10/24] Fix tests --- .../src/main/resources/completion.csv-spec | 14 +- .../src/main/resources/fork.csv-spec | 10 +- .../src/main/resources/fuse.csv-spec | 120 +++---- .../src/main/resources/rerank.csv-spec | 40 +-- .../main/resources/score-function.csv-spec | 34 +- .../src/main/resources/scoring.csv-spec | 300 +++++++++--------- .../src/main/resources/subquery.csv-spec | 16 +- 7 files changed, 267 insertions(+), 267 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec index 9e2f88fd99d42..4e339fe828e04 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec @@ -38,11 +38,11 @@ FROM books METADATA _score | KEEP title, completion ; -title:text | completion:keyword -War and Peace | WAR AND PEACE -War and Peace (Signet Classics) | WAR AND PEACE (SIGNET CLASSICS) +title:text | completion:keyword +War and Peace | WAR AND PEACE +War and Peace: A Novel (6 Volumes) | WAR AND PEACE: A NOVEL (6 VOLUMES) ; - + completion using a function as a prompt required_capability: completion required_capability: match_operator_colon @@ -55,7 +55,7 @@ FROM books METADATA _score | KEEP title, completion ; -title:text | completion:keyword -War and Peace | THIS IS A PROMPT: WAR AND PEACE -War and Peace (Signet Classics) | THIS IS A PROMPT: WAR AND PEACE (SIGNET CLASSICS) +title:text | completion:keyword +War and Peace | THIS IS A PROMPT: WAR AND PEACE +War and Peace: A Novel (6 Volumes) | THIS IS A PROMPT: WAR AND PEACE: A NOVEL (6 VOLUMES) ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec index 131facf5074e3..57ca9931b6292 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec @@ -36,11 +36,11 @@ FROM books METADATA _score // tag::simpleForkWithStats-result[] author:text | score:double | _fork:keyword | total:long -William Faulkner | 2.39 | fork1 | null -William Faulkner | 2.39 | fork1 | null -Colleen Faulkner | 1.59 | fork1 | null -Danny Faulkner | 1.59 | fork1 | null -Keith Faulkner | 1.59 | fork1 | null +Colleen Faulkner | 2.18 | fork1 | null +William Faulkner | 2.18 | fork1 | null +Danny Faulkner | 2.02 | fork1 | null +Paul Faulkner | 2.02 | fork1 | null +William Faulkner | 2.02 | fork1 | null null | null | fork2 | 18 // end::simpleForkWithStats-result[] ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec index eee7bce1d1042..deb6fbf8cbfd4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec @@ -40,8 +40,8 @@ _score:double | _fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithDisjunctionAndPostFilter @@ -62,9 +62,9 @@ FROM books METADATA _id, _index, _score _score:double | _fork:keyword | _id:keyword 0.03252 | [fork1, fork2] | 60 -0.032 | [fork1, fork2] | 1 +0.032 | [fork1, fork2] | 29 0.01639 | fork2 | 4 -0.01587 | fork1 | 40 +0.01587 | fork1 | 1 ; fuseWithStats @@ -106,12 +106,12 @@ FROM books METADATA _id, _index, _score ; _score:double | author:keyword | title:keyword | _fork:keyword -0.0328 | [Keith Faulkner, Rory Tyger] | Pop! Went Another Ba | [fork1, fork4] +0.0325 | [Keith Faulkner, Rory Tyger] | Pop! Went Another Ba | [fork1, fork4] 0.0164 | J.R.R. Tolkien | Letters of J R R Tol | fork3 -0.0164 | Ursula K. Le Guin | The wind's twelve qu | fork2 -0.0161 | [Beverlie Manson, Keith Faulkner] | Rainbow's End: A Mag | fork1 +0.0164 | [Beverlie Manson, Keith Faulkner] | Rainbow's End: A Mag | fork1 +0.0164 | Ursula K. Le Guin | Worlds of Exile and | fork2 0.0161 | Ursula K. Le Guin | The Word For World i | fork2 -0.0159 | Ursula K. Le Guin | The Dispossessed | fork2 +0.0159 | Ursula K. Le Guin | Steering the Craft | fork2 ; fuseWithSemanticSearch @@ -155,8 +155,8 @@ _score:double | _fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndRankConstant @@ -179,8 +179,8 @@ _score:double | _fork:keyword | _id:keyword 0.03922 | [fork1, fork2] | 4 0.01923 | fork1 | 56 0.01923 | fork2 | 60 -0.01887 | fork2 | 1 0.01887 | fork1 | 26 +0.01887 | fork2 | 29 ; fuseWithRrfAndWeights @@ -202,7 +202,7 @@ FROM books METADATA _id, _index, _score _score:double | _fork:keyword | _id:keyword 0.01639 | [fork1, fork2] | 4 0.01129 | fork2 | 60 -0.01111 | fork2 | 1 +0.01111 | fork2 | 29 0.00484 | fork1 | 56 0.00476 | fork1 | 26 ; @@ -226,7 +226,7 @@ FROM books METADATA _id, _score, _index _score:double | _fork:keyword | _id:keyword 0.01639 | [fork1, fork2] | 4 0.01129 | fork2 | 60 -0.01111 | fork2 | 1 +0.01111 | fork2 | 29 0.00484 | fork1 | 56 0.00476 | fork1 | 26 ; @@ -251,8 +251,8 @@ my_score:double | _fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndDiscriminatorColumn @@ -277,8 +277,8 @@ _score:double | new_fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndDiscriminatorColumnWithDots @@ -304,8 +304,8 @@ _score:double | new.fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndKeyColumns @@ -329,8 +329,8 @@ _score:double | _fork:keyword | new_id:keyword 0.03279 | [fork1, fork2] | 44 0.01613 | fork1 | 5656 0.01613 | fork2 | 6060 -0.01587 | fork2 | 11 0.01587 | fork1 | 2626 +0.01587 | fork2 | 2929 ; @@ -358,8 +358,8 @@ new_score:double | new_fork:keyword | new_id:keyword 0.03279 | [A, B] | 44 0.01613 | A | 5656 0.01613 | B | 6060 -0.01587 | B | 11 0.01587 | A | 2626 +0.01587 | B | 2929 ; fuseWithSimpleLinear @@ -379,11 +379,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -2.44669 | [fork1, fork2] | 4 -2.2543 | fork1 | 26 -2.10316 | fork1 | 56 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 +2.09421 | fork1 | 26 +2.08412 | [fork1, fork2] | 4 +1.96182 | fork1 | 56 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 ; fuseWithLinearAndL2Norm @@ -403,11 +403,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -0.93394 | [fork1, fork2] | 4 -0.67019 | fork2 | 1 -0.62834 | fork1 | 26 -0.61018 | fork2 | 60 -0.58621 | fork1 | 56 +0.89318 | [fork1, fork2] | 4 +0.69946 | fork2 | 29 +0.65012 | fork1 | 26 +0.60902 | fork1 | 56 +0.56408 | fork2 | 60 ; fuseWithLinearAndMinMax @@ -428,10 +428,10 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -1.0 | fork2 | 1 1.0 | fork1 | 26 -0.7577 | fork2 | 60 -0.63972 | fork1 | 56 +1.0 | fork2 | 29 +0.79006 | fork1 | 56 +0.4806 | fork2 | 60 0.0 | [fork1, fork2] | 4 ; @@ -452,11 +452,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -0.97876 | [fork1, fork2] | 4 -0.67939 | fork2 | 1 -0.67629 | fork1 | 26 -0.63095 | fork1 | 56 -0.61856 | fork2 | 60 +0.87343 | [fork1, fork2] | 4 +0.69234 | fork2 | 29 +0.62826 | fork1 | 26 +0.58855 | fork1 | 56 +0.55834 | fork2 | 60 ; fuseWithLinearAndPartialWeights @@ -476,11 +476,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -1.16233 | [fork1, fork2] | 4 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 -0.67629 | fork1 | 26 -0.63095 | fork1 | 56 +1.05958 | [fork1, fork2] | 4 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 +0.62826 | fork1 | 26 +0.58855 | fork1 | 56 ; fuseWithLinearWeightsAndMinMax @@ -500,10 +500,10 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -0.7 | fork2 | 1 -0.53039 | fork2 | 60 +0.7 | fork2 | 29 +0.33642 | fork2 | 60 0.3 | fork1 | 26 -0.19191 | fork1 | 56 +0.23702 | fork1 | 56 0.0 | [fork1, fork2] | 4 ; @@ -525,11 +525,11 @@ FROM books METADATA _id, _index, _score ; my_score:double | _fork:keyword | _id:keyword -2.44669 | [fork1, fork2] | 4 -2.2543 | fork1 | 26 -2.10316 | fork1 | 56 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 +2.09421 | fork1 | 26 +2.08412 | [fork1, fork2] | 4 +1.96182 | fork1 | 56 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 ; @@ -552,11 +552,11 @@ FROM books METADATA _id, _index, _score ; _score:double | new_fork:keyword | _id:keyword -2.44669 | [fork1, fork2] | 4 -2.2543 | fork1 | 26 -2.10316 | fork1 | 56 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 +2.09421 | fork1 | 26 +2.08412 | [fork1, fork2] | 4 +1.96182 | fork1 | 56 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 ; fuseWithLinearAndKeyColumns @@ -577,11 +577,11 @@ FROM books METADATA _id, _score ; _score:double | _fork:keyword | new_id:keyword -2.44669 | [fork1, fork2] | 44 -2.2543 | fork1 | 2626 -2.10316 | fork1 | 5656 -0.97056 | fork2 | 11 -0.88365 | fork2 | 6060 +2.09421 | fork1 | 2626 +2.08412 | [fork1, fork2] | 44 +1.96182 | fork1 | 5656 +0.98906 | fork2 | 2929 +0.79763 | fork2 | 6060 ; fuseWithLinearAllOptionsScoreGroupAndKeyColumns @@ -605,10 +605,10 @@ FROM books METADATA _id, _score ; new_score:double | new_fork:keyword | new_id:keyword -1.0 | B | 11 1.0 | A | 2626 -0.7577 | B | 6060 -0.63972 | A | 5656 +1.0 | B | 2929 +0.79006 | A | 5656 +0.4806 | B | 6060 0.0 | [A, B] | 44 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec index e7eb7dd3c7021..9eaa4e803e397 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec @@ -17,8 +17,8 @@ FROM books METADATA _score book_no:keyword | title:text | author:text | _score:double 5327 | War and Peace | Leo Tolstoy | 0.08 -4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 +4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 ; @@ -35,10 +35,10 @@ FROM books METADATA _score ; book_no:keyword | title:text | author:text | rerank_score:double -5327 | War and Peace | Leo Tolstoy | 0.08 +5327 | War and Peace | Leo Tolstoy | 0.08 +9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 -9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 -2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 +2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 ; reranker using a single field, create a mew column, sort by rerank_score @@ -56,9 +56,9 @@ FROM books METADATA _score book_no:keyword | title:text | author:text | rerank_score:double 2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 -9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 -5327 | War and Peace | Leo Tolstoy | 0.08 +9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 +5327 | War and Peace | Leo Tolstoy | 0.08 ; reranker using a non text fields @@ -94,10 +94,10 @@ FROM books METADATA _score | KEEP book_no, title, author, collection, rerank_score, _score ; -book_no:keyword | title:text | author:text | collection:text | rerank_score:double | _score:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | J. R. R. Tolkien | The Lord of the Rings | 0.05 | 8.56 -2675 | The Lord of the Rings - Boxed Set | J.R.R. Tolkien | The Lord of the Rings | 0.05 | 8.35 -7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | [Alan Lee, J. R. R. Tolkien] | null | null | 5.5 +book_no:keyword | title:text | author:text | collection:text | rerank_score:double | _score:double +2675 | The Lord of the Rings - Boxed Set | J.R.R. Tolkien | The Lord of the Rings | 0.05 | 7.28 +2714 | Return of the King Being the Third Part of The Lord of the Rings | J. R. R. Tolkien | The Lord of the Rings | 0.05 | 7.25 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | [Agnes Perkins, Charles Adolph Huttar, John Ronald Reuel Tolkien, Walter Scheps] | null | null | 5.36 ; @@ -219,9 +219,9 @@ FROM books METADATA _id, _index, _score | KEEP book_no, title, author, _score ; -book_no:keyword | title:keyword | author:keyword | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 0.04 -2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 0.03 +book_no:keyword | title:keyword | author:keyword | _score:double +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 0.04 +7635 | Oliphaunt (Beastly Verse) | J. R. R. Tolkien | 0.04 ; @@ -240,10 +240,10 @@ FROM books METADATA _score ; // tag::simple-query-result[] -title:text | _score:double -Poems from the Hobbit | 0.0015673980815336108 -A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 0.007936508394777775 -Return of the King Being the Third Part of The Lord of the Rings | 9.960159659385681E-4 +title:text | _score:double +Poems from the Hobbit | 0.0015673980815336108 +The Lord of the Rings - Boxed Set | 0.0011135857785120606 +Letters of J R R Tolkien | 0.0024999999441206455 // end::simple-query-result[] ; @@ -264,9 +264,9 @@ FROM books METADATA _score // tag::two-queries-result[] title:text | _score:double | rerank_score:double -Return of the Shadow | 2.8181066513061523 | 5.740527994930744E-4 -Return of the King Being the Third Part of The Lord of the Rings | 3.6248698234558105 | 9.000900317914784E-4 -The Lays of Beleriand | 1.3002015352249146 | 9.36329597607255E-4 +Return of the Shadow | 3.4218082427978516 | 5.740527994930744E-4 +Return of the King Being the Third Part of The Lord of the Rings | 2.8398752212524414 | 9.000900317914784E-4 +The Lays of Beleriand | 1.5629040002822876 | 9.36329597607255E-4 // end::two-queries-result[] ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec index 00c5fde9e4ee9..ba3c18b9f1c0e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec @@ -18,9 +18,9 @@ FROM books METADATA _score ; // tag::score-single-result[] -book_no:keyword | title:text | _score:double | first_score:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 3.1309072971343994 | 1.9245924949645996 -7350 | Return of the Shadow | 4.8434343338012695 | 3.5432329177856445 +book_no:keyword | title:text | _score:double | first_score:double +2714 | Return of the King Being the Third Part of The Lord of the Rings | 3.167952537536621 | 2.0707292556762695 +7350 | Return of the Shadow | 5.181426048278809 | 3.6185221672058105 // end::score-single-result[] ; @@ -36,9 +36,9 @@ FROM books | LIMIT 5 ; -book_no:keyword | title:text | first_score:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7350 | Return of the Shadow | 3.5432329177856445 +book_no:keyword | title:text | first_score:double +2714 | Return of the King Being the Third Part of The Lord of the Rings | 2.0707292556762695 +7350 | Return of the Shadow | 3.6185221672058105 ; scoreAfterEval @@ -56,9 +56,9 @@ FROM books METADATA _score book_no:keyword | author:text | stars:long | s1:double 2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3 | 0.0 -2713 | William Faulkner | 2 | 1.9043500423431396 +2713 | William Faulkner | 2 | 2.9992568492889404 2847 | Colleen Faulkner | 3 | 0.0 -2883 | William Faulkner | 2 | 1.9043500423431396 +2883 | William Faulkner | 2 | 1.820595145225525 3293 | Danny Faulkner | 2 | 0.0 ; @@ -74,7 +74,7 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 7350 | Return of the Shadow | 0.0 ; @@ -90,8 +90,8 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7350 | Return of the Shadow | 3.5432329177856445 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +7350 | Return of the Shadow | 3.6185221672058105 ; scoreMatchWithDisjunctionAndFilter @@ -106,8 +106,8 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7350 | Return of the Shadow | 3.5432329177856445 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +7350 | Return of the Shadow | 3.6185221672058105 ; scoreMatchDisjunctionNonPushable @@ -122,7 +122,7 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 7350 | Return of the Shadow | 0.0 ; @@ -154,7 +154,7 @@ FROM books ; title_match_ring:double | title_match_hobbit:double -0 | 3.45605206489563 +0 | 3.03169584274292 ; evalUsingScoreAndTopN @@ -175,7 +175,7 @@ book_no:keyword | title_match_towers:double 9607 | 0 9032 | 0 8956 | 0 -8875 | 3.5007452964782715 +8875 | 3.9344332218170166 8873 | 0 8678 | 0 8615 | 0 @@ -198,5 +198,5 @@ FROM books ; book_no:keyword | title_match_world:double -7912 | 2.7460334300994873 +7912 | 2.6122074127197266 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index 221e913e83497..32117c3de948e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -6,37 +6,37 @@ singleQstrBoostScoringSorted required_capability: metadata_score required_capability: qstr_function -from books metadata _score +from books metadata _score | where qstr("author:Lord Rings^2") -| eval c_score = ceil(_score) -| keep book_no, title, c_score +| eval c_score = ceil(_score) +| keep book_no, title, c_score | sort c_score desc, book_no asc | LIMIT 2; -book_no:keyword | title:text | c_score:double -1463 | Realms of Tolkien: Images of Middle-earth | 6.0 -2675 | The Lord of the Rings - Boxed Set | 6.0 +book_no:keyword | title:text | c_score:double +8875 | The Two Towers | 7.0 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 6.0 ; singleMatchWithKeywordFieldScoring required_capability: metadata_score required_capability: match_operator_colon -from books metadata _score -| where author.keyword:"William Faulkner" -| keep book_no, author, _score +from books metadata _score +| where author.keyword:"William Faulkner" +| keep book_no, author, _score | sort book_no; -book_no:keyword | author:text | _score:double -2713 | William Faulkner | 1.7589385509490967 -2883 | William Faulkner | 1.7589385509490967 -4724 | William Faulkner | 1.7589385509490967 -4977 | William Faulkner | 2.6145541667938232 -5119 | William Faulkner | 2.513157367706299 -5404 | William Faulkner | 1.7589385509490967 -5578 | William Faulkner | 2.513157367706299 -8077 | William Faulkner | 1.7589385509490967 -9896 | William Faulkner | 2.6145541667938232 +book_no:keyword | author:text | _score:double +2713 | William Faulkner | 2.743302583694458 +2883 | William Faulkner | 1.6706234216690063 +4724 | William Faulkner | 1.6706234216690063 +4977 | William Faulkner | 1.6706234216690063 +5119 | William Faulkner | 1.6706234216690063 +5404 | William Faulkner | 1.6706234216690063 +5578 | William Faulkner | 3.1982219219207764 +8077 | William Faulkner | 1.6706234216690063 +9896 | William Faulkner | 1.6706234216690063 ; qstrWithFieldAndScoringSortedEval @@ -50,10 +50,10 @@ from books metadata _score | keep book_no, title, _score | limit 3; -book_no:keyword | title:text | _score:double -2675 | The Lord of the Rings - Boxed Set | 2.5619282722473145 -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.746896743774414 +book_no:keyword | title:text | _score:double +2675 | The Lord of the Rings - Boxed Set | 1.9618241786956787 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 1.4636166095733643 ; qstrWithFieldAndScoringSorted @@ -66,26 +66,26 @@ from books metadata _score | keep book_no, title, _score | limit 3; -book_no:keyword | title:text | _score:double -2675 | The Lord of the Rings - Boxed Set | 2.5619282722473145 -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.746896743774414 +book_no:keyword | title:text | _score:double +2675 | The Lord of the Rings - Boxed Set | 1.9618241786956787 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 1.4636166095733643 ; singleQstrScoringManipulated required_capability: metadata_score required_capability: qstr_function -from books metadata _score -| where qstr("author:William Faulkner") -| eval add_score = ceil(_score) + 1 -| keep book_no, author, add_score -| sort book_no +from books metadata _score +| where qstr("author:William Faulkner") +| eval add_score = ceil(_score) + 1 +| keep book_no, author, add_score +| sort book_no | LIMIT 2; book_no:keyword | author:text | add_score:double 2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3.0 -2713 | William Faulkner | 6.0 +2713 | William Faulkner | 7.0 ; testMultiValuedFieldWithConjunctionWithScore @@ -97,7 +97,7 @@ from books metadata _score | keep book_no, title, author, _score; book_no:keyword | title:text | author:text | _score:double -6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.5822172164917 +6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.821435928344727 ; testMatchAndQueryStringFunctionsWithScore @@ -110,8 +110,8 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | _score:double -3535 | Rainbow's End: A Magical Story and Moneybox | [Beverlie Manson, Keith Faulkner] | 6.5579609870910645 -6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 5.975414276123047 +3535 | Rainbow's End: A Magical Story and Moneybox | [Beverlie Manson, Keith Faulkner] | 5.961376190185547 +6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 5.961376190185547 ; testMultiMatchWithScore @@ -124,7 +124,7 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | _score:double -2847 | To Love A Dark Stranger (Lovegram Historical Romance) | Colleen Faulkner | 1.9662091732025146 +2847 | To Love A Dark Stranger (Lovegram Historical Romance) | Colleen Faulkner | 2.0541491508483887 ; testMultiMatchWithScore1 @@ -139,19 +139,19 @@ from books metadata _score ignoreOrder:true book_no:keyword | _score:double -1463 | 2.0 -2301 | 2.0 -2675 | 2.0 -2714 | 2.0 -2936 | 1.0 -4023 | 2.0 -4289 | 3.0 -5335 | 2.0 -5996 | 2.0 -6405 | 2.0 -6760 | 2.0 -7350 | 2.0 -7480 | 3.0 +1463 | 2.0 +2301 | 1.0 +2675 | 3.0 +2714 | 2.0 +2936 | 2.0 +4023 | 2.0 +4289 | 3.0 +5335 | 2.0 +5996 | 1.0 +6405 | 3.0 +6760 | 2.0 +7350 | 2.0 +7480 | 4.0 ; testMultiMatchWithScore2 @@ -166,19 +166,19 @@ from books metadata _score ignoreOrder:true book_no:keyword | _score:double -1463 | 2.0 -2301 | 2.0 -2675 | 2.0 -2714 | 2.0 -2936 | 1.0 -4023 | 2.0 -4289 | 6.0 -5335 | 2.0 -5996 | 2.0 -6405 | 2.0 -6760 | 2.0 -7350 | 2.0 -7480 | 3.0 +1463 | 2.0 +2301 | 1.0 +2675 | 3.0 +2714 | 2.0 +2936 | 2.0 +4023 | 2.0 +4289 | 6.0 +5335 | 2.0 +5996 | 1.0 +6405 | 3.0 +6760 | 2.0 +7350 | 2.0 +7480 | 4.0 ; multipleWhereWithMatchScoringNoSort @@ -191,8 +191,8 @@ from books metadata _score | keep book_no, title, author, _score; ignoreOrder:true -book_no:keyword | title:text | author:text | _score:double -8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 11.193471908569336 +book_no:keyword | title:text | author:text | _score:double +8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 13.820621490478516 ; multipleWhereWithMatchScoring @@ -205,8 +205,8 @@ from books metadata _score | keep book_no, title, author, _score | sort book_no; -book_no:keyword | title:text | author:text | _score:double -8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 11.193471908569336 +book_no:keyword | title:text | author:text | _score:double +8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 13.820621490478516 ; combinedMatchWithFunctionsScoring @@ -221,39 +221,39 @@ from books metadata _score | sort book_no; book_no:keyword | title:text | author:text | year:integer | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.733664035797119 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.3956053256988525 ; singleQstrScoring required_capability: metadata_score required_capability: qstr_function -from books metadata _score -| where qstr("author:William Faulkner") -| keep book_no, author, _score -| sort book_no +from books metadata _score +| where qstr("author:William Faulkner") +| keep book_no, author, _score +| sort book_no | LIMIT 2; book_no:keyword | author:text | _score:double -2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.3697924613952637 -2713 | William Faulkner | 4.631696701049805 +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.2796473503112793 +2713 | William Faulkner | 5.796701431274414 ; singleQstrScoringGrok required_capability: metadata_score required_capability: qstr_function -from books metadata _score -| where qstr("author:Lord Rings") -| GROK title "%{WORD:title} %{WORD}" -| sort _score desc -| keep book_no, title, _score +from books metadata _score +| where qstr("author:Lord Rings") +| GROK title "%{WORD:title} %{WORD}" +| sort _score desc +| keep book_no, title, _score | LIMIT 3; book_no:keyword | title:keyword | _score:double -8875 | The | 2.769660472869873 -1463 | Realms | 2.6714818477630615 -2675 | The | 2.5619282722473145 +8875 | The | 3.341614246368408 +4023 | A | 2.855410575866699 +1463 | Realms | 2.465149402618408 ; combinedMatchWithScoringEvalNoSort @@ -276,35 +276,35 @@ singleQstrScoringRename required_capability: metadata_score required_capability: qstr_function -from books metadata _score -| where qstr("author:Lord Rings") -| rename _score as rank -| sort rank desc -| keep book_no, rank +from books metadata _score +| where qstr("author:Lord Rings") +| rename _score as rank +| sort rank desc +| keep book_no, rank | LIMIT 3; book_no:keyword | rank:double -8875 | 2.769660472869873 -1463 | 2.6714818477630615 -2675 | 2.5619282722473145 +8875 | 3.341614246368408 +4023 | 2.855410575866699 +1463 | 2.465149402618408 ; singleMatchWithTextFieldScoring required_capability: metadata_score required_capability: match_operator_colon -from books metadata _score -| where author:"William Faulkner" -| sort book_no -| keep book_no, author, _score +from books metadata _score +| where author:"William Faulkner" +| sort book_no +| keep book_no, author, _score | limit 5; book_no:keyword | author:text | _score:double -2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.3697924613952637 -2713 | William Faulkner | 3.2750158309936523 -2847 | Colleen Faulkner | 1.593343734741211 -2883 | William Faulkner | 3.2750158309936523 -3293 | Danny Faulkner | 1.593343734741211 +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.2796473503112793 +2713 | William Faulkner | 5.01743221282959 +2847 | Colleen Faulkner | 1.3154147863388062 +2883 | William Faulkner | 3.136009931564331 +3293 | Danny Faulkner | 2.0181751251220703 ; combinedMatchWithFunctionsScoringNoSort @@ -319,7 +319,7 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | year:integer | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.733664035797119 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.3956053256988525 ; combinedMatchWithScoringEval @@ -342,34 +342,34 @@ singleQstrScoringEval required_capability: metadata_score required_capability: qstr_function -from books metadata _score -| where qstr("author:Lord Rings") -| eval c_score = ceil(_score) -| keep book_no, c_score -| sort book_no desc +from books metadata _score +| where qstr("author:Lord Rings") +| eval c_score = ceil(_score) +| keep book_no, c_score +| sort book_no desc | LIMIT 3; book_no:keyword | c_score:double -8875 | 3.0 -7480 | 1.0 -7350 | 1.0 +8875 | 4.0 +7480 | 2.0 +7350 | 2.0 ; QstrScoreManipulation required_capability: metadata_score required_capability: qstr_function -from books metadata _score -| where qstr("title:gentle") -| eval _score = _score + 1 +from books metadata _score +| where qstr("title:gentle") +| eval _score = _score + 1 | keep book_no, title, _score | limit 10 ; ignoreOrder:true book_no:keyword | title:text | _score:double -2924 | A Gentle Creature and Other Stories: White Nights, A Gentle Creature, and The Dream of a Ridiculous Man (The World's Classics) | 3.158426523208618 -5948 | That We Are Gentle Creatures | 3.727346897125244 +2924 | A Gentle Creature and Other Stories: White Nights, A Gentle Creature, and The Dream of a Ridiculous Man (The World's Classics) | 3.112499237060547 +5948 | That We Are Gentle Creatures | 3.709021806716919 ; QstrScoreOverride @@ -469,17 +469,17 @@ conjunctionScoresPushableNonPushableFunctions required_capability: metadata_score required_capability: match_function -from books metadata _score +from books metadata _score | where match(title, "Lord") and length(title) > 20 | keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -2675 | 2.5619282722473145 -2714 | 1.9245924949645996 -7140 | 1.746896743774414 -4023 | 1.5062403678894043 +book_no:keyword | _score:double +2675 | 1.9618241786956787 +2714 | 1.7047617435455322 +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 ; conjunctionScoresPushableFunctions @@ -488,15 +488,15 @@ required_capability: metadata_score required_capability: match_function required_capability: non_full_text_functions_scoring -from books metadata _score +from books metadata _score | where match(title, "Lord") and ratings > 4.6 | keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -7140 | 1.746896743774414 -4023 | 1.5062403678894043 +book_no:keyword | _score:double +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 ; disjunctionScoresPushableNonPushableFunctions @@ -506,17 +506,17 @@ required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score required_capability: non_full_text_functions_scoring -from books metadata _score +from books metadata _score | where match(title, "Lord") or length(title) > 100 | keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -2675 | 2.5619282722473145 -2714 | 1.9245924949645996 -7140 | 1.746896743774414 -4023 | 1.5062403678894043 +book_no:keyword | _score:double +2675 | 1.9618241786956787 +2714 | 1.7047617435455322 +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 2924 | 0.0 8678 | 0.0 ; @@ -527,21 +527,21 @@ required_capability: metadata_score required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score -from books metadata _score -| where (title: "Lord" and length(title) > 40) or (author: "Dostoevsky" and length(title) > 40) -| keep book_no, _score +from books metadata _score +| where (title: "Lord" and length(title) > 40) or (author: "Dostoevsky" and length(title) > 40) +| keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -8086 | 2.786686897277832 -9801 | 2.786686897277832 -1937 | 2.1503653526306152 -8534 | 2.1503653526306152 -2714 | 1.9245924949645996 -7140 | 1.746896743774414 -4023 | 1.5062403678894043 -2924 | 1.2732219696044922 +book_no:keyword | _score:double +1937 | 2.7213401794433594 +9801 | 2.7213401794433594 +8086 | 2.4077744483947754 +8534 | 2.4077744483947754 +2714 | 1.7047617435455322 +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 +2924 | 1.017024040222168 ; statsScores @@ -550,13 +550,13 @@ required_capability: metadata_score required_capability: match_function required_capability: full_text_functions_in_stats_where -from books metadata _score +from books metadata _score | where match(title, "Lord Rings", {"operator": "AND"}) | stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score) ; -avg_score:double | max_score:double | min_score:double -3.869828939437866 | 5.123856544494629 | 3.0124807357788086 +avg_score:double | max_score:double | min_score:double +3.245103120803833 | 3.9236483573913574 | 2.7200074195861816 ; testMatchPhraseWithScore @@ -570,9 +570,9 @@ from books metadata _score | sort _score desc ; -book_no:keyword | title:text | author:text | _score:double - 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 9.017186164855957 - 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 8.412636756896973 +book_no:keyword | title:text | author:text | _score:double + 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 8.376850128173828 + 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 7.847296714782715 ; testMatchPhraseWithScoreBoost @@ -584,7 +584,7 @@ from books metadata _score | sort _score desc ; -book_no:keyword | title:text | author:text | _score:double - 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 45.0859260559082 - 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 42.06318283081055 +book_no:keyword | title:text | author:text | _score:double + 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 41.88425064086914 + 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 39.23648452758789 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec index 930ca874488b6..2e0efb627e532 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec @@ -722,10 +722,10 @@ FROM sample_data, (FROM books metadata _score ; ignoreOrder:true -title:text | completion:keyword | @timestamp:datetime | client_ip:ip -War and Peace | WAR AND PEACE | null | null -War and Peace (Signet Classics) | WAR AND PEACE (SIGNET CLASSICS) | null | null -null | null | 2023-10-23T13:33:34.937Z | 172.21.0.5 +title:text | completion:keyword | @timestamp:datetime | client_ip:ip +War and Peace | WAR AND PEACE | null | null +War and Peace: A Novel (6 Volumes) | WAR AND PEACE: A NOVEL (6 VOLUMES) | null | null +null | null | 2023-10-23T13:33:34.937Z | 172.21.0.5 ; subqueryInFromWithCompletionInMainQuery @@ -766,10 +766,10 @@ FROM sample_data, (FROM books METADATA _score ; ignoreOrder:true -book_no:keyword | title:text | author:text | _score:double | @timestamp:datetime -4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 | null -5327 | War and Peace | Leo Tolstoy | 0.08 | null -null | null | null | null | 2023-10-23T13:33:34.937Z +book_no:keyword | title:text | author:text | _score:double | @timestamp:datetime +5327 | War and Peace | Leo Tolstoy | 0.08 | null +9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 | null +null | null | null | null | 2023-10-23T13:33:34.937Z ; subqueryInFromWithRerankInMainQuery From 02d92b446ddc416b4b99a913c9bd352d8a773563 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 17 Nov 2025 14:52:21 -0700 Subject: [PATCH 11/24] Fixes --- .../examples/fork.csv-spec/simpleForkWithStats.md | 10 +++++----- .../commands/examples/rerank.csv-spec/simple-query.md | 4 ++-- .../commands/examples/rerank.csv-spec/two-queries.md | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md b/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md index d72ea42b3106f..db97c8e77d4bf 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md +++ b/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md @@ -11,9 +11,9 @@ FROM books METADATA _score | author:text | score:double | _fork:keyword | total:long | | --- | --- | --- | --- | -| William Faulkner | 2.39 | fork1 | null | -| William Faulkner | 2.39 | fork1 | null | -| Colleen Faulkner | 1.59 | fork1 | null | -| Danny Faulkner | 1.59 | fork1 | null | -| Keith Faulkner | 1.59 | fork1 | null | +| Colleen Faulkner | 2.18 | fork1 | null | +| William Faulkner | 2.18 | fork1 | null | +| Danny Faulkner | 2.02 | fork1 | null | +| Paul Faulkner | 2.02 | fork1 | null | +| William Faulkner | 2.02 | fork1 | null | | null | null | fork2 | 18 | diff --git a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md index 45e7c34030162..d8dd01e0f6b9b 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md +++ b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md @@ -13,5 +13,5 @@ FROM books METADATA _score | title:text | _score:double | | --- | --- | | Poems from the Hobbit | 0.0015673980815336108 | -| A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 0.007936508394777775 | -| Return of the King Being the Third Part of The Lord of the Rings | 9.960159659385681E-4 | +| The Lord of the Rings - Boxed Set | 0.0011135857785120606 | +| Letters of J R R Tolkien | 0.0024999999441206455 | diff --git a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md index d9ed44560d698..20d1f2dd3cadf 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md +++ b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md @@ -13,6 +13,6 @@ FROM books METADATA _score | title:text | _score:double | rerank_score:double | | --- | --- | --- | -| Return of the Shadow | 2.8181066513061523 | 5.740527994930744E-4 | -| Return of the King Being the Third Part of The Lord of the Rings | 3.6248698234558105 | 9.000900317914784E-4 | -| The Lays of Beleriand | 1.3002015352249146 | 9.36329597607255E-4 | +| Return of the Shadow | 3.4218082427978516 | 5.740527994930744E-4 | +| Return of the King Being the Third Part of The Lord of the Rings | 2.8398752212524414 | 9.000900317914784E-4 | +| The Lays of Beleriand | 1.5629040002822876 | 9.36329597607255E-4 | From 77d0e8d7f037603471096982a9351444b29a2778 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 18 Nov 2025 14:46:24 -0700 Subject: [PATCH 12/24] Change --- .../src/main/resources/completion.csv-spec | 2 ++ .../src/main/resources/fork.csv-spec | 1 + .../src/main/resources/fuse.csv-spec | 22 +++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 +++++ 4 files changed, 30 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec index 4e339fe828e04..5b0049ec7d80f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec @@ -29,6 +29,7 @@ prompt:keyword | completion_output:keywor completion after a search required_capability: completion required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -46,6 +47,7 @@ War and Peace: A Novel (6 Volumes) | WAR AND PEACE: A NOVEL (6 VOLUMES) completion using a function as a prompt required_capability: completion required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec index 57ca9931b6292..d11a83499dfa8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec @@ -23,6 +23,7 @@ emp_no:integer | _fork:keyword simpleForkWithStats required_capability: fork_v9 +required_capability: routing_function_update // tag::simpleForkWithStats[] FROM books METADATA _score diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec index deb6fbf8cbfd4..fd7b1880c07c1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec @@ -25,6 +25,7 @@ fuseWithMatchAndScore required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -48,6 +49,7 @@ fuseWithDisjunctionAndPostFilter required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" OR author:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -91,6 +93,7 @@ fuseWithMultipleForkBranches required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK (WHERE author:"Keith Faulkner" AND qstr("author:Rory or author:Beverlie") | SORT _score, _id DESC | LIMIT 3) @@ -140,6 +143,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -164,6 +168,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -188,6 +193,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -212,6 +218,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score, _index | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -236,6 +243,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL my_score = _score | DROP _score) @@ -260,6 +268,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3) @@ -287,6 +296,7 @@ required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score required_capability: dots_in_fuse +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3) @@ -313,6 +323,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL new_id = CONCAT(_id, _id) | DROP _id) @@ -339,6 +350,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 @@ -367,6 +379,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -391,6 +404,7 @@ fuseWithLinearAndL2Norm required_capability: fork_v9 required_capability: fuse_v6 required_capability: fuse_l2_norm +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -416,6 +430,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -440,6 +455,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -464,6 +480,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -488,6 +505,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -513,6 +531,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL my_score = _score | DROP _score) @@ -538,6 +557,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3) @@ -564,6 +584,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL new_id = CONCAT(_id, _id) | DROP _id) @@ -589,6 +610,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 8b826ea05cc94..dc4391ad3a05c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1677,6 +1677,11 @@ public enum Cap { */ TIME_SERIES_WINDOW_V0, + /** + * Marks the move to the hash(doc) % shard_count routing function. Added in #137062. + */ + ROUTING_FUNCTION_UPDATE, + // Last capability should still have a comma for fewer merge conflicts when adding new ones :) // This comment prevents the semicolon from being on the previous capability when Spotless formats the file. ; From 9c9704de81fde2a92c5ee5604a5e922b6162f99e Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 18 Nov 2025 15:04:29 -0700 Subject: [PATCH 13/24] Change --- .../src/main/resources/rerank.csv-spec | 7 +++++ .../main/resources/score-function.csv-spec | 10 +++++++ .../src/main/resources/scoring.csv-spec | 27 +++++++++++++++++++ .../src/main/resources/subquery.csv-spec | 2 ++ 4 files changed, 46 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec index 9eaa4e803e397..373a4d3a8e6ee 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec @@ -6,6 +6,7 @@ reranker using a single field, overwrite existing _score column required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -25,6 +26,7 @@ book_no:keyword | title:text | author reranker using a single field, create a mew column required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -44,6 +46,7 @@ book_no:keyword | title:text | author reranker using a single field, create a mew column, sort by rerank_score required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -84,6 +87,7 @@ book_no:keyword | title:text | rating reranker using a sparse input field required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE MATCH(title, "lord of the rings", {"minimum_should_match": "100%"}) AND author:"tolkien" @@ -207,6 +211,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon required_capability: rerank +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -227,6 +232,7 @@ book_no:keyword | title:keyword | author:keyword | _score:double simple required_capability: rerank +required_capability: routing_function_update // tag::simple-query[] FROM books METADATA _score @@ -249,6 +255,7 @@ Letters of J R R Tolkien | 0.0024999999441206455 two_queries required_capability: rerank +required_capability: routing_function_update // tag::two-queries[] FROM books METADATA _score diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec index ba3c18b9f1c0e..681151e976357 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec @@ -6,6 +6,7 @@ scoreSingle required_capability: metadata_score required_capability: score_function required_capability: match_function +required_capability: routing_function_update // tag::score-function[] FROM books METADATA _score @@ -27,6 +28,7 @@ book_no:keyword | title:text scoreSingleNoMetadata required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -45,6 +47,7 @@ scoreAfterEval required_capability: score_function required_capability: metadata_score required_capability: match_function +required_capability: routing_function_update FROM books METADATA _score | EVAL stars = to_long(ratings / 2.0) @@ -65,6 +68,7 @@ book_no:keyword | author:text | stars:lon scoreMatchWithFilterConjunction required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -81,6 +85,7 @@ book_no:keyword | title:text scoreMatchWithDisjunction required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -97,6 +102,7 @@ book_no:keyword | title:text scoreMatchWithDisjunctionAndFilter required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -113,6 +119,7 @@ book_no:keyword | title:text scoreMatchDisjunctionNonPushable required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -146,6 +153,7 @@ evalUsingScoreAndLimit2 required_capability: score_function required_capability: match_function required_capability: pushing_down_eval_with_score +required_capability: routing_function_update FROM books | WHERE author == "J R R Tolkien" @@ -161,6 +169,7 @@ evalUsingScoreAndTopN required_capability: score_function required_capability: match_function required_capability: pushing_down_eval_with_score +required_capability: routing_function_update FROM books | SORT book_no desc @@ -186,6 +195,7 @@ evalUsingScoreAndFilter required_capability: score_function required_capability: match_function required_capability: pushing_down_eval_with_score +required_capability: routing_function_update FROM books | sort book_no desc diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index 32117c3de948e..dbb1ea49a3521 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -5,6 +5,7 @@ singleQstrBoostScoringSorted required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("author:Lord Rings^2") @@ -21,6 +22,7 @@ book_no:keyword | title:text singleMatchWithKeywordFieldScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update from books metadata _score | where author.keyword:"William Faulkner" @@ -42,6 +44,7 @@ book_no:keyword | author:text | _score:double qstrWithFieldAndScoringSortedEval required_capability: qstr_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where qstr("title:rings") @@ -59,6 +62,7 @@ book_no:keyword | title:text qstrWithFieldAndScoringSorted required_capability: qstr_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where qstr("title:rings") @@ -75,6 +79,7 @@ book_no:keyword | title:text singleQstrScoringManipulated required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("author:William Faulkner") @@ -91,6 +96,7 @@ book_no:keyword | author:text | add_score testMultiValuedFieldWithConjunctionWithScore required_capability: match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where match(author, "Keith Faulkner") and match(author, "Rory Tyger") @@ -103,6 +109,7 @@ book_no:keyword | title:text testMatchAndQueryStringFunctionsWithScore required_capability: match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where match(author, "Keith Faulkner") and qstr("author:Rory or author:Beverlie") @@ -117,6 +124,7 @@ book_no:keyword | title:text testMultiMatchWithScore required_capability: multi_match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where multi_match("Mark", author, title, {"fuzziness": 1}) @@ -130,6 +138,7 @@ book_no:keyword | title:text | author testMultiMatchWithScore1 required_capability: multi_match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where multi_match("Hobbit", description, title, {"type": "best_fields"}) @@ -157,6 +166,7 @@ book_no:keyword | _score:double testMultiMatchWithScore2 required_capability: multi_match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where multi_match("Hobbit", description, title, {"type": "most_fields"}) @@ -184,6 +194,7 @@ book_no:keyword | _score:double multipleWhereWithMatchScoringNoSort required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update from books metadata _score | where title:"short stories" @@ -198,6 +209,7 @@ book_no:keyword | title:text | author:text multipleWhereWithMatchScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update from books metadata _score | where title:"short stories" @@ -213,6 +225,7 @@ combinedMatchWithFunctionsScoring required_capability: metadata_score required_capability: match_operator_colon required_capability: non_full_text_functions_scoring +required_capability: routing_function_update from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -227,6 +240,7 @@ book_no:keyword | title:text | author:text | year:integer | _sc singleQstrScoring required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("author:William Faulkner") @@ -242,6 +256,7 @@ book_no:keyword | author:text | _score:do singleQstrScoringGrok required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("author:Lord Rings") @@ -275,6 +290,7 @@ book_no:keyword | title:text | author:text | year:integer | c_s singleQstrScoringRename required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("author:Lord Rings") @@ -292,6 +308,7 @@ book_no:keyword | rank:double singleMatchWithTextFieldScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update from books metadata _score | where author:"William Faulkner" @@ -311,6 +328,7 @@ combinedMatchWithFunctionsScoringNoSort required_capability: metadata_score required_capability: match_operator_colon required_capability: non_full_text_functions_scoring +required_capability: routing_function_update from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -341,6 +359,7 @@ book_no:keyword | title:text | author:text | year:integer | c_s singleQstrScoringEval required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("author:Lord Rings") @@ -358,6 +377,7 @@ book_no:keyword | c_score:double QstrScoreManipulation required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update from books metadata _score | where qstr("title:gentle") @@ -468,6 +488,7 @@ conjunctionScoresPushableNonPushableFunctions required_capability: metadata_score required_capability: match_function +required_capability: routing_function_update from books metadata _score | where match(title, "Lord") and length(title) > 20 @@ -487,6 +508,7 @@ conjunctionScoresPushableFunctions required_capability: metadata_score required_capability: match_function required_capability: non_full_text_functions_scoring +required_capability: routing_function_update from books metadata _score | where match(title, "Lord") and ratings > 4.6 @@ -505,6 +527,7 @@ required_capability: metadata_score required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score required_capability: non_full_text_functions_scoring +required_capability: routing_function_update from books metadata _score | where match(title, "Lord") or length(title) > 100 @@ -526,6 +549,7 @@ disjunctionScoresMultipleClauses required_capability: metadata_score required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score +required_capability: routing_function_update from books metadata _score | where (title: "Lord" and length(title) > 40) or (author: "Dostoevsky" and length(title) > 40) @@ -549,6 +573,7 @@ statsScores required_capability: metadata_score required_capability: match_function required_capability: full_text_functions_in_stats_where +required_capability: routing_function_update from books metadata _score | where match(title, "Lord Rings", {"operator": "AND"}) @@ -563,6 +588,7 @@ testMatchPhraseWithScore required_capability: match_phrase_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where match_phrase(title, "J. R. R. Tolkien") @@ -577,6 +603,7 @@ book_no:keyword | title:text | author:text testMatchPhraseWithScoreBoost required_capability: match_phrase_function +required_capability: routing_function_update from books metadata _score | where match_phrase(title, "J. R. R. Tolkien", {"boost": 5}) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec index 2e0efb627e532..2d5be37b7d226 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec @@ -710,6 +710,7 @@ required_capability: fork_v9 required_capability: completion required_capability: match_operator_colon required_capability: subquery_in_from_command +required_capability: routing_function_update FROM sample_data, (FROM books metadata _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -753,6 +754,7 @@ required_capability: fork_v9 required_capability: rerank required_capability: match_operator_colon required_capability: subquery_in_from_command +required_capability: routing_function_update FROM sample_data, (FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" From 1fad860ff99d63040aba0774daadc7ccc341ec8a Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 19 Nov 2025 14:37:41 -0700 Subject: [PATCH 14/24] Change --- .../rest-api-spec/test/reindex/35_search_failures.yml | 4 +++- .../rest-api-spec/test/update_by_query/35_search_failure.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml index 64e5533d2135c..5c4429ba6259a 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml @@ -33,7 +33,9 @@ - match: {error.phase: query} - match: {error.root_cause.0.type: script_exception} - match: {error.root_cause.0.reason: runtime error} - - match: {error.failed_shards.0.shard: 1} + # The shard number can vary based on the shard routing function version. Just check that it's between 0 and 1. + - gte: { error.failed_shards.0.shard: 0 } + - lte: { error.failed_shards.0.shard: 1 } - match: {error.failed_shards.0.index: source} - is_true: error.failed_shards.0.node - match: {error.failed_shards.0.reason.type: script_exception} diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml index 70f6c538dae46..36239b736c979 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml @@ -30,7 +30,9 @@ - match: {error.phase: query} - match: {error.root_cause.0.type: script_exception} - match: {error.root_cause.0.reason: runtime error} - - match: {error.failed_shards.0.shard: 1} + # The shard number can vary based on the shard routing function version. Just check that it's between 0 and 1. + - gte: { error.failed_shards.0.shard: 0 } + - lte: { error.failed_shards.0.shard: 1 } - match: {error.failed_shards.0.index: source} - is_true: error.failed_shards.0.node - match: {error.failed_shards.0.reason.type: script_exception} From bf723d117f7d6c95b3a93866ce6e64fa377ed231 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 19 Nov 2025 14:58:02 -0700 Subject: [PATCH 15/24] Change --- .../org/elasticsearch/cluster/metadata/IndexMetadata.java | 6 ++---- .../org/elasticsearch/cluster/routing/IndexRouting.java | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 9c375acfdec93..5238684c6faa0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -3177,12 +3177,10 @@ public static ShardId selectSplitShard(int shardId, IndexMetadata sourceIndexMet "the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId ); } - // TODO: Simplify validation for new version final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); - if (sourceIndexMetadata.getCreationVersion().onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION)) { + if (IndexRouting.shouldUseShardCountModRouting(sourceIndexMetadata.getCreationVersion())) { return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards())); - } else { return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); } @@ -3275,7 +3273,7 @@ public static Set selectShrinkShards(int shardId, IndexMetadata sourceI } int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = Sets.newHashSetWithExpectedSize(routingFactor); - if (sourceIndexMetadata.getCreationVersion().onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION)) { + if (IndexRouting.shouldUseShardCountModRouting(sourceIndexMetadata.getCreationVersion())) { for (int i = shardId; i < sourceIndexMetadata.getNumberOfShards(); i += numTargetShards) { assert Math.floorMod(i, numTargetShards) == shardId; shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 012ead4b08423..858ae15fee046 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -142,7 +142,7 @@ public void postProcess(IndexRequest indexRequest) {} */ public abstract void collectSearchShards(String routing, IntConsumer consumer); - private static boolean shouldUseShardCountModRouting(final IndexVersion creationVersion) { + public static boolean shouldUseShardCountModRouting(final IndexVersion creationVersion) { return creationVersion.onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION); } From 3d490289459d3c6cf9e4c8c559d98b513287b6af Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 19 Nov 2025 16:18:51 -0700 Subject: [PATCH 16/24] Change --- .../routing/PartitionedRoutingIT.java | 2 -- .../cluster/metadata/IndexMetadata.java | 3 +-- .../SearchPhaseCoordinatorAPMMetricsTests.java | 14 ++++++++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 67cb68d1e3587..18c332515b253 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -81,8 +81,6 @@ public void testShrinking() throws Exception { verifyGets(index, routingToDocumentIds); verifyBroadSearches(index, routingToDocumentIds, currentShards); - // we need the floor and ceiling of the routing_partition_size / factor since the partition size of the shrunken - // index will be one of those, depending on the routing value verifyRoutedSearches(index, routingToDocumentIds, Math.min(partitionSize, currentShards)); updateIndexSettings( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 5238684c6faa0..1f399d0be369d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -278,8 +278,7 @@ public Iterator> settings() { } }, - Property.IndexScope, - Property.DeprecatedWarning + Property.IndexScope ); public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java index 6c48e2fd20873..a5d7d71b3195d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java @@ -72,6 +72,9 @@ private void setUpIndex() throws Exception { prepareIndex(indexName).setId("1").setSource("body", "doc1", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); prepareIndex(indexName).setId("2").setSource("body", "doc2", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); prepareIndex(indexName).setId("3").setSource("body", "doc3", "@timestamp", "2025-01-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("4").setSource("body", "doc4", "@timestamp", "2025-02-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("5").setSource("body", "doc5", "@timestamp", "2025-03-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("6").setSource("body", "doc6", "@timestamp", "2025-04-01").setRefreshPolicy(IMMEDIATE).get(); createIndex( secondIndexName, @@ -82,9 +85,12 @@ private void setUpIndex() throws Exception { ); ensureGreen(secondIndexName); - prepareIndex(secondIndexName).setId("4").setSource("body", "doc1", "@timestamp", "2025-11-01").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(secondIndexName).setId("5").setSource("body", "doc2", "@timestamp", "2025-12-01").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(secondIndexName).setId("6").setSource("body", "doc3", "@timestamp", "2026-01-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("7").setSource("body", "doc1", "@timestamp", "2025-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("8").setSource("body", "doc2", "@timestamp", "2025-12-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("9").setSource("body", "doc3", "@timestamp", "2026-01-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("10").setSource("body", "doc4", "@timestamp", "2026-02-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("11").setSource("body", "doc5", "@timestamp", "2026-03-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("12").setSource("body", "doc6", "@timestamp", "2026-04-01").setRefreshPolicy(IMMEDIATE).get(); } @After @@ -151,7 +157,7 @@ public void testPointInTimeWithPreFiltering() { .setSize(1) .setPreFilterShardSize(1) .setQuery(simpleQueryStringQuery("doc3")), - "6" + "9" ); assertMeasurements(List.of(OPEN_PIT_SEARCH_PHASE_METRIC, QUERY_SEARCH_PHASE_METRIC, FETCH_SEARCH_PHASE_METRIC), 1); assertMeasurements( From 0622d1a8079f6339fed2358c825c8a0fe3e2c820 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 17 Dec 2025 21:51:58 +0000 Subject: [PATCH 17/24] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f2ec23249b0fc..35575aa5ac9f4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -19,8 +19,6 @@ import java.util.Locale; import java.util.Set; -import static org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin.T_DIGEST_ESQL_SUPPORT; - /** * A {@link Set} of "capabilities" supported by the {@link RestEsqlQueryAction} * and {@link RestEsqlAsyncQueryAction} APIs. These are exposed over the @@ -1788,7 +1786,6 @@ public enum Cap { */ FN_MV_INTERSECTION, - /** * Marks the move to the hash(doc) % shard_count routing function. Added in #137062. */ From 5a3966c6f08914ab0ca04a9b6b34e5287d9571f4 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 17 Dec 2025 15:21:53 -0700 Subject: [PATCH 18/24] Fix --- .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 35575aa5ac9f4..f105362744726 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -957,6 +957,11 @@ public enum Cap { */ AGGREGATE_METRIC_DOUBLE_V0, + /** + * Support running all aggregations on aggregate_metric_double using the default metric + */ + AGGREGATE_METRIC_DOUBLE_DEFAULT_METRIC, + /** * Support change point detection "CHANGE_POINT". */ From efcce49abb1daf56b2583df2d902fce31abc7a67 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 17 Dec 2025 16:57:14 -0700 Subject: [PATCH 19/24] Fix --- .../qa/testFixtures/src/main/resources/subquery.csv-spec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec index e9f4549b8ac42..d1ae7a0a1991e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec @@ -714,9 +714,9 @@ FROM sample_data, (FROM books METADATA _score, _index ignoreOrder:true _index:keyword | book_no:keyword | title:text | author:text | _score:double | @timestamp:datetime -books | 4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 | null -books | 5327 | War and Peace | Leo Tolstoy | 0.08 | null -sample_data | null | null | null | null | 2023-10-23T13:33:34.937Z +books | 5327 | War and Peace | Leo Tolstoy | 0.08 | null +books | 9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 | null +sample_data | null | null | null | null | 2023-10-23T13:33:34.937Z ; subqueryInFromWithRerankInMainQuery From c3d0b5094a990a492fb790cb299568cd2193c1d7 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 17 Dec 2025 21:02:02 -0700 Subject: [PATCH 20/24] Change --- .../routing/PartitionedRoutingIT.java | 1 - .../cluster/routing/IndexRoutingTests.java | 39 +++++++++---------- .../src/main/resources/subquery.csv-spec | 6 +-- .../test/rrf/300_rrf_retriever.yml | 1 - 4 files changed, 22 insertions(+), 25 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 18c332515b253..b05d231cb7fbb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -28,7 +28,6 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; -// TODO: Double check logic public class PartitionedRoutingIT extends ESIntegTestCase { public void testVariousPartitionSizes() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index c8bb4f2c58c27..17b59a6e71bbe 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -206,22 +206,21 @@ public void testPartitionedIndex() { } } - @AwaitsFix(bugUrl = "I believe it is valid that these change but need to check.") public void testPartitionedIndexShrunk() { Map> routingIdToShard = new HashMap<>(); Map routingA = new HashMap<>(); - routingA.put("a_0", 1); - routingA.put("a_1", 2); - routingA.put("a_2", 2); - routingA.put("a_3", 2); - routingA.put("a_4", 1); - routingA.put("a_5", 2); + routingA.put("a_0", 2); + routingA.put("a_1", 0); + routingA.put("a_2", 0); + routingA.put("a_3", 0); + routingA.put("a_4", 3); + routingA.put("a_5", 0); routingIdToShard.put("a", routingA); Map routingB = new HashMap<>(); routingB.put("b_0", 0); - routingB.put("b_1", 0); + routingB.put("b_1", 1); routingB.put("b_2", 0); routingB.put("b_3", 0); routingB.put("b_4", 3); @@ -229,21 +228,21 @@ public void testPartitionedIndexShrunk() { routingIdToShard.put("b", routingB); Map routingC = new HashMap<>(); - routingC.put("c_0", 1); - routingC.put("c_1", 1); - routingC.put("c_2", 0); - routingC.put("c_3", 0); - routingC.put("c_4", 0); - routingC.put("c_5", 1); + routingC.put("c_0", 3); + routingC.put("c_1", 3); + routingC.put("c_2", 1); + routingC.put("c_3", 1); + routingC.put("c_4", 1); + routingC.put("c_5", 3); routingIdToShard.put("c", routingC); Map routingD = new HashMap<>(); - routingD.put("d_0", 2); - routingD.put("d_1", 2); - routingD.put("d_2", 3); - routingD.put("d_3", 3); - routingD.put("d_4", 3); - routingD.put("d_5", 3); + routingD.put("d_0", 1); + routingD.put("d_1", 1); + routingD.put("d_2", 2); + routingD.put("d_3", 2); + routingD.put("d_4", 2); + routingD.put("d_5", 2); routingIdToShard.put("d", routingD); IndexRouting indexRouting = IndexRouting.fromIndexMetadata( diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec index d1ae7a0a1991e..e6b46a5d2fe9b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec @@ -669,9 +669,9 @@ FROM sample_data, (FROM books metadata _score ignoreOrder:true title:text | completion:keyword | @timestamp:datetime | client_ip:ip -War and Peace | WAR AND PEACE | null | null -War and Peace (Signet Classics) | WAR AND PEACE (SIGNET CLASSICS) | null | null -null | null | 2023-10-23T13:33:34.937Z | 172.21.0.5 +War and Peace | WAR AND PEACE | null | null +War and Peace: A Novel (6 Volumes) | WAR AND PEACE: A NOVEL (6 VOLUMES) | null | null +null | null | 2023-10-23T13:33:34.937Z | 172.21.0.5 ; subqueryInFromWithCompletionInMainQuery diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml index 5cc0a9b5608ef..4da6abfc64331 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml @@ -254,7 +254,6 @@ setup: - match: { hits.total.value : 3 } - length: { hits.hits: 1 } -# TODO: Maybe changed ordering of hits? - match: { hits.hits.0._id: "1" } - match: { hits.hits.0.fields.text.0: "term term" } - match: { hits.hits.0.fields.keyword.0: "other" } From 16d1186c0c5b51fd98c7290fa94068625bdd79fc Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 6 Jan 2026 20:33:00 -0700 Subject: [PATCH 21/24] Change --- .../cluster/metadata/IndexMetadataTests.java | 39 ++++++++++++------- .../cluster/routing/IndexRoutingTests.java | 22 +++++++++++ 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index 7793f8a12d784..c1b4698cc508f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -307,9 +307,14 @@ public void testGetRoutingFactor() { } public void testSelectShrinkShards() { + int version = randomBoolean() + ? randomIntBetween(IndexVersions.MOD_ROUTING_FUNCTION.id(), IndexVersion.current().id()) + : randomIntBetween(IndexVersions.FIRST_DETACHED_INDEX_VERSION.id(), IndexVersions.MOD_ROUTING_FUNCTION.id() - 1); + boolean postModRoutingFunction = version >= IndexVersions.MOD_ROUTING_FUNCTION.id(); + int numberOfReplicas = randomIntBetween(0, 10); IndexMetadata metadata = IndexMetadata.builder("foo") - .settings(indexSettings(32, numberOfReplicas).put("index.version.created", IndexVersions.MOD_ROUTING_FUNCTION.id())) + .settings(indexSettings(32, numberOfReplicas).put("index.version.created", version)) .creationDate(randomLong()) .build(); Set shardIds = IndexMetadata.selectShrinkShards(0, metadata, 8); @@ -317,29 +322,29 @@ public void testSelectShrinkShards() { shardIds, Sets.newHashSet( new ShardId(metadata.getIndex(), 0), - new ShardId(metadata.getIndex(), 8), - new ShardId(metadata.getIndex(), 16), - new ShardId(metadata.getIndex(), 24) + new ShardId(metadata.getIndex(), postModRoutingFunction ? 8 : 1), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 16 : 2), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 24 : 3) ) ); shardIds = IndexMetadata.selectShrinkShards(1, metadata, 8); assertEquals( shardIds, Sets.newHashSet( - new ShardId(metadata.getIndex(), 1), - new ShardId(metadata.getIndex(), 9), - new ShardId(metadata.getIndex(), 17), - new ShardId(metadata.getIndex(), 25) + new ShardId(metadata.getIndex(), postModRoutingFunction ? 1 : 4), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 9 : 5), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 17 : 6), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 25 : 7) ) ); shardIds = IndexMetadata.selectShrinkShards(7, metadata, 8); assertEquals( shardIds, Sets.newHashSet( - new ShardId(metadata.getIndex(), 7), - new ShardId(metadata.getIndex(), 15), - new ShardId(metadata.getIndex(), 23), - new ShardId(metadata.getIndex(), 31) + new ShardId(metadata.getIndex(), postModRoutingFunction ? 7 : 28), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 15 : 29), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 23 : 30), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 31 : 31) ) ); @@ -379,17 +384,21 @@ public void testSelectResizeShards() { } public void testSelectSplitShard() { + int version = randomBoolean() + ? IndexVersions.MOD_ROUTING_FUNCTION.id() + : randomIntBetween(IndexVersions.FIRST_DETACHED_INDEX_VERSION.id(), IndexVersions.MOD_ROUTING_FUNCTION.id() - 1); + boolean postModRoutingFunction = version >= IndexVersions.MOD_ROUTING_FUNCTION.id(); IndexMetadata metadata = IndexMetadata.builder("foo") - .settings(indexSettings(2, 0).put("index.version.created", IndexVersions.MOD_ROUTING_FUNCTION.id())) + .settings(indexSettings(2, 0).put("index.version.created", version)) .creationDate(randomLong()) .setRoutingNumShards(4) .build(); ShardId shardId = IndexMetadata.selectSplitShard(0, metadata, 4); assertEquals(0, shardId.getId()); shardId = IndexMetadata.selectSplitShard(1, metadata, 4); - assertEquals(1, shardId.getId()); + assertEquals(postModRoutingFunction ? 1 : 0, shardId.getId()); shardId = IndexMetadata.selectSplitShard(2, metadata, 4); - assertEquals(0, shardId.getId()); + assertEquals(postModRoutingFunction ? 0 : 1, shardId.getId()); shardId = IndexMetadata.selectSplitShard(3, metadata, 4); assertEquals(1, shardId.getId()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 17b59a6e71bbe..46c93647e3e1f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -51,6 +51,7 @@ import static org.hamcrest.Matchers.stringContainsInOrder; public class IndexRoutingTests extends ESTestCase { + public void testSimpleRoutingRejectsEmptyId() { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() @@ -71,6 +72,27 @@ public void testSimpleRoutingAcceptsId() { assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP)); } + public void testSimpleRoutingPreAndPostModRoutingChange() { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() + ); + String id = "1"; + IndexRequest req = new IndexRequest().id(id); + indexRouting.preProcess(req); + assertThat(indexRouting.indexShard(req), equalTo(1)); + + IndexRouting indexRoutingOld = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test") + .settings(settings(IndexVersions.STORE_FALLBACK_TEXT_FIELDS_IN_BINARY_DOC_VALUES)) + .numberOfShards(2) + .numberOfReplicas(1) + .setRoutingNumShards(1024) + .build() + ); + indexRoutingOld.preProcess(req); + assertThat(indexRoutingOld.indexShard(req), equalTo(0)); + } + public void testSimpleRoutingAssignedRandomId() { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() From e531f2e2ddc0e4f3f22bc9bc5ba58e82eaa19ccc Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 12 Jan 2026 12:04:49 -0700 Subject: [PATCH 22/24] Change --- .../rest-api-spec/test/patterntext/30_template_id.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml index e51a859323c71..4c07d1c555a51 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml @@ -142,8 +142,8 @@ Range query: lt: "l" # one doc has a null value for template_id and two start with m > l - match: { hits.total.value: 2 } - - match: { hits.hits.0._id: "4" } - - match: { hits.hits.1._id: "3" } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.1._id: "4" } --- Term aggregation: From 4b23459c54f26f118003ba9a5107dddce63af085 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 12 Jan 2026 12:49:45 -0700 Subject: [PATCH 23/24] Fix --- .../rest-api-spec/test/patterntext/30_template_id.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml index 4c07d1c555a51..bdfadf010908f 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml @@ -141,9 +141,12 @@ Range query: foo.template_id: lt: "l" # one doc has a null value for template_id and two start with m > l + # The order of these hits varies between Serverless (multi shard) and core ES (single shard) - match: { hits.total.value: 2 } - - match: { hits.hits.0._id: "3" } - - match: { hits.hits.1._id: "4" } + - gte: { hits.hits.0._id: "3" } + - lte: { hits.hits.0._id: "4" } + - gte: { hits.hits.1._id: "3" } + - lte: { hits.hits.1._id: "4" } --- Term aggregation: From 3e33537a1911c89fc72168f7155c70161b85e93e Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 12 Jan 2026 14:08:20 -0700 Subject: [PATCH 24/24] Change --- .../rest-api-spec/test/patterntext/30_template_id.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml index bdfadf010908f..5e489e060f522 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml @@ -8,6 +8,7 @@ setup: index: test body: settings: + number_of_shards: 1 index.mapping.use_doc_values_skipper: true index.mode: logsdb sort.field: [ "foo.template_id" ] @@ -141,12 +142,9 @@ Range query: foo.template_id: lt: "l" # one doc has a null value for template_id and two start with m > l - # The order of these hits varies between Serverless (multi shard) and core ES (single shard) - match: { hits.total.value: 2 } - - gte: { hits.hits.0._id: "3" } - - lte: { hits.hits.0._id: "4" } - - gte: { hits.hits.1._id: "3" } - - lte: { hits.hits.1._id: "4" } + - match: { hits.hits.0._id: "4" } + - match: { hits.hits.1._id: "3" } --- Term aggregation: