diff --git a/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md b/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md index d72ea42b3106f..db97c8e77d4bf 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md +++ b/docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md @@ -11,9 +11,9 @@ FROM books METADATA _score | author:text | score:double | _fork:keyword | total:long | | --- | --- | --- | --- | -| William Faulkner | 2.39 | fork1 | null | -| William Faulkner | 2.39 | fork1 | null | -| Colleen Faulkner | 1.59 | fork1 | null | -| Danny Faulkner | 1.59 | fork1 | null | -| Keith Faulkner | 1.59 | fork1 | null | +| Colleen Faulkner | 2.18 | fork1 | null | +| William Faulkner | 2.18 | fork1 | null | +| Danny Faulkner | 2.02 | fork1 | null | +| Paul Faulkner | 2.02 | fork1 | null | +| William Faulkner | 2.02 | fork1 | null | | null | null | fork2 | 18 | diff --git a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md index 45e7c34030162..d8dd01e0f6b9b 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md +++ b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md @@ -13,5 +13,5 @@ FROM books METADATA _score | title:text | _score:double | | --- | --- | | Poems from the Hobbit | 0.0015673980815336108 | -| A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 0.007936508394777775 | -| Return of the King Being the Third Part of The Lord of the Rings | 9.960159659385681E-4 | +| The Lord of the Rings - Boxed Set | 0.0011135857785120606 | +| Letters of J R R Tolkien | 0.0024999999441206455 | diff --git a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md index d9ed44560d698..20d1f2dd3cadf 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md +++ b/docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md @@ -13,6 +13,6 @@ FROM books METADATA _score | title:text | _score:double | rerank_score:double | | --- | --- | --- | -| Return of the Shadow | 2.8181066513061523 | 5.740527994930744E-4 | -| Return of the King Being the Third Part of The Lord of the Rings | 3.6248698234558105 | 9.000900317914784E-4 | -| The Lays of Beleriand | 1.3002015352249146 | 9.36329597607255E-4 | +| Return of the Shadow | 3.4218082427978516 | 5.740527994930744E-4 | +| Return of the King Being the Third Part of The Lord of the Rings | 2.8398752212524414 | 9.000900317914784E-4 | +| The Lays of Beleriand | 1.5629040002822876 | 9.36329597607255E-4 | diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 2cd319d148321..35235c310d166 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -83,7 +83,7 @@ public void testBulkWithWriteIndexAndRouting() { indexRequestWithAlias.source(Collections.singletonMap("foo", "baz")); BulkResponse bulkResponse = client().prepareBulk().add(indexRequestWithAlias).get(); assertThat(bulkResponse.getItems()[0].getResponse().getIndex(), equalTo("index3")); - assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(0)); + assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(1)); assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); assertThat(bulkResponse.getItems()[0].getResponse().status(), equalTo(RestStatus.CREATED)); assertThat(client().prepareGet("index3", "id").setRouting("1").get().getSource().get("foo"), equalTo("baz")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index adda57d8d290d..b05d231cb7fbb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.mockito.internal.util.collections.Sets; import java.util.HashMap; import java.util.HashSet; @@ -27,6 +26,7 @@ import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; public class PartitionedRoutingIT extends ESIntegTestCase { @@ -50,7 +50,7 @@ public void testVariousPartitionSizes() throws Exception { verifyGets(index, routingToDocumentIds); verifyBroadSearches(index, routingToDocumentIds, shards); - verifyRoutedSearches(index, routingToDocumentIds, Sets.newSet(partitionSize)); + verifyRoutedSearches(index, routingToDocumentIds, partitionSize); } } } @@ -77,20 +77,10 @@ public void testShrinking() throws Exception { Map> routingToDocumentIds = generateRoutedDocumentIds(index); while (true) { - int factor = originalShards / currentShards; - verifyGets(index, routingToDocumentIds); verifyBroadSearches(index, routingToDocumentIds, currentShards); - // we need the floor and ceiling of the routing_partition_size / factor since the partition size of the shrunken - // index will be one of those, depending on the routing value - verifyRoutedSearches( - index, - routingToDocumentIds, - Math.floorDiv(partitionSize, factor) == 0 - ? Sets.newSet(1, 2) - : Sets.newSet(Math.floorDiv(partitionSize, factor), -Math.floorDiv(-partitionSize, factor)) - ); + verifyRoutedSearches(index, routingToDocumentIds, Math.min(partitionSize, currentShards)); updateIndexSettings( Settings.builder() @@ -149,7 +139,7 @@ public void testUnableToUpdateIndexRoutingPartitionSizes() throws Exception { assertThat(exc.getMessage(), containsString("final indexMetadata setting [index.routing_partition_size]")); } - private void verifyRoutedSearches(String index, Map> routingToDocumentIds, Set expectedShards) { + private void verifyRoutedSearches(String index, Map> routingToDocumentIds, int expectedShards) { for (Map.Entry> routingEntry : routingToDocumentIds.entrySet()) { String routing = routingEntry.getKey(); int expectedDocuments = routingEntry.getValue().size(); @@ -169,9 +159,10 @@ private void verifyRoutedSearches(String index, Map> routing + "]" ); - assertTrue( + assertThat( response.getTotalShards() + " was not in " + expectedShards + " for " + index, - expectedShards.contains(response.getTotalShards()) + expectedShards, + equalTo(response.getTotalShards()) ); assertEquals(expectedDocuments, response.getHits().getTotalHits().value()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 8c3035096f434..b4abc4d99f60c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -3128,7 +3128,11 @@ public static ShardId selectSplitShard(int shardId, IndexMetadata sourceIndexMet } final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); - return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); + if (IndexRouting.shouldUseShardCountModRouting(sourceIndexMetadata.getCreationVersion())) { + return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards())); + } else { + return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor); + } } /** @@ -3218,8 +3222,15 @@ public static Set selectShrinkShards(int shardId, IndexMetadata sourceI } int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = Sets.newHashSetWithExpectedSize(routingFactor); - for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) { - shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + if (IndexRouting.shouldUseShardCountModRouting(sourceIndexMetadata.getCreationVersion())) { + for (int i = shardId; i < sourceIndexMetadata.getNumberOfShards(); i += numTargetShards) { + assert Math.floorMod(i, numTargetShards) == shardId; + shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + } + } else { + for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) { + shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); + } } return shards; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 7966f105a977f..6beb3db89161a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -72,15 +72,19 @@ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { } protected final String indexName; + private final int numberOfShards; private final int routingNumShards; private final int routingFactor; + protected final IndexVersion creationVersion; @Nullable private final IndexReshardingMetadata indexReshardingMetadata; private IndexRouting(IndexMetadata metadata) { this.indexName = metadata.getIndex().getName(); + this.numberOfShards = metadata.getNumberOfShards(); this.routingNumShards = metadata.getRoutingNumShards(); this.routingFactor = metadata.getRoutingFactor(); + this.creationVersion = metadata.getCreationVersion(); this.indexReshardingMetadata = metadata.getReshardingMetadata(); } @@ -139,11 +143,27 @@ public void postProcess(IndexRequest indexRequest) {} */ public abstract void collectSearchShards(String routing, IntConsumer consumer); + public static boolean shouldUseShardCountModRouting(final IndexVersion creationVersion) { + return creationVersion.onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION); + } + /** * Convert a hash generated from an {@code (id, routing}) pair into a * shard id. */ protected final int hashToShardId(int hash) { + if (shouldUseShardCountModRouting(creationVersion)) { + return Math.floorMod(hash, numberOfShards); + } else { + return hashToShardIdOld(hash); + } + } + + /** + * Convert a hash generated from an {@code (id, routing}) pair into a + * shard id using the old routingNumShards mechanism. + */ + protected final int hashToShardIdOld(int hash) { return Math.floorMod(hash, routingNumShards) / routingFactor; } @@ -186,12 +206,10 @@ private int rerouteFromSplitTargetShard(int shardId, IndexReshardingState.Split. private abstract static class IdAndRoutingOnly extends IndexRouting { private final boolean routingRequired; - private final IndexVersion creationVersion; private final IndexMode indexMode; IdAndRoutingOnly(IndexMetadata metadata) { super(metadata); - this.creationVersion = metadata.getCreationVersion(); MappingMetadata mapping = metadata.mapping(); this.routingRequired = mapping == null ? false : mapping.routingRequired(); this.indexMode = metadata.getIndexMode(); diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 104fab8a03e02..b3e5624c3bf95 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -214,6 +214,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion KEYWORD_FIELDS_KEEP_DUPLICATES_IN_BINARY_DOC_VALUES = def(9_061_0_00, Version.LUCENE_10_3_2); public static final IndexVersion STORE_FALLBACK_TEXT_FIELDS_IN_BINARY_DOC_VALUES = def(9_062_0_00, Version.LUCENE_10_3_2); public static final IndexVersion STORE_FALLBACK_MOT_FIELDS_IN_BINARY_DOC_VALUES = def(9_063_0_00, Version.LUCENE_10_3_2); + public static final IndexVersion MOD_ROUTING_FUNCTION = def(9_064_0_00, Version.LUCENE_10_3_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index dac353919d0f9..fc23934e624ca 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -119,7 +119,7 @@ public void testErrorCondition() { "target", new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator( new StoreStats(between(1, 100), between(0, 100), between(1, 100)), - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + (i) -> i == 1 || i == 5 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null ) ) ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ") diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index 134797809b6ed..c1b4698cc508f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -307,9 +307,14 @@ public void testGetRoutingFactor() { } public void testSelectShrinkShards() { + int version = randomBoolean() + ? randomIntBetween(IndexVersions.MOD_ROUTING_FUNCTION.id(), IndexVersion.current().id()) + : randomIntBetween(IndexVersions.FIRST_DETACHED_INDEX_VERSION.id(), IndexVersions.MOD_ROUTING_FUNCTION.id() - 1); + boolean postModRoutingFunction = version >= IndexVersions.MOD_ROUTING_FUNCTION.id(); + int numberOfReplicas = randomIntBetween(0, 10); IndexMetadata metadata = IndexMetadata.builder("foo") - .settings(indexSettings(32, numberOfReplicas).put("index.version.created", 1)) + .settings(indexSettings(32, numberOfReplicas).put("index.version.created", version)) .creationDate(randomLong()) .build(); Set shardIds = IndexMetadata.selectShrinkShards(0, metadata, 8); @@ -317,29 +322,29 @@ public void testSelectShrinkShards() { shardIds, Sets.newHashSet( new ShardId(metadata.getIndex(), 0), - new ShardId(metadata.getIndex(), 1), - new ShardId(metadata.getIndex(), 2), - new ShardId(metadata.getIndex(), 3) + new ShardId(metadata.getIndex(), postModRoutingFunction ? 8 : 1), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 16 : 2), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 24 : 3) ) ); shardIds = IndexMetadata.selectShrinkShards(1, metadata, 8); assertEquals( shardIds, Sets.newHashSet( - new ShardId(metadata.getIndex(), 4), - new ShardId(metadata.getIndex(), 5), - new ShardId(metadata.getIndex(), 6), - new ShardId(metadata.getIndex(), 7) + new ShardId(metadata.getIndex(), postModRoutingFunction ? 1 : 4), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 9 : 5), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 17 : 6), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 25 : 7) ) ); shardIds = IndexMetadata.selectShrinkShards(7, metadata, 8); assertEquals( shardIds, Sets.newHashSet( - new ShardId(metadata.getIndex(), 28), - new ShardId(metadata.getIndex(), 29), - new ShardId(metadata.getIndex(), 30), - new ShardId(metadata.getIndex(), 31) + new ShardId(metadata.getIndex(), postModRoutingFunction ? 7 : 28), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 15 : 29), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 23 : 30), + new ShardId(metadata.getIndex(), postModRoutingFunction ? 31 : 31) ) ); @@ -379,17 +384,21 @@ public void testSelectResizeShards() { } public void testSelectSplitShard() { + int version = randomBoolean() + ? IndexVersions.MOD_ROUTING_FUNCTION.id() + : randomIntBetween(IndexVersions.FIRST_DETACHED_INDEX_VERSION.id(), IndexVersions.MOD_ROUTING_FUNCTION.id() - 1); + boolean postModRoutingFunction = version >= IndexVersions.MOD_ROUTING_FUNCTION.id(); IndexMetadata metadata = IndexMetadata.builder("foo") - .settings(indexSettings(2, 0).put("index.version.created", 1)) + .settings(indexSettings(2, 0).put("index.version.created", version)) .creationDate(randomLong()) .setRoutingNumShards(4) .build(); ShardId shardId = IndexMetadata.selectSplitShard(0, metadata, 4); assertEquals(0, shardId.getId()); shardId = IndexMetadata.selectSplitShard(1, metadata, 4); - assertEquals(0, shardId.getId()); + assertEquals(postModRoutingFunction ? 1 : 0, shardId.getId()); shardId = IndexMetadata.selectSplitShard(2, metadata, 4); - assertEquals(1, shardId.getId()); + assertEquals(postModRoutingFunction ? 0 : 1, shardId.getId()); shardId = IndexMetadata.selectSplitShard(3, metadata, 4); assertEquals(1, shardId.getId()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 122a6190c06fa..46c93647e3e1f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -51,6 +51,7 @@ import static org.hamcrest.Matchers.stringContainsInOrder; public class IndexRoutingTests extends ESTestCase { + public void testSimpleRoutingRejectsEmptyId() { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() @@ -71,6 +72,27 @@ public void testSimpleRoutingAcceptsId() { assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP)); } + public void testSimpleRoutingPreAndPostModRoutingChange() { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() + ); + String id = "1"; + IndexRequest req = new IndexRequest().id(id); + indexRouting.preProcess(req); + assertThat(indexRouting.indexShard(req), equalTo(1)); + + IndexRouting indexRoutingOld = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test") + .settings(settings(IndexVersions.STORE_FALLBACK_TEXT_FIELDS_IN_BINARY_DOC_VALUES)) + .numberOfShards(2) + .numberOfReplicas(1) + .setRoutingNumShards(1024) + .build() + ); + indexRoutingOld.preProcess(req); + assertThat(indexRoutingOld.indexShard(req), equalTo(0)); + } + public void testSimpleRoutingAssignedRandomId() { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() @@ -210,17 +232,17 @@ public void testPartitionedIndexShrunk() { Map> routingIdToShard = new HashMap<>(); Map routingA = new HashMap<>(); - routingA.put("a_0", 1); - routingA.put("a_1", 2); - routingA.put("a_2", 2); - routingA.put("a_3", 2); - routingA.put("a_4", 1); - routingA.put("a_5", 2); + routingA.put("a_0", 2); + routingA.put("a_1", 0); + routingA.put("a_2", 0); + routingA.put("a_3", 0); + routingA.put("a_4", 3); + routingA.put("a_5", 0); routingIdToShard.put("a", routingA); Map routingB = new HashMap<>(); routingB.put("b_0", 0); - routingB.put("b_1", 0); + routingB.put("b_1", 1); routingB.put("b_2", 0); routingB.put("b_3", 0); routingB.put("b_4", 3); @@ -228,21 +250,21 @@ public void testPartitionedIndexShrunk() { routingIdToShard.put("b", routingB); Map routingC = new HashMap<>(); - routingC.put("c_0", 1); - routingC.put("c_1", 1); - routingC.put("c_2", 0); - routingC.put("c_3", 0); - routingC.put("c_4", 0); - routingC.put("c_5", 1); + routingC.put("c_0", 3); + routingC.put("c_1", 3); + routingC.put("c_2", 1); + routingC.put("c_3", 1); + routingC.put("c_4", 1); + routingC.put("c_5", 3); routingIdToShard.put("c", routingC); Map routingD = new HashMap<>(); - routingD.put("d_0", 2); - routingD.put("d_1", 2); - routingD.put("d_2", 3); - routingD.put("d_3", 3); - routingD.put("d_4", 3); - routingD.put("d_5", 3); + routingD.put("d_0", 1); + routingD.put("d_1", 1); + routingD.put("d_2", 2); + routingD.put("d_3", 2); + routingD.put("d_4", 2); + routingD.put("d_5", 2); routingIdToShard.put("d", routingD); IndexRouting indexRouting = IndexRouting.fromIndexMetadata( diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 37510f0774312..4e8bcaeb857fb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -725,7 +725,7 @@ public void testSizeShrinkIndex() { new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"), ShardRouting.Role.DEFAULT ); - assertEquals(110L, getExpectedShardSize(target2, 0L, allocation)); + assertEquals(510L, getExpectedShardSize(target2, 0L, allocation)); target2 = ShardRouting.newUnassigned( new ShardId(new Index("target2", "9101112"), 1), @@ -734,7 +734,7 @@ public void testSizeShrinkIndex() { new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"), ShardRouting.Role.DEFAULT ); - assertEquals(1000L, getExpectedShardSize(target2, 0L, allocation)); + assertEquals(600L, getExpectedShardSize(target2, 0L, allocation)); // check that the DiskThresholdDecider still works even if the source index has been deleted ClusterState clusterStateWithMissingSourceIndex = ClusterState.builder(clusterState) diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java index 6c48e2fd20873..a5d7d71b3195d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java @@ -72,6 +72,9 @@ private void setUpIndex() throws Exception { prepareIndex(indexName).setId("1").setSource("body", "doc1", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); prepareIndex(indexName).setId("2").setSource("body", "doc2", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); prepareIndex(indexName).setId("3").setSource("body", "doc3", "@timestamp", "2025-01-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("4").setSource("body", "doc4", "@timestamp", "2025-02-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("5").setSource("body", "doc5", "@timestamp", "2025-03-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("6").setSource("body", "doc6", "@timestamp", "2025-04-01").setRefreshPolicy(IMMEDIATE).get(); createIndex( secondIndexName, @@ -82,9 +85,12 @@ private void setUpIndex() throws Exception { ); ensureGreen(secondIndexName); - prepareIndex(secondIndexName).setId("4").setSource("body", "doc1", "@timestamp", "2025-11-01").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(secondIndexName).setId("5").setSource("body", "doc2", "@timestamp", "2025-12-01").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(secondIndexName).setId("6").setSource("body", "doc3", "@timestamp", "2026-01-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("7").setSource("body", "doc1", "@timestamp", "2025-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("8").setSource("body", "doc2", "@timestamp", "2025-12-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("9").setSource("body", "doc3", "@timestamp", "2026-01-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("10").setSource("body", "doc4", "@timestamp", "2026-02-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("11").setSource("body", "doc5", "@timestamp", "2026-03-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(secondIndexName).setId("12").setSource("body", "doc6", "@timestamp", "2026-04-01").setRefreshPolicy(IMMEDIATE).get(); } @After @@ -151,7 +157,7 @@ public void testPointInTimeWithPreFiltering() { .setSize(1) .setPreFilterShardSize(1) .setQuery(simpleQueryStringQuery("doc3")), - "6" + "9" ); assertMeasurements(List.of(OPEN_PIT_SEARCH_PHASE_METRIC, QUERY_SEARCH_PHASE_METRIC, FETCH_SEARCH_PHASE_METRIC), 1); assertMeasurements( diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec index 0b0d874823511..67b08371f0b6c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/completion.csv-spec @@ -29,6 +29,7 @@ prompt:keyword | completion_output:keywor completion after a search required_capability: completion required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -38,14 +39,15 @@ FROM books METADATA _score | KEEP title, completion ; -title:text | completion:keyword -War and Peace | WAR AND PEACE -War and Peace (Signet Classics) | WAR AND PEACE (SIGNET CLASSICS) +title:text | completion:keyword +War and Peace | WAR AND PEACE +War and Peace: A Novel (6 Volumes) | WAR AND PEACE: A NOVEL (6 VOLUMES) ; - + completion using a function as a prompt required_capability: completion required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -55,9 +57,9 @@ FROM books METADATA _score | KEEP title, completion ; -title:text | completion:keyword -War and Peace | THIS IS A PROMPT: WAR AND PEACE -War and Peace (Signet Classics) | THIS IS A PROMPT: WAR AND PEACE (SIGNET CLASSICS) +title:text | completion:keyword +War and Peace | THIS IS A PROMPT: WAR AND PEACE +War and Peace: A Novel (6 Volumes) | THIS IS A PROMPT: WAR AND PEACE: A NOVEL (6 VOLUMES) ; completion followed by stats diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec index 4fa289f79f9da..128e3eb304c0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec @@ -23,6 +23,7 @@ emp_no:integer | _fork:keyword simpleForkWithStats required_capability: fork_v9 +required_capability: routing_function_update // tag::simpleForkWithStats[] FROM books METADATA _score @@ -36,11 +37,11 @@ FROM books METADATA _score // tag::simpleForkWithStats-result[] author:text | score:double | _fork:keyword | total:long -William Faulkner | 2.39 | fork1 | null -William Faulkner | 2.39 | fork1 | null -Colleen Faulkner | 1.59 | fork1 | null -Danny Faulkner | 1.59 | fork1 | null -Keith Faulkner | 1.59 | fork1 | null +Colleen Faulkner | 2.18 | fork1 | null +William Faulkner | 2.18 | fork1 | null +Danny Faulkner | 2.02 | fork1 | null +Paul Faulkner | 2.02 | fork1 | null +William Faulkner | 2.02 | fork1 | null null | null | fork2 | 18 // end::simpleForkWithStats-result[] ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec index eee7bce1d1042..fd7b1880c07c1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec @@ -25,6 +25,7 @@ fuseWithMatchAndScore required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -40,14 +41,15 @@ _score:double | _fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithDisjunctionAndPostFilter required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" OR author:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -62,9 +64,9 @@ FROM books METADATA _id, _index, _score _score:double | _fork:keyword | _id:keyword 0.03252 | [fork1, fork2] | 60 -0.032 | [fork1, fork2] | 1 +0.032 | [fork1, fork2] | 29 0.01639 | fork2 | 4 -0.01587 | fork1 | 40 +0.01587 | fork1 | 1 ; fuseWithStats @@ -91,6 +93,7 @@ fuseWithMultipleForkBranches required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK (WHERE author:"Keith Faulkner" AND qstr("author:Rory or author:Beverlie") | SORT _score, _id DESC | LIMIT 3) @@ -106,12 +109,12 @@ FROM books METADATA _id, _index, _score ; _score:double | author:keyword | title:keyword | _fork:keyword -0.0328 | [Keith Faulkner, Rory Tyger] | Pop! Went Another Ba | [fork1, fork4] +0.0325 | [Keith Faulkner, Rory Tyger] | Pop! Went Another Ba | [fork1, fork4] 0.0164 | J.R.R. Tolkien | Letters of J R R Tol | fork3 -0.0164 | Ursula K. Le Guin | The wind's twelve qu | fork2 -0.0161 | [Beverlie Manson, Keith Faulkner] | Rainbow's End: A Mag | fork1 +0.0164 | [Beverlie Manson, Keith Faulkner] | Rainbow's End: A Mag | fork1 +0.0164 | Ursula K. Le Guin | Worlds of Exile and | fork2 0.0161 | Ursula K. Le Guin | The Word For World i | fork2 -0.0159 | Ursula K. Le Guin | The Dispossessed | fork2 +0.0159 | Ursula K. Le Guin | Steering the Craft | fork2 ; fuseWithSemanticSearch @@ -140,6 +143,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -155,8 +159,8 @@ _score:double | _fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndRankConstant @@ -164,6 +168,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -179,8 +184,8 @@ _score:double | _fork:keyword | _id:keyword 0.03922 | [fork1, fork2] | 4 0.01923 | fork1 | 56 0.01923 | fork2 | 60 -0.01887 | fork2 | 1 0.01887 | fork1 | 26 +0.01887 | fork2 | 29 ; fuseWithRrfAndWeights @@ -188,6 +193,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -202,7 +208,7 @@ FROM books METADATA _id, _index, _score _score:double | _fork:keyword | _id:keyword 0.01639 | [fork1, fork2] | 4 0.01129 | fork2 | 60 -0.01111 | fork2 | 1 +0.01111 | fork2 | 29 0.00484 | fork1 | 56 0.00476 | fork1 | 26 ; @@ -212,6 +218,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score, _index | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -226,7 +233,7 @@ FROM books METADATA _id, _score, _index _score:double | _fork:keyword | _id:keyword 0.01639 | [fork1, fork2] | 4 0.01129 | fork2 | 60 -0.01111 | fork2 | 1 +0.01111 | fork2 | 29 0.00484 | fork1 | 56 0.00476 | fork1 | 26 ; @@ -236,6 +243,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL my_score = _score | DROP _score) @@ -251,8 +259,8 @@ my_score:double | _fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndDiscriminatorColumn @@ -260,6 +268,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3) @@ -277,8 +286,8 @@ _score:double | new_fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndDiscriminatorColumnWithDots @@ -287,6 +296,7 @@ required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score required_capability: dots_in_fuse +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3) @@ -304,8 +314,8 @@ _score:double | new.fork:keyword | _id:keyword 0.03279 | [fork1, fork2] | 4 0.01613 | fork1 | 56 0.01613 | fork2 | 60 -0.01587 | fork2 | 1 0.01587 | fork1 | 26 +0.01587 | fork2 | 29 ; fuseWithRrfAndKeyColumns @@ -313,6 +323,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL new_id = CONCAT(_id, _id) | DROP _id) @@ -329,8 +340,8 @@ _score:double | _fork:keyword | new_id:keyword 0.03279 | [fork1, fork2] | 44 0.01613 | fork1 | 5656 0.01613 | fork2 | 6060 -0.01587 | fork2 | 11 0.01587 | fork1 | 2626 +0.01587 | fork2 | 2929 ; @@ -339,6 +350,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 @@ -358,8 +370,8 @@ new_score:double | new_fork:keyword | new_id:keyword 0.03279 | [A, B] | 44 0.01613 | A | 5656 0.01613 | B | 6060 -0.01587 | B | 11 0.01587 | A | 2626 +0.01587 | B | 2929 ; fuseWithSimpleLinear @@ -367,6 +379,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -379,11 +392,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -2.44669 | [fork1, fork2] | 4 -2.2543 | fork1 | 26 -2.10316 | fork1 | 56 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 +2.09421 | fork1 | 26 +2.08412 | [fork1, fork2] | 4 +1.96182 | fork1 | 56 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 ; fuseWithLinearAndL2Norm @@ -391,6 +404,7 @@ fuseWithLinearAndL2Norm required_capability: fork_v9 required_capability: fuse_v6 required_capability: fuse_l2_norm +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -403,11 +417,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -0.93394 | [fork1, fork2] | 4 -0.67019 | fork2 | 1 -0.62834 | fork1 | 26 -0.61018 | fork2 | 60 -0.58621 | fork1 | 56 +0.89318 | [fork1, fork2] | 4 +0.69946 | fork2 | 29 +0.65012 | fork1 | 26 +0.60902 | fork1 | 56 +0.56408 | fork2 | 60 ; fuseWithLinearAndMinMax @@ -416,6 +430,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -428,10 +443,10 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -1.0 | fork2 | 1 1.0 | fork1 | 26 -0.7577 | fork2 | 60 -0.63972 | fork1 | 56 +1.0 | fork2 | 29 +0.79006 | fork1 | 56 +0.4806 | fork2 | 60 0.0 | [fork1, fork2] | 4 ; @@ -440,6 +455,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -452,11 +468,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -0.97876 | [fork1, fork2] | 4 -0.67939 | fork2 | 1 -0.67629 | fork1 | 26 -0.63095 | fork1 | 56 -0.61856 | fork2 | 60 +0.87343 | [fork1, fork2] | 4 +0.69234 | fork2 | 29 +0.62826 | fork1 | 26 +0.58855 | fork1 | 56 +0.55834 | fork2 | 60 ; fuseWithLinearAndPartialWeights @@ -464,6 +480,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -476,11 +493,11 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -1.16233 | [fork1, fork2] | 4 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 -0.67629 | fork1 | 26 -0.63095 | fork1 | 56 +1.05958 | [fork1, fork2] | 4 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 +0.62826 | fork1 | 26 +0.58855 | fork1 | 56 ; fuseWithLinearWeightsAndMinMax @@ -488,6 +505,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -500,10 +518,10 @@ FROM books METADATA _id, _index, _score ; _score:double | _fork:keyword | _id:keyword -0.7 | fork2 | 1 -0.53039 | fork2 | 60 +0.7 | fork2 | 29 +0.33642 | fork2 | 60 0.3 | fork1 | 26 -0.19191 | fork1 | 56 +0.23702 | fork1 | 56 0.0 | [fork1, fork2] | 4 ; @@ -513,6 +531,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL my_score = _score | DROP _score) @@ -525,11 +544,11 @@ FROM books METADATA _id, _index, _score ; my_score:double | _fork:keyword | _id:keyword -2.44669 | [fork1, fork2] | 4 -2.2543 | fork1 | 26 -2.10316 | fork1 | 56 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 +2.09421 | fork1 | 26 +2.08412 | [fork1, fork2] | 4 +1.96182 | fork1 | 56 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 ; @@ -538,6 +557,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3) @@ -552,11 +572,11 @@ FROM books METADATA _id, _index, _score ; _score:double | new_fork:keyword | _id:keyword -2.44669 | [fork1, fork2] | 4 -2.2543 | fork1 | 26 -2.10316 | fork1 | 56 -0.97056 | fork2 | 1 -0.88365 | fork2 | 60 +2.09421 | fork1 | 26 +2.08412 | [fork1, fork2] | 4 +1.96182 | fork1 | 56 +0.98906 | fork2 | 29 +0.79763 | fork2 | 60 ; fuseWithLinearAndKeyColumns @@ -564,6 +584,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 | EVAL new_id = CONCAT(_id, _id) | DROP _id) @@ -577,11 +598,11 @@ FROM books METADATA _id, _score ; _score:double | _fork:keyword | new_id:keyword -2.44669 | [fork1, fork2] | 44 -2.2543 | fork1 | 2626 -2.10316 | fork1 | 5656 -0.97056 | fork2 | 11 -0.88365 | fork2 | 6060 +2.09421 | fork1 | 2626 +2.08412 | [fork1, fork2] | 44 +1.96182 | fork1 | 5656 +0.98906 | fork2 | 2929 +0.79763 | fork2 | 6060 ; fuseWithLinearAllOptionsScoreGroupAndKeyColumns @@ -589,6 +610,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: semantic_text_field_caps required_capability: metadata_score +required_capability: routing_function_update FROM books METADATA _id, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 @@ -605,10 +627,10 @@ FROM books METADATA _id, _score ; new_score:double | new_fork:keyword | new_id:keyword -1.0 | B | 11 1.0 | A | 2626 -0.7577 | B | 6060 -0.63972 | A | 5656 +1.0 | B | 2929 +0.79006 | A | 5656 +0.4806 | B | 6060 0.0 | [A, B] | 44 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec index 2e912c7337534..6b3aee57bcd26 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec @@ -6,6 +6,7 @@ reranker using a single field, overwrite existing _score column required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -17,14 +18,15 @@ FROM books METADATA _score book_no:keyword | title:text | author:text | _score:double 5327 | War and Peace | Leo Tolstoy | 0.08 -4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 +4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 ; reranker using a single field, create a mew column required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -35,15 +37,16 @@ FROM books METADATA _score ; book_no:keyword | title:text | author:text | rerank_score:double -5327 | War and Peace | Leo Tolstoy | 0.08 +5327 | War and Peace | Leo Tolstoy | 0.08 +9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 -9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 -2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 +2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 ; reranker using a single field, create a mew column, sort by rerank_score required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -56,9 +59,9 @@ FROM books METADATA _score book_no:keyword | title:text | author:text | rerank_score:double 2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02 -9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 -5327 | War and Peace | Leo Tolstoy | 0.08 +9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 +5327 | War and Peace | Leo Tolstoy | 0.08 ; reranker using a non text fields @@ -84,6 +87,7 @@ book_no:keyword | title:text | rating reranker using a sparse input field required_capability: rerank required_capability: match_operator_colon +required_capability: routing_function_update FROM books METADATA _score | WHERE MATCH(title, "lord of the rings", {"minimum_should_match": "100%"}) AND author:"tolkien" @@ -94,10 +98,10 @@ FROM books METADATA _score | KEEP book_no, title, author, collection, rerank_score, _score ; -book_no:keyword | title:text | author:text | collection:text | rerank_score:double | _score:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | J. R. R. Tolkien | The Lord of the Rings | 0.05 | 8.56 -2675 | The Lord of the Rings - Boxed Set | J.R.R. Tolkien | The Lord of the Rings | 0.05 | 8.35 -7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | [Alan Lee, J. R. R. Tolkien] | null | null | 5.5 +book_no:keyword | title:text | author:text | collection:text | rerank_score:double | _score:double +2675 | The Lord of the Rings - Boxed Set | J.R.R. Tolkien | The Lord of the Rings | 0.05 | 7.28 +2714 | Return of the King Being the Third Part of The Lord of the Rings | J. R. R. Tolkien | The Lord of the Rings | 0.05 | 7.25 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | [Agnes Perkins, Charles Adolph Huttar, John Ronald Reuel Tolkien, Walter Scheps] | null | null | 5.36 ; @@ -207,6 +211,7 @@ required_capability: fork_v9 required_capability: fuse_v6 required_capability: match_operator_colon required_capability: rerank +required_capability: routing_function_update FROM books METADATA _id, _index, _score | FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3 ) @@ -219,14 +224,15 @@ FROM books METADATA _id, _index, _score | KEEP book_no, title, author, _score ; -book_no:keyword | title:keyword | author:keyword | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 0.04 -2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 0.03 +book_no:keyword | title:keyword | author:keyword | _score:double +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 0.04 +7635 | Oliphaunt (Beastly Verse) | J. R. R. Tolkien | 0.04 ; simple required_capability: rerank +required_capability: routing_function_update // tag::simple-query[] FROM books METADATA _score @@ -240,15 +246,16 @@ FROM books METADATA _score ; // tag::simple-query-result[] -title:text | _score:double -Poems from the Hobbit | 0.0015673980815336108 -A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 0.007936508394777775 -Return of the King Being the Third Part of The Lord of the Rings | 9.960159659385681E-4 +title:text | _score:double +Poems from the Hobbit | 0.0015673980815336108 +The Lord of the Rings - Boxed Set | 0.0011135857785120606 +Letters of J R R Tolkien | 0.0024999999441206455 // end::simple-query-result[] ; two_queries required_capability: rerank +required_capability: routing_function_update // tag::two-queries[] FROM books METADATA _score @@ -264,9 +271,9 @@ FROM books METADATA _score // tag::two-queries-result[] title:text | _score:double | rerank_score:double -Return of the Shadow | 2.8181066513061523 | 5.740527994930744E-4 -Return of the King Being the Third Part of The Lord of the Rings | 3.6248698234558105 | 9.000900317914784E-4 -The Lays of Beleriand | 1.3002015352249146 | 9.36329597607255E-4 +Return of the Shadow | 3.4218082427978516 | 5.740527994930744E-4 +Return of the King Being the Third Part of The Lord of the Rings | 2.8398752212524414 | 9.000900317914784E-4 +The Lays of Beleriand | 1.5629040002822876 | 9.36329597607255E-4 // end::two-queries-result[] ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec index 00c5fde9e4ee9..681151e976357 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/score-function.csv-spec @@ -6,6 +6,7 @@ scoreSingle required_capability: metadata_score required_capability: score_function required_capability: match_function +required_capability: routing_function_update // tag::score-function[] FROM books METADATA _score @@ -18,15 +19,16 @@ FROM books METADATA _score ; // tag::score-single-result[] -book_no:keyword | title:text | _score:double | first_score:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 3.1309072971343994 | 1.9245924949645996 -7350 | Return of the Shadow | 4.8434343338012695 | 3.5432329177856445 +book_no:keyword | title:text | _score:double | first_score:double +2714 | Return of the King Being the Third Part of The Lord of the Rings | 3.167952537536621 | 2.0707292556762695 +7350 | Return of the Shadow | 5.181426048278809 | 3.6185221672058105 // end::score-single-result[] ; scoreSingleNoMetadata required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -36,15 +38,16 @@ FROM books | LIMIT 5 ; -book_no:keyword | title:text | first_score:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7350 | Return of the Shadow | 3.5432329177856445 +book_no:keyword | title:text | first_score:double +2714 | Return of the King Being the Third Part of The Lord of the Rings | 2.0707292556762695 +7350 | Return of the Shadow | 3.6185221672058105 ; scoreAfterEval required_capability: score_function required_capability: metadata_score required_capability: match_function +required_capability: routing_function_update FROM books METADATA _score | EVAL stars = to_long(ratings / 2.0) @@ -56,15 +59,16 @@ FROM books METADATA _score book_no:keyword | author:text | stars:long | s1:double 2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3 | 0.0 -2713 | William Faulkner | 2 | 1.9043500423431396 +2713 | William Faulkner | 2 | 2.9992568492889404 2847 | Colleen Faulkner | 3 | 0.0 -2883 | William Faulkner | 2 | 1.9043500423431396 +2883 | William Faulkner | 2 | 1.820595145225525 3293 | Danny Faulkner | 2 | 0.0 ; scoreMatchWithFilterConjunction required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -74,13 +78,14 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 7350 | Return of the Shadow | 0.0 ; scoreMatchWithDisjunction required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -90,13 +95,14 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7350 | Return of the Shadow | 3.5432329177856445 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +7350 | Return of the Shadow | 3.6185221672058105 ; scoreMatchWithDisjunctionAndFilter required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -106,13 +112,14 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7350 | Return of the Shadow | 3.5432329177856445 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +7350 | Return of the Shadow | 3.6185221672058105 ; scoreMatchDisjunctionNonPushable required_capability: score_function required_capability: match_function +required_capability: routing_function_update FROM books | WHERE match(title, "Return") AND match(author, "Tolkien") @@ -122,7 +129,7 @@ FROM books | LIMIT 5; book_no:keyword | title:text | s1:double -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 7350 | Return of the Shadow | 0.0 ; @@ -146,6 +153,7 @@ evalUsingScoreAndLimit2 required_capability: score_function required_capability: match_function required_capability: pushing_down_eval_with_score +required_capability: routing_function_update FROM books | WHERE author == "J R R Tolkien" @@ -154,13 +162,14 @@ FROM books ; title_match_ring:double | title_match_hobbit:double -0 | 3.45605206489563 +0 | 3.03169584274292 ; evalUsingScoreAndTopN required_capability: score_function required_capability: match_function required_capability: pushing_down_eval_with_score +required_capability: routing_function_update FROM books | SORT book_no desc @@ -175,7 +184,7 @@ book_no:keyword | title_match_towers:double 9607 | 0 9032 | 0 8956 | 0 -8875 | 3.5007452964782715 +8875 | 3.9344332218170166 8873 | 0 8678 | 0 8615 | 0 @@ -186,6 +195,7 @@ evalUsingScoreAndFilter required_capability: score_function required_capability: match_function required_capability: pushing_down_eval_with_score +required_capability: routing_function_update FROM books | sort book_no desc @@ -198,5 +208,5 @@ FROM books ; book_no:keyword | title_match_world:double -7912 | 2.7460334300994873 +7912 | 2.6122074127197266 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index 221e913e83497..dbb1ea49a3521 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -5,43 +5,46 @@ singleQstrBoostScoringSorted required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score +from books metadata _score | where qstr("author:Lord Rings^2") -| eval c_score = ceil(_score) -| keep book_no, title, c_score +| eval c_score = ceil(_score) +| keep book_no, title, c_score | sort c_score desc, book_no asc | LIMIT 2; -book_no:keyword | title:text | c_score:double -1463 | Realms of Tolkien: Images of Middle-earth | 6.0 -2675 | The Lord of the Rings - Boxed Set | 6.0 +book_no:keyword | title:text | c_score:double +8875 | The Two Towers | 7.0 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 6.0 ; singleMatchWithKeywordFieldScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update -from books metadata _score -| where author.keyword:"William Faulkner" -| keep book_no, author, _score +from books metadata _score +| where author.keyword:"William Faulkner" +| keep book_no, author, _score | sort book_no; -book_no:keyword | author:text | _score:double -2713 | William Faulkner | 1.7589385509490967 -2883 | William Faulkner | 1.7589385509490967 -4724 | William Faulkner | 1.7589385509490967 -4977 | William Faulkner | 2.6145541667938232 -5119 | William Faulkner | 2.513157367706299 -5404 | William Faulkner | 1.7589385509490967 -5578 | William Faulkner | 2.513157367706299 -8077 | William Faulkner | 1.7589385509490967 -9896 | William Faulkner | 2.6145541667938232 +book_no:keyword | author:text | _score:double +2713 | William Faulkner | 2.743302583694458 +2883 | William Faulkner | 1.6706234216690063 +4724 | William Faulkner | 1.6706234216690063 +4977 | William Faulkner | 1.6706234216690063 +5119 | William Faulkner | 1.6706234216690063 +5404 | William Faulkner | 1.6706234216690063 +5578 | William Faulkner | 3.1982219219207764 +8077 | William Faulkner | 1.6706234216690063 +9896 | William Faulkner | 1.6706234216690063 ; qstrWithFieldAndScoringSortedEval required_capability: qstr_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where qstr("title:rings") @@ -50,15 +53,16 @@ from books metadata _score | keep book_no, title, _score | limit 3; -book_no:keyword | title:text | _score:double -2675 | The Lord of the Rings - Boxed Set | 2.5619282722473145 -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.746896743774414 +book_no:keyword | title:text | _score:double +2675 | The Lord of the Rings - Boxed Set | 1.9618241786956787 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 1.4636166095733643 ; qstrWithFieldAndScoringSorted required_capability: qstr_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where qstr("title:rings") @@ -66,43 +70,46 @@ from books metadata _score | keep book_no, title, _score | limit 3; -book_no:keyword | title:text | _score:double -2675 | The Lord of the Rings - Boxed Set | 2.5619282722473145 -2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.9245924949645996 -7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.746896743774414 +book_no:keyword | title:text | _score:double +2675 | The Lord of the Rings - Boxed Set | 1.9618241786956787 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 1.7047617435455322 +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 1.4636166095733643 ; singleQstrScoringManipulated required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score -| where qstr("author:William Faulkner") -| eval add_score = ceil(_score) + 1 -| keep book_no, author, add_score -| sort book_no +from books metadata _score +| where qstr("author:William Faulkner") +| eval add_score = ceil(_score) + 1 +| keep book_no, author, add_score +| sort book_no | LIMIT 2; book_no:keyword | author:text | add_score:double 2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3.0 -2713 | William Faulkner | 6.0 +2713 | William Faulkner | 7.0 ; testMultiValuedFieldWithConjunctionWithScore required_capability: match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where match(author, "Keith Faulkner") and match(author, "Rory Tyger") | keep book_no, title, author, _score; book_no:keyword | title:text | author:text | _score:double -6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.5822172164917 +6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.821435928344727 ; testMatchAndQueryStringFunctionsWithScore required_capability: match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where match(author, "Keith Faulkner") and qstr("author:Rory or author:Beverlie") @@ -110,13 +117,14 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | _score:double -3535 | Rainbow's End: A Magical Story and Moneybox | [Beverlie Manson, Keith Faulkner] | 6.5579609870910645 -6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 5.975414276123047 +3535 | Rainbow's End: A Magical Story and Moneybox | [Beverlie Manson, Keith Faulkner] | 5.961376190185547 +6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 5.961376190185547 ; testMultiMatchWithScore required_capability: multi_match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where multi_match("Mark", author, title, {"fuzziness": 1}) @@ -124,12 +132,13 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | _score:double -2847 | To Love A Dark Stranger (Lovegram Historical Romance) | Colleen Faulkner | 1.9662091732025146 +2847 | To Love A Dark Stranger (Lovegram Historical Romance) | Colleen Faulkner | 2.0541491508483887 ; testMultiMatchWithScore1 required_capability: multi_match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where multi_match("Hobbit", description, title, {"type": "best_fields"}) @@ -139,24 +148,25 @@ from books metadata _score ignoreOrder:true book_no:keyword | _score:double -1463 | 2.0 -2301 | 2.0 -2675 | 2.0 -2714 | 2.0 -2936 | 1.0 -4023 | 2.0 -4289 | 3.0 -5335 | 2.0 -5996 | 2.0 -6405 | 2.0 -6760 | 2.0 -7350 | 2.0 -7480 | 3.0 +1463 | 2.0 +2301 | 1.0 +2675 | 3.0 +2714 | 2.0 +2936 | 2.0 +4023 | 2.0 +4289 | 3.0 +5335 | 2.0 +5996 | 1.0 +6405 | 3.0 +6760 | 2.0 +7350 | 2.0 +7480 | 4.0 ; testMultiMatchWithScore2 required_capability: multi_match_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where multi_match("Hobbit", description, title, {"type": "most_fields"}) @@ -166,24 +176,25 @@ from books metadata _score ignoreOrder:true book_no:keyword | _score:double -1463 | 2.0 -2301 | 2.0 -2675 | 2.0 -2714 | 2.0 -2936 | 1.0 -4023 | 2.0 -4289 | 6.0 -5335 | 2.0 -5996 | 2.0 -6405 | 2.0 -6760 | 2.0 -7350 | 2.0 -7480 | 3.0 +1463 | 2.0 +2301 | 1.0 +2675 | 3.0 +2714 | 2.0 +2936 | 2.0 +4023 | 2.0 +4289 | 6.0 +5335 | 2.0 +5996 | 1.0 +6405 | 3.0 +6760 | 2.0 +7350 | 2.0 +7480 | 4.0 ; multipleWhereWithMatchScoringNoSort required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update from books metadata _score | where title:"short stories" @@ -191,13 +202,14 @@ from books metadata _score | keep book_no, title, author, _score; ignoreOrder:true -book_no:keyword | title:text | author:text | _score:double -8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 11.193471908569336 +book_no:keyword | title:text | author:text | _score:double +8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 13.820621490478516 ; multipleWhereWithMatchScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update from books metadata _score | where title:"short stories" @@ -205,14 +217,15 @@ from books metadata _score | keep book_no, title, author, _score | sort book_no; -book_no:keyword | title:text | author:text | _score:double -8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 11.193471908569336 +book_no:keyword | title:text | author:text | _score:double +8480 | The wind's twelve quarters: Short stories | Ursula K. Le Guin | 13.820621490478516 ; combinedMatchWithFunctionsScoring required_capability: metadata_score required_capability: match_operator_colon required_capability: non_full_text_functions_scoring +required_capability: routing_function_update from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -221,39 +234,41 @@ from books metadata _score | sort book_no; book_no:keyword | title:text | author:text | year:integer | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.733664035797119 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.3956053256988525 ; singleQstrScoring required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score -| where qstr("author:William Faulkner") -| keep book_no, author, _score -| sort book_no +from books metadata _score +| where qstr("author:William Faulkner") +| keep book_no, author, _score +| sort book_no | LIMIT 2; book_no:keyword | author:text | _score:double -2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.3697924613952637 -2713 | William Faulkner | 4.631696701049805 +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.2796473503112793 +2713 | William Faulkner | 5.796701431274414 ; singleQstrScoringGrok required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score -| where qstr("author:Lord Rings") -| GROK title "%{WORD:title} %{WORD}" -| sort _score desc -| keep book_no, title, _score +from books metadata _score +| where qstr("author:Lord Rings") +| GROK title "%{WORD:title} %{WORD}" +| sort _score desc +| keep book_no, title, _score | LIMIT 3; book_no:keyword | title:keyword | _score:double -8875 | The | 2.769660472869873 -1463 | Realms | 2.6714818477630615 -2675 | The | 2.5619282722473145 +8875 | The | 3.341614246368408 +4023 | A | 2.855410575866699 +1463 | Realms | 2.465149402618408 ; combinedMatchWithScoringEvalNoSort @@ -275,42 +290,45 @@ book_no:keyword | title:text | author:text | year:integer | c_s singleQstrScoringRename required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score -| where qstr("author:Lord Rings") -| rename _score as rank -| sort rank desc -| keep book_no, rank +from books metadata _score +| where qstr("author:Lord Rings") +| rename _score as rank +| sort rank desc +| keep book_no, rank | LIMIT 3; book_no:keyword | rank:double -8875 | 2.769660472869873 -1463 | 2.6714818477630615 -2675 | 2.5619282722473145 +8875 | 3.341614246368408 +4023 | 2.855410575866699 +1463 | 2.465149402618408 ; singleMatchWithTextFieldScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: routing_function_update -from books metadata _score -| where author:"William Faulkner" -| sort book_no -| keep book_no, author, _score +from books metadata _score +| where author:"William Faulkner" +| sort book_no +| keep book_no, author, _score | limit 5; book_no:keyword | author:text | _score:double -2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.3697924613952637 -2713 | William Faulkner | 3.2750158309936523 -2847 | Colleen Faulkner | 1.593343734741211 -2883 | William Faulkner | 3.2750158309936523 -3293 | Danny Faulkner | 1.593343734741211 +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.2796473503112793 +2713 | William Faulkner | 5.01743221282959 +2847 | Colleen Faulkner | 1.3154147863388062 +2883 | William Faulkner | 3.136009931564331 +3293 | Danny Faulkner | 2.0181751251220703 ; combinedMatchWithFunctionsScoringNoSort required_capability: metadata_score required_capability: match_operator_colon required_capability: non_full_text_functions_scoring +required_capability: routing_function_update from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -319,7 +337,7 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | year:integer | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.733664035797119 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.3956053256988525 ; combinedMatchWithScoringEval @@ -341,35 +359,37 @@ book_no:keyword | title:text | author:text | year:integer | c_s singleQstrScoringEval required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score -| where qstr("author:Lord Rings") -| eval c_score = ceil(_score) -| keep book_no, c_score -| sort book_no desc +from books metadata _score +| where qstr("author:Lord Rings") +| eval c_score = ceil(_score) +| keep book_no, c_score +| sort book_no desc | LIMIT 3; book_no:keyword | c_score:double -8875 | 3.0 -7480 | 1.0 -7350 | 1.0 +8875 | 4.0 +7480 | 2.0 +7350 | 2.0 ; QstrScoreManipulation required_capability: metadata_score required_capability: qstr_function +required_capability: routing_function_update -from books metadata _score -| where qstr("title:gentle") -| eval _score = _score + 1 +from books metadata _score +| where qstr("title:gentle") +| eval _score = _score + 1 | keep book_no, title, _score | limit 10 ; ignoreOrder:true book_no:keyword | title:text | _score:double -2924 | A Gentle Creature and Other Stories: White Nights, A Gentle Creature, and The Dream of a Ridiculous Man (The World's Classics) | 3.158426523208618 -5948 | That We Are Gentle Creatures | 3.727346897125244 +2924 | A Gentle Creature and Other Stories: White Nights, A Gentle Creature, and The Dream of a Ridiculous Man (The World's Classics) | 3.112499237060547 +5948 | That We Are Gentle Creatures | 3.709021806716919 ; QstrScoreOverride @@ -468,18 +488,19 @@ conjunctionScoresPushableNonPushableFunctions required_capability: metadata_score required_capability: match_function +required_capability: routing_function_update -from books metadata _score +from books metadata _score | where match(title, "Lord") and length(title) > 20 | keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -2675 | 2.5619282722473145 -2714 | 1.9245924949645996 -7140 | 1.746896743774414 -4023 | 1.5062403678894043 +book_no:keyword | _score:double +2675 | 1.9618241786956787 +2714 | 1.7047617435455322 +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 ; conjunctionScoresPushableFunctions @@ -487,16 +508,17 @@ conjunctionScoresPushableFunctions required_capability: metadata_score required_capability: match_function required_capability: non_full_text_functions_scoring +required_capability: routing_function_update -from books metadata _score +from books metadata _score | where match(title, "Lord") and ratings > 4.6 | keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -7140 | 1.746896743774414 -4023 | 1.5062403678894043 +book_no:keyword | _score:double +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 ; disjunctionScoresPushableNonPushableFunctions @@ -505,18 +527,19 @@ required_capability: metadata_score required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score required_capability: non_full_text_functions_scoring +required_capability: routing_function_update -from books metadata _score +from books metadata _score | where match(title, "Lord") or length(title) > 100 | keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -2675 | 2.5619282722473145 -2714 | 1.9245924949645996 -7140 | 1.746896743774414 -4023 | 1.5062403678894043 +book_no:keyword | _score:double +2675 | 1.9618241786956787 +2714 | 1.7047617435455322 +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 2924 | 0.0 8678 | 0.0 ; @@ -526,22 +549,23 @@ disjunctionScoresMultipleClauses required_capability: metadata_score required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score +required_capability: routing_function_update -from books metadata _score -| where (title: "Lord" and length(title) > 40) or (author: "Dostoevsky" and length(title) > 40) -| keep book_no, _score +from books metadata _score +| where (title: "Lord" and length(title) > 40) or (author: "Dostoevsky" and length(title) > 40) +| keep book_no, _score | sort _score desc, book_no asc ; -book_no:keyword | _score:double -8086 | 2.786686897277832 -9801 | 2.786686897277832 -1937 | 2.1503653526306152 -8534 | 2.1503653526306152 -2714 | 1.9245924949645996 -7140 | 1.746896743774414 -4023 | 1.5062403678894043 -2924 | 1.2732219696044922 +book_no:keyword | _score:double +1937 | 2.7213401794433594 +9801 | 2.7213401794433594 +8086 | 2.4077744483947754 +8534 | 2.4077744483947754 +2714 | 1.7047617435455322 +4023 | 1.4636166095733643 +7140 | 1.3600037097930908 +2924 | 1.017024040222168 ; statsScores @@ -549,20 +573,22 @@ statsScores required_capability: metadata_score required_capability: match_function required_capability: full_text_functions_in_stats_where +required_capability: routing_function_update -from books metadata _score +from books metadata _score | where match(title, "Lord Rings", {"operator": "AND"}) | stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score) ; -avg_score:double | max_score:double | min_score:double -3.869828939437866 | 5.123856544494629 | 3.0124807357788086 +avg_score:double | max_score:double | min_score:double +3.245103120803833 | 3.9236483573913574 | 2.7200074195861816 ; testMatchPhraseWithScore required_capability: match_phrase_function required_capability: metadata_score +required_capability: routing_function_update from books metadata _score | where match_phrase(title, "J. R. R. Tolkien") @@ -570,13 +596,14 @@ from books metadata _score | sort _score desc ; -book_no:keyword | title:text | author:text | _score:double - 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 9.017186164855957 - 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 8.412636756896973 +book_no:keyword | title:text | author:text | _score:double + 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 8.376850128173828 + 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 7.847296714782715 ; testMatchPhraseWithScoreBoost required_capability: match_phrase_function +required_capability: routing_function_update from books metadata _score | where match_phrase(title, "J. R. R. Tolkien", {"boost": 5}) @@ -584,7 +611,7 @@ from books metadata _score | sort _score desc ; -book_no:keyword | title:text | author:text | _score:double - 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 45.0859260559082 - 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 42.06318283081055 +book_no:keyword | title:text | author:text | _score:double + 5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 41.88425064086914 + 2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 39.23648452758789 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec index 9eafeb2663cb8..e6b46a5d2fe9b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec @@ -655,6 +655,7 @@ subqueryInFromWithCompletionInSubquery required_capability: completion required_capability: match_operator_colon required_capability: subquery_in_from_command +required_capability: routing_function_update FROM sample_data, (FROM books metadata _score | WHERE title:"war and peace" AND author:"Tolstoy" @@ -668,9 +669,9 @@ FROM sample_data, (FROM books metadata _score ignoreOrder:true title:text | completion:keyword | @timestamp:datetime | client_ip:ip -War and Peace | WAR AND PEACE | null | null -War and Peace (Signet Classics) | WAR AND PEACE (SIGNET CLASSICS) | null | null -null | null | 2023-10-23T13:33:34.937Z | 172.21.0.5 +War and Peace | WAR AND PEACE | null | null +War and Peace: A Novel (6 Volumes) | WAR AND PEACE: A NOVEL (6 VOLUMES) | null | null +null | null | 2023-10-23T13:33:34.937Z | 172.21.0.5 ; subqueryInFromWithCompletionInMainQuery @@ -697,6 +698,7 @@ subqueryInFromWithRerankInSubquery required_capability: rerank required_capability: match_operator_colon required_capability: subquery_in_from_command +required_capability: routing_function_update FROM sample_data, (FROM books METADATA _score, _index | WHERE title:"war and peace" AND author:"Tolstoy" @@ -712,9 +714,9 @@ FROM sample_data, (FROM books METADATA _score, _index ignoreOrder:true _index:keyword | book_no:keyword | title:text | author:text | _score:double | @timestamp:datetime -books | 4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03 | null -books | 5327 | War and Peace | Leo Tolstoy | 0.08 | null -sample_data | null | null | null | null | 2023-10-23T13:33:34.937Z +books | 5327 | War and Peace | Leo Tolstoy | 0.08 | null +books | 9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03 | null +sample_data | null | null | null | null | 2023-10-23T13:33:34.937Z ; subqueryInFromWithRerankInMainQuery diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 80339966ae51f..7287cd4c4358d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1873,6 +1873,10 @@ public enum Cap { * Dense_vector aggregation functions */ DENSE_VECTOR_AGG_FUNCTIONS, + /** + * Marks the move to the hash(doc) % shard_count routing function. Added in #137062. + */ + ROUTING_FUNCTION_UPDATE, // Last capability should still have a comma for fewer merge conflicts when adding new ones :) // This comment prevents the semicolon from being on the previous capability when Spotless formats the file. diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml index e51a859323c71..5e489e060f522 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patterntext/30_template_id.yml @@ -8,6 +8,7 @@ setup: index: test body: settings: + number_of_shards: 1 index.mapping.use_doc_values_skipper: true index.mode: logsdb sort.field: [ "foo.template_id" ] diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml index ac328967d9fcf..4da6abfc64331 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml @@ -254,9 +254,9 @@ setup: - match: { hits.total.value : 3 } - length: { hits.hits: 1 } - - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0.fields.text.0: "term" } - - match: { hits.hits.0.fields.keyword.0: "keyword" } + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.fields.text.0: "term term" } + - match: { hits.hits.0.fields.keyword.0: "other" } --- "rrf retriever with multiple standard retrievers and multiple knn retriever and a filter":