Use the a mod number of shard function to route

docs/reference/query-languages/esql/_snippets/commands/examples/fork.csv-spec/simpleForkWithStats.md

@@ -11,9 +11,9 @@ FROM books METADATA _score
 
 | author:text | score:double | _fork:keyword | total:long |
 | --- | --- | --- | --- |
-| William Faulkner | 2.39 | fork1 | null |
-| William Faulkner | 2.39 | fork1 | null |
-| Colleen Faulkner | 1.59 | fork1 | null |
-| Danny Faulkner | 1.59 | fork1 | null |
-| Keith Faulkner | 1.59 | fork1 | null |
+| Colleen Faulkner | 2.18 | fork1 | null |
+| William Faulkner | 2.18 | fork1 | null |
+| Danny Faulkner | 2.02 | fork1 | null |
+| Paul Faulkner | 2.02 | fork1 | null |
+| William Faulkner | 2.02 | fork1 | null |
 | null | null | fork2 | 18 |

docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/simple-query.md

@@ -13,5 +13,5 @@ FROM books METADATA _score
 | title:text | _score:double |
 | --- | --- |
 | Poems from the Hobbit | 0.0015673980815336108 |
-| A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 0.007936508394777775 |
-| Return of the King Being the Third Part of The Lord of the Rings | 9.960159659385681E-4 |
+| The Lord of the Rings - Boxed Set | 0.0011135857785120606 |
+| Letters of J R R Tolkien | 0.0024999999441206455 |

docs/reference/query-languages/esql/_snippets/commands/examples/rerank.csv-spec/two-queries.md

@@ -13,6 +13,6 @@ FROM books METADATA _score
 
 | title:text | _score:double | rerank_score:double |
 | --- | --- | --- |
-| Return of the Shadow | 2.8181066513061523 | 5.740527994930744E-4 |
-| Return of the King Being the Third Part of The Lord of the Rings | 3.6248698234558105 | 9.000900317914784E-4 |
-| The Lays of Beleriand | 1.3002015352249146 | 9.36329597607255E-4 |
+| Return of the Shadow | 3.4218082427978516 | 5.740527994930744E-4 |
+| Return of the King Being the Third Part of The Lord of the Rings | 2.8398752212524414 | 9.000900317914784E-4 |
+| The Lays of Beleriand | 1.5629040002822876 | 9.36329597607255E-4 |

server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java

@@ -83,7 +83,7 @@ public void testBulkWithWriteIndexAndRouting() {
         indexRequestWithAlias.source(Collections.singletonMap("foo", "baz"));
         BulkResponse bulkResponse = client().prepareBulk().add(indexRequestWithAlias).get();
         assertThat(bulkResponse.getItems()[0].getResponse().getIndex(), equalTo("index3"));
-        assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(0));
+        assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(1));
         assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L));
         assertThat(bulkResponse.getItems()[0].getResponse().status(), equalTo(RestStatus.CREATED));
         assertThat(client().prepareGet("index3", "id").setRouting("1").get().getSource().get("foo"), equalTo("baz"));

server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.mockito.internal.util.collections.Sets;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -27,6 +26,7 @@
 import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
 
 public class PartitionedRoutingIT extends ESIntegTestCase {
 
@@ -50,7 +50,7 @@ public void testVariousPartitionSizes() throws Exception {
 
                 verifyGets(index, routingToDocumentIds);
                 verifyBroadSearches(index, routingToDocumentIds, shards);
-                verifyRoutedSearches(index, routingToDocumentIds, Sets.newSet(partitionSize));
+                verifyRoutedSearches(index, routingToDocumentIds, partitionSize);
             }
         }
     }
@@ -77,20 +77,10 @@ public void testShrinking() throws Exception {
         Map<String, Set<String>> routingToDocumentIds = generateRoutedDocumentIds(index);
 
         while (true) {
-            int factor = originalShards / currentShards;
-
             verifyGets(index, routingToDocumentIds);
             verifyBroadSearches(index, routingToDocumentIds, currentShards);
 
-            // we need the floor and ceiling of the routing_partition_size / factor since the partition size of the shrunken
-            // index will be one of those, depending on the routing value
-            verifyRoutedSearches(
-                index,
-                routingToDocumentIds,
-                Math.floorDiv(partitionSize, factor) == 0
-                    ? Sets.newSet(1, 2)
-                    : Sets.newSet(Math.floorDiv(partitionSize, factor), -Math.floorDiv(-partitionSize, factor))
-            );
+            verifyRoutedSearches(index, routingToDocumentIds, Math.min(partitionSize, currentShards));
 
             updateIndexSettings(
                 Settings.builder()
@@ -149,7 +139,7 @@ public void testUnableToUpdateIndexRoutingPartitionSizes() throws Exception {
         assertThat(exc.getMessage(), containsString("final indexMetadata setting [index.routing_partition_size]"));
     }
 
-    private void verifyRoutedSearches(String index, Map<String, Set<String>> routingToDocumentIds, Set<Integer> expectedShards) {
+    private void verifyRoutedSearches(String index, Map<String, Set<String>> routingToDocumentIds, int expectedShards) {
         for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
             String routing = routingEntry.getKey();
             int expectedDocuments = routingEntry.getValue().size();
@@ -169,9 +159,10 @@ private void verifyRoutedSearches(String index, Map<String, Set<String>> routing
                             + "]"
                     );
 
-                    assertTrue(
+                    assertThat(
                         response.getTotalShards() + " was not in " + expectedShards + " for " + index,
-                        expectedShards.contains(response.getTotalShards())
+                        expectedShards,
+                        equalTo(response.getTotalShards())
                     );
                     assertEquals(expectedDocuments, response.getHits().getTotalHits().value());
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -3128,7 +3128,11 @@ public static ShardId selectSplitShard(int shardId, IndexMetadata sourceIndexMet
         }
         final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards);
         assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata);
-        return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor);
+        if (IndexRouting.shouldUseShardCountModRouting(sourceIndexMetadata.getCreationVersion())) {
+            return new ShardId(sourceIndexMetadata.getIndex(), Math.floorMod(shardId, sourceIndexMetadata.getNumberOfShards()));
+        } else {
+            return new ShardId(sourceIndexMetadata.getIndex(), shardId / routingFactor);
+        }
     }
 
     /**
@@ -3218,8 +3222,15 @@ public static Set<ShardId> selectShrinkShards(int shardId, IndexMetadata sourceI
         }
         int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards);
         Set<ShardId> shards = Sets.newHashSetWithExpectedSize(routingFactor);
-        for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) {
-            shards.add(new ShardId(sourceIndexMetadata.getIndex(), i));
+        if (IndexRouting.shouldUseShardCountModRouting(sourceIndexMetadata.getCreationVersion())) {
+            for (int i = shardId; i < sourceIndexMetadata.getNumberOfShards(); i += numTargetShards) {
+                assert Math.floorMod(i, numTargetShards) == shardId;
+                shards.add(new ShardId(sourceIndexMetadata.getIndex(), i));
+            }
+        } else {
+            for (int i = shardId * routingFactor; i < routingFactor * shardId + routingFactor; i++) {
+                shards.add(new ShardId(sourceIndexMetadata.getIndex(), i));
+            }
         }
         return shards;
     }

server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java

@@ -72,15 +72,19 @@ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) {
     }
 
     protected final String indexName;
+    private final int numberOfShards;
     private final int routingNumShards;
     private final int routingFactor;
+    protected final IndexVersion creationVersion;
     @Nullable
     private final IndexReshardingMetadata indexReshardingMetadata;
 
     private IndexRouting(IndexMetadata metadata) {
         this.indexName = metadata.getIndex().getName();
+        this.numberOfShards = metadata.getNumberOfShards();
         this.routingNumShards = metadata.getRoutingNumShards();
         this.routingFactor = metadata.getRoutingFactor();
+        this.creationVersion = metadata.getCreationVersion();
         this.indexReshardingMetadata = metadata.getReshardingMetadata();
     }
 
@@ -139,11 +143,27 @@ public void postProcess(IndexRequest indexRequest) {}
      */
     public abstract void collectSearchShards(String routing, IntConsumer consumer);
 
+    public static boolean shouldUseShardCountModRouting(final IndexVersion creationVersion) {
+        return creationVersion.onOrAfter(IndexVersions.MOD_ROUTING_FUNCTION);
+    }
+
     /**
      * Convert a hash generated from an {@code (id, routing}) pair into a
      * shard id.
      */
     protected final int hashToShardId(int hash) {
+        if (shouldUseShardCountModRouting(creationVersion)) {
+            return Math.floorMod(hash, numberOfShards);
+        } else {
+            return hashToShardIdOld(hash);
+        }
+    }
+
+    /**
+     * Convert a hash generated from an {@code (id, routing}) pair into a
+     * shard id using the old routingNumShards mechanism.
+     */
+    protected final int hashToShardIdOld(int hash) {
         return Math.floorMod(hash, routingNumShards) / routingFactor;
     }
 
@@ -186,12 +206,10 @@ private int rerouteFromSplitTargetShard(int shardId, IndexReshardingState.Split.
 
     private abstract static class IdAndRoutingOnly extends IndexRouting {
         private final boolean routingRequired;
-        private final IndexVersion creationVersion;
         private final IndexMode indexMode;
 
         IdAndRoutingOnly(IndexMetadata metadata) {
             super(metadata);
-            this.creationVersion = metadata.getCreationVersion();
             MappingMetadata mapping = metadata.mapping();
             this.routingRequired = mapping == null ? false : mapping.routingRequired();
             this.indexMode = metadata.getIndexMode();

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -214,6 +214,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion KEYWORD_FIELDS_KEEP_DUPLICATES_IN_BINARY_DOC_VALUES = def(9_061_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion STORE_FALLBACK_TEXT_FIELDS_IN_BINARY_DOC_VALUES = def(9_062_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion STORE_FALLBACK_MOT_FIELDS_IN_BINARY_DOC_VALUES = def(9_063_0_00, Version.LUCENE_10_3_2);
+    public static final IndexVersion MOD_ROUTING_FUNCTION = def(9_064_0_00, Version.LUCENE_10_3_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java

@@ -119,7 +119,7 @@ public void testErrorCondition() {
                     "target",
                     new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator(
                         new StoreStats(between(1, 100), between(0, 100), between(1, 100)),
-                        (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null
+                        (i) -> i == 1 || i == 5 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null
                     )
                 )
             ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")

server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java

@@ -307,39 +307,44 @@ public void testGetRoutingFactor() {
     }
 
     public void testSelectShrinkShards() {
+        int version = randomBoolean()
+            ? randomIntBetween(IndexVersions.MOD_ROUTING_FUNCTION.id(), IndexVersion.current().id())
+            : randomIntBetween(IndexVersions.FIRST_DETACHED_INDEX_VERSION.id(), IndexVersions.MOD_ROUTING_FUNCTION.id() - 1);
+        boolean postModRoutingFunction = version >= IndexVersions.MOD_ROUTING_FUNCTION.id();
+
         int numberOfReplicas = randomIntBetween(0, 10);
         IndexMetadata metadata = IndexMetadata.builder("foo")
-            .settings(indexSettings(32, numberOfReplicas).put("index.version.created", 1))
+            .settings(indexSettings(32, numberOfReplicas).put("index.version.created", version))
             .creationDate(randomLong())
             .build();
         Set<ShardId> shardIds = IndexMetadata.selectShrinkShards(0, metadata, 8);
         assertEquals(
             shardIds,
             Sets.newHashSet(
                 new ShardId(metadata.getIndex(), 0),
-                new ShardId(metadata.getIndex(), 1),
-                new ShardId(metadata.getIndex(), 2),
-                new ShardId(metadata.getIndex(), 3)
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 8 : 1),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 16 : 2),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 24 : 3)
             )
         );
         shardIds = IndexMetadata.selectShrinkShards(1, metadata, 8);
         assertEquals(
             shardIds,
             Sets.newHashSet(
-                new ShardId(metadata.getIndex(), 4),
-                new ShardId(metadata.getIndex(), 5),
-                new ShardId(metadata.getIndex(), 6),
-                new ShardId(metadata.getIndex(), 7)
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 1 : 4),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 9 : 5),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 17 : 6),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 25 : 7)
             )
         );
         shardIds = IndexMetadata.selectShrinkShards(7, metadata, 8);
         assertEquals(
             shardIds,
             Sets.newHashSet(
-                new ShardId(metadata.getIndex(), 28),
-                new ShardId(metadata.getIndex(), 29),
-                new ShardId(metadata.getIndex(), 30),
-                new ShardId(metadata.getIndex(), 31)
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 7 : 28),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 15 : 29),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 23 : 30),
+                new ShardId(metadata.getIndex(), postModRoutingFunction ? 31 : 31)
             )
         );
 
@@ -379,17 +384,21 @@ public void testSelectResizeShards() {
     }
 
     public void testSelectSplitShard() {
+        int version = randomBoolean()
+            ? IndexVersions.MOD_ROUTING_FUNCTION.id()
+            : randomIntBetween(IndexVersions.FIRST_DETACHED_INDEX_VERSION.id(), IndexVersions.MOD_ROUTING_FUNCTION.id() - 1);
+        boolean postModRoutingFunction = version >= IndexVersions.MOD_ROUTING_FUNCTION.id();
         IndexMetadata metadata = IndexMetadata.builder("foo")
-            .settings(indexSettings(2, 0).put("index.version.created", 1))
+            .settings(indexSettings(2, 0).put("index.version.created", version))
             .creationDate(randomLong())
             .setRoutingNumShards(4)
             .build();
         ShardId shardId = IndexMetadata.selectSplitShard(0, metadata, 4);
         assertEquals(0, shardId.getId());
         shardId = IndexMetadata.selectSplitShard(1, metadata, 4);
-        assertEquals(0, shardId.getId());
+        assertEquals(postModRoutingFunction ? 1 : 0, shardId.getId());
         shardId = IndexMetadata.selectSplitShard(2, metadata, 4);
-        assertEquals(1, shardId.getId());
+        assertEquals(postModRoutingFunction ? 0 : 1, shardId.getId());
         shardId = IndexMetadata.selectSplitShard(3, metadata, 4);
         assertEquals(1, shardId.getId());
 

server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java

@@ -51,6 +51,7 @@
 import static org.hamcrest.Matchers.stringContainsInOrder;
 
 public class IndexRoutingTests extends ESTestCase {
+
     public void testSimpleRoutingRejectsEmptyId() {
         IndexRouting indexRouting = IndexRouting.fromIndexMetadata(
             IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build()
@@ -71,6 +72,27 @@ public void testSimpleRoutingAcceptsId() {
         assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP));
     }
 
+    public void testSimpleRoutingPreAndPostModRoutingChange() {
+        IndexRouting indexRouting = IndexRouting.fromIndexMetadata(
+            IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build()
+        );
+        String id = "1";
+        IndexRequest req = new IndexRequest().id(id);
+        indexRouting.preProcess(req);
+        assertThat(indexRouting.indexShard(req), equalTo(1));
+
+        IndexRouting indexRoutingOld = IndexRouting.fromIndexMetadata(
+            IndexMetadata.builder("test")
+                .settings(settings(IndexVersions.STORE_FALLBACK_TEXT_FIELDS_IN_BINARY_DOC_VALUES))
+                .numberOfShards(2)
+                .numberOfReplicas(1)
+                .setRoutingNumShards(1024)
+                .build()
+        );
+        indexRoutingOld.preProcess(req);
+        assertThat(indexRoutingOld.indexShard(req), equalTo(0));
+    }
+
     public void testSimpleRoutingAssignedRandomId() {
         IndexRouting indexRouting = IndexRouting.fromIndexMetadata(
             IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build()
@@ -210,39 +232,39 @@ public void testPartitionedIndexShrunk() {
         Map<String, Map<String, Integer>> routingIdToShard = new HashMap<>();
 
         Map<String, Integer> routingA = new HashMap<>();
-        routingA.put("a_0", 1);
-        routingA.put("a_1", 2);
-        routingA.put("a_2", 2);
-        routingA.put("a_3", 2);
-        routingA.put("a_4", 1);
-        routingA.put("a_5", 2);
+        routingA.put("a_0", 2);
+        routingA.put("a_1", 0);
+        routingA.put("a_2", 0);
+        routingA.put("a_3", 0);
+        routingA.put("a_4", 3);
+        routingA.put("a_5", 0);
         routingIdToShard.put("a", routingA);
 
         Map<String, Integer> routingB = new HashMap<>();
         routingB.put("b_0", 0);
-        routingB.put("b_1", 0);
+        routingB.put("b_1", 1);
         routingB.put("b_2", 0);
         routingB.put("b_3", 0);
         routingB.put("b_4", 3);
         routingB.put("b_5", 3);
         routingIdToShard.put("b", routingB);
 
         Map<String, Integer> routingC = new HashMap<>();
-        routingC.put("c_0", 1);
-        routingC.put("c_1", 1);
-        routingC.put("c_2", 0);
-        routingC.put("c_3", 0);
-        routingC.put("c_4", 0);
-        routingC.put("c_5", 1);
+        routingC.put("c_0", 3);
+        routingC.put("c_1", 3);
+        routingC.put("c_2", 1);
+        routingC.put("c_3", 1);
+        routingC.put("c_4", 1);
+        routingC.put("c_5", 3);
         routingIdToShard.put("c", routingC);
 
         Map<String, Integer> routingD = new HashMap<>();
-        routingD.put("d_0", 2);
-        routingD.put("d_1", 2);
-        routingD.put("d_2", 3);
-        routingD.put("d_3", 3);
-        routingD.put("d_4", 3);
-        routingD.put("d_5", 3);
+        routingD.put("d_0", 1);
+        routingD.put("d_1", 1);
+        routingD.put("d_2", 2);
+        routingD.put("d_3", 2);
+        routingD.put("d_4", 2);
+        routingD.put("d_5", 2);
         routingIdToShard.put("d", routingD);
 
         IndexRouting indexRouting = IndexRouting.fromIndexMetadata(