From 47bfa503d51b90b90275db90edeb4bfce1c91dad Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Sat, 14 Feb 2026 01:30:03 +0530 Subject: [PATCH 1/6] Add Virtual Shards routing and mapping overrides Implements the initial routing foundation for Virtual Shards. Adds settings validation for virtual shards and routing overrides via IndexMetadata custom data. Signed-off-by: Atri Sharma --- .../cluster/metadata/IndexMetadata.java | 32 ++++++ .../metadata/VirtualShardRoutingHelper.java | 60 ++++++++++ .../cluster/routing/OperationRouting.java | 8 ++ .../common/settings/IndexScopedSettings.java | 1 + .../VirtualShardRoutingHelperTests.java | 71 ++++++++++++ .../VirtualShardOperationRoutingTests.java | 106 ++++++++++++++++++ 6 files changed, 278 insertions(+) create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 0df977f1ece12..fa3b11807b733 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -264,9 +264,18 @@ static Setting buildNumberOfShardsSetting() { public static final String INDEX_SETTING_PREFIX = "index."; public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards"; + public static final String SETTING_NUMBER_OF_VIRTUAL_SHARDS = "index.number_of_virtual_shards"; static final String DEFAULT_NUMBER_OF_SHARDS = "opensearch.index.default_number_of_shards"; static final String MAX_NUMBER_OF_SHARDS = "opensearch.index.max_number_of_shards"; public static final Setting INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting(); + public static final Setting INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING = Setting.intSetting( + SETTING_NUMBER_OF_VIRTUAL_SHARDS, + -1, + -1, + Property.IndexScope, + Property.Final + ); + public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; public static final Setting INDEX_NUMBER_OF_REPLICAS_SETTING = Setting.intSetting( SETTING_NUMBER_OF_REPLICAS, @@ -1324,6 +1333,16 @@ public int getNumberOfShards() { return numberOfShards; } + /** + * Returns the number of virtual shards for this index. + * Returns -1 if virtual shards are disabled. + * + * @return the number of virtual shards or -1 + */ + public int getNumberOfVirtualShards() { + return settings.getAsInt(SETTING_NUMBER_OF_VIRTUAL_SHARDS, -1); + } + public int getNumberOfReplicas() { return numberOfReplicas; } @@ -2190,6 +2209,19 @@ public IndexMetadata build() { ); } + final int numberOfVirtualShards = INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING.get(settings); + if (numberOfVirtualShards != -1 && numberOfVirtualShards < numberOfShards) { + throw new IllegalArgumentException( + "number of virtual shards [" + + numberOfVirtualShards + + "] must be >= number of shards [" + + numberOfShards + + "] for [" + + index + + "]" + ); + } + // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable final Map> filledInSyncAllocationIds = new HashMap<>(); for (int i = 0; i < numberOfShards; i++) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java new file mode 100644 index 0000000000000..0d6b67ad7d1a8 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.PublicApi; + +import java.util.Map; + +/** + * Resolves virtual shard routing to physical shard IDs. + * + * @opensearch.api + */ +@PublicApi(since = "3.6.0") +public final class VirtualShardRoutingHelper { + + private VirtualShardRoutingHelper() {} + + private static final Logger logger = LogManager.getLogger(VirtualShardRoutingHelper.class); + + /** + * Custom Metadata key for storing virtual shard routing overrides. + */ + public static final String VIRTUAL_SHARDS_CUSTOM_METADATA_KEY = "virtual_shards_routing"; + + /** + * Resolves the Physical Shard ID for a given Virtual Shard ID. + * + * @param indexMetadata The index metadata. + * @param vShardId The virtual shard ID. + * @return The physical shard ID. + */ + public static int resolvePhysicalShardId(IndexMetadata indexMetadata, int vShardId) { + Map overrides = indexMetadata.getCustomData(VIRTUAL_SHARDS_CUSTOM_METADATA_KEY); + if (overrides != null) { + String pShardIdStr = overrides.get(String.valueOf(vShardId)); + if (pShardIdStr != null) { + try { + int pShardId = Integer.parseInt(pShardIdStr); + if (pShardId >= 0 && pShardId < indexMetadata.getNumberOfShards()) { + return pShardId; + } + logger.trace("Invalid override value [{}] for vShard [{}]: out of bounds", pShardId, vShardId); + } catch (NumberFormatException e) { + logger.trace("Invalid override value [{}] for vShard [{}]: not a number", pShardIdStr, vShardId); + } + } + } + + return Math.floorMod(vShardId, indexMetadata.getNumberOfShards()); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index 31d5c6a6e8bbe..a2cbf191572b6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -35,6 +35,7 @@ import org.apache.lucene.util.CollectionUtil; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.VirtualShardRoutingHelper; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; @@ -521,6 +522,13 @@ public static int generateShardId(IndexMetadata indexMetadata, @Nullable String partitionOffset = 0; } + int numVirtualShards = indexMetadata.getNumberOfVirtualShards(); + if (numVirtualShards != -1) { + final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset; + int vShardId = Math.floorMod(hash, numVirtualShards); + return VirtualShardRoutingHelper.resolvePhysicalShardId(indexMetadata, vShardId); + } + return calculateScaledShardId(indexMetadata, effectiveRouting, partitionOffset); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 5486f1c019695..ce63a66a4b27c 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -105,6 +105,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING, + IndexMetadata.INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING, IndexMetadata.INDEX_READ_ONLY_SETTING, IndexMetadata.INDEX_BLOCKS_READ_SETTING, IndexMetadata.INDEX_BLOCKS_WRITE_SETTING, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java new file mode 100644 index 0000000000000..c5aeb5dd21683 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class VirtualShardRoutingHelperTests extends OpenSearchTestCase { + + public void testResolvePhysicalShardIdDefaultModulo() { + int numPhysicalShards = 5; + IndexMetadata metadata = IndexMetadata.builder("test") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 4)); + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 5)); + } + + public void testResolvePhysicalShardIdWithOverrides() { + int numPhysicalShards = 5; + Map overrides = new HashMap<>(); + overrides.put("7", "1"); + + IndexMetadata.Builder builder = IndexMetadata.builder("test") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1); + builder.putCustom(VirtualShardRoutingHelper.VIRTUAL_SHARDS_CUSTOM_METADATA_KEY, overrides); + + IndexMetadata metadata = builder.build(); + + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); + } + + public void testInvalidOverridesFallBackToModulo() { + int numPhysicalShards = 5; + Map overrides = new HashMap<>(); + overrides.put("7", "not_a_number"); + overrides.put("8", "-1"); + overrides.put("9", "5"); + + IndexMetadata.Builder builder = IndexMetadata.builder("test") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1); + builder.putCustom(VirtualShardRoutingHelper.VIRTUAL_SHARDS_CUSTOM_METADATA_KEY, overrides); + + IndexMetadata metadata = builder.build(); + + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(3, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 9)); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java new file mode 100644 index 0000000000000..3377d5909fcfd --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing; + +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +public class VirtualShardOperationRoutingTests extends OpenSearchTestCase { + + public void testGenerateVirtualShardId() { + int numPhysicalShards = 4; + int numVirtualShards = 16; + + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShards) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + String routing = "user1"; + int routingHash = Murmur3HashFunction.hash(routing); + int expectedVShard = Math.floorMod(routingHash, numVirtualShards); + int expectedPShard = Math.floorMod(expectedVShard, numPhysicalShards); + + assertEquals(expectedPShard, OperationRouting.generateShardId(metadata, "doc1", routing)); + } + + public void testVirtualShardDisabledUsesLegacy() { + int numPhysicalShards = 4; + + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, -1) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + String routing = "user1"; + int legacyHash = Murmur3HashFunction.hash(routing); + int expectedLegacyShard = Math.floorMod(legacyHash, numPhysicalShards); + + assertEquals(expectedLegacyShard, OperationRouting.generateShardId(metadata, "doc1", routing)); + } + + public void testVirtualShardValidation() { + int numPhysicalShards = 10; + int numVirtualShards = 5; + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShards) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build() + ); + + assertTrue(e.getMessage().contains("must be >= number of shards")); + } + + public void testVirtualShardWithRoutingPartition() { + int numPhysicalShards = 4; + int numVirtualShards = 16; + int partitionSize = 2; + + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShards) + ) + .routingPartitionSize(partitionSize) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + String id = "doc1"; + String routing = "user1"; + + int partitionOffset = Math.floorMod(Murmur3HashFunction.hash(id), partitionSize); + int routingHash = Murmur3HashFunction.hash(routing) + partitionOffset; + int expectedVShard = Math.floorMod(routingHash, numVirtualShards); + int expectedPShard = Math.floorMod(expectedVShard, numPhysicalShards); + + assertEquals(expectedPShard, OperationRouting.generateShardId(metadata, id, routing)); + } +} From 2b48195e760bbbe2ed6414fcad246b9fe5e41bd1 Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Fri, 27 Feb 2026 23:21:44 +0530 Subject: [PATCH 2/6] Add Virtual Shards routing and mapping overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the initial routing and metadata groundwork for virtual shards. When enabled, routing now uses a virtual shard id (vShardId) before resolving to a physical shard. This separates hash-space size from physical shard count and prepares the path for future shard movement workflows. What’s included - New static index setting: index.number_of_virtual_shards (default -1, disabled). - Routing update in OperationRouting.generateShardId: - virtual-shard path when enabled, - existing behavior unchanged when disabled. - New VirtualShardRoutingHelper for vShardId -> physical shard resolution. - Optional per-index override support via virtual_shards_routing custom metadata. - Test coverage for: - enabled/disabled routing behavior, - validation rules, - override and fallback behavior. Out of scope - Side-car segment extraction flow. - Transport/state-orchestration for managing override lifecycle. Signed-off-by: Atri Sharma --- .../settings/ConsistentSettingsServiceTests.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java b/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java index 1f63a3724f258..6b80fd635285b 100644 --- a/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java +++ b/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java @@ -36,8 +36,12 @@ import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.service.ClusterService; import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assume; import org.junit.Before; +import javax.crypto.SecretKeyFactory; + +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Locale; import java.util.concurrent.atomic.AtomicReference; @@ -55,6 +59,12 @@ public class ConsistentSettingsServiceTests extends OpenSearchTestCase { @Before public void init() throws Exception { + try { + SecretKeyFactory.getInstance("PBKDF2WithHmacSHA512"); + } catch (NoSuchAlgorithmException e) { + Assume.assumeNoException("PBKDF2WithHmacSHA512 algorithm is not available", e); + } + clusterState.set(ClusterState.EMPTY_STATE); clusterService = mock(ClusterService.class); Mockito.doAnswer((Answer) invocation -> { return clusterState.get(); }).when(clusterService).state(); From 2ccbc63d4c5b0c291e72741b1a44a27a797ded34 Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Fri, 27 Feb 2026 23:21:44 +0530 Subject: [PATCH 3/6] Add Virtual Shards routing and mapping overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the initial routing and metadata groundwork for virtual shards. When enabled, routing now uses a virtual shard id (vShardId) before resolving to a physical shard. This separates hash-space size from physical shard count and prepares the path for future shard movement workflows. What’s included - New static index setting: index.number_of_virtual_shards (default -1, disabled). - Routing update in OperationRouting.generateShardId: - virtual-shard path when enabled, - existing behavior unchanged when disabled. - New VirtualShardRoutingHelper for vShardId -> physical shard resolution. - Optional per-index override support via virtual_shards_routing custom metadata. - Test coverage for: - enabled/disabled routing behavior, - validation rules, - override and fallback behavior. Out of scope - Side-car segment extraction flow. - Transport/state-orchestration for managing override lifecycle. Signed-off-by: Atri Sharma --- .../opensearch/cluster/metadata/VirtualShardRoutingHelper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java index 0d6b67ad7d1a8..37a47d37af57d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.annotation.PublicApi; import java.util.Map; @@ -19,7 +18,7 @@ * * @opensearch.api */ -@PublicApi(since = "3.6.0") + public final class VirtualShardRoutingHelper { private VirtualShardRoutingHelper() {} From 2144e227190d7f9ac2115de463a46981e00d7c11 Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Wed, 11 Mar 2026 21:37:23 +0530 Subject: [PATCH 4/6] Refactor Virtual Shards to use range-based routing Signed-off-by: Atri Sharma --- .../cluster/metadata/IndexMetadata.java | 33 +++++++++---- .../metadata/VirtualShardRoutingHelper.java | 5 +- .../VirtualShardRoutingHelperTests.java | 47 ++++++++++++++----- .../VirtualShardOperationRoutingTests.java | 21 ++++++++- 4 files changed, 78 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index fa3b11807b733..8bc56d7515402 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -2210,16 +2210,29 @@ public IndexMetadata build() { } final int numberOfVirtualShards = INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING.get(settings); - if (numberOfVirtualShards != -1 && numberOfVirtualShards < numberOfShards) { - throw new IllegalArgumentException( - "number of virtual shards [" - + numberOfVirtualShards - + "] must be >= number of shards [" - + numberOfShards - + "] for [" - + index - + "]" - ); + if (numberOfVirtualShards != -1) { + if (numberOfVirtualShards < numberOfShards) { + throw new IllegalArgumentException( + "number of virtual shards [" + + numberOfVirtualShards + + "] must be >= number of shards [" + + numberOfShards + + "] for [" + + index + + "]" + ); + } + if (numberOfVirtualShards % numberOfShards != 0) { + throw new IllegalArgumentException( + "number of virtual shards [" + + numberOfVirtualShards + + "] must be a multiple of number of shards [" + + numberOfShards + + "] for [" + + index + + "]" + ); + } } // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable diff --git a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java index 37a47d37af57d..d517b96ef5114 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java @@ -15,8 +15,6 @@ /** * Resolves virtual shard routing to physical shard IDs. - * - * @opensearch.api */ public final class VirtualShardRoutingHelper { @@ -54,6 +52,7 @@ public static int resolvePhysicalShardId(IndexMetadata indexMetadata, int vShard } } - return Math.floorMod(vShardId, indexMetadata.getNumberOfShards()); + int virtualShardsPerPhysical = indexMetadata.getNumberOfVirtualShards() / indexMetadata.getNumberOfShards(); + return vShardId / virtualShardsPerPhysical; } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java index c5aeb5dd21683..35ec78cb5a3e1 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java @@ -17,27 +17,37 @@ public class VirtualShardRoutingHelperTests extends OpenSearchTestCase { - public void testResolvePhysicalShardIdDefaultModulo() { + public void testResolvePhysicalShardIdDefaultRangeBased() { int numPhysicalShards = 5; IndexMetadata metadata = IndexMetadata.builder("test") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, 20) + ) .numberOfShards(numPhysicalShards) .numberOfReplicas(1) .build(); assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); - assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); - assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 4)); - assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 5)); + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 3)); + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 4)); + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 19)); } public void testResolvePhysicalShardIdWithOverrides() { int numPhysicalShards = 5; Map overrides = new HashMap<>(); - overrides.put("7", "1"); + overrides.put("7", "1"); // mapped out of standard routing + overrides.put("8", "2"); // mapped out of standard routing IndexMetadata.Builder builder = IndexMetadata.builder("test") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, 20) + ) .numberOfShards(numPhysicalShards) .numberOfReplicas(1); builder.putCustom(VirtualShardRoutingHelper.VIRTUAL_SHARDS_CUSTOM_METADATA_KEY, overrides); @@ -45,27 +55,38 @@ public void testResolvePhysicalShardIdWithOverrides() { IndexMetadata metadata = builder.build(); assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); + // Default falls back to range-based formula (20 / 5 = 4 vshards per pshard) assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 9)); } - public void testInvalidOverridesFallBackToModulo() { + public void testInvalidOverridesFallBackToRangeBased() { int numPhysicalShards = 5; Map overrides = new HashMap<>(); overrides.put("7", "not_a_number"); overrides.put("8", "-1"); - overrides.put("9", "5"); + overrides.put("19", "5"); IndexMetadata.Builder builder = IndexMetadata.builder("test") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, 20) + ) .numberOfShards(numPhysicalShards) .numberOfReplicas(1); builder.putCustom(VirtualShardRoutingHelper.VIRTUAL_SHARDS_CUSTOM_METADATA_KEY, overrides); IndexMetadata metadata = builder.build(); - assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); - assertEquals(3, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); - assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 9)); + // Standard range-based routing expects 4 vshards per physical shard + // vShard 7 -> 7 / 4 = 1 + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + // vShard 8 -> 8 / 4 = 2 + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); + // vShard 19 -> 19 / 4 = 4 + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 19)); } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java index 3377d5909fcfd..3d3897955eb3d 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java @@ -32,7 +32,7 @@ public void testGenerateVirtualShardId() { String routing = "user1"; int routingHash = Murmur3HashFunction.hash(routing); int expectedVShard = Math.floorMod(routingHash, numVirtualShards); - int expectedPShard = Math.floorMod(expectedVShard, numPhysicalShards); + int expectedPShard = expectedVShard / (numVirtualShards / numPhysicalShards); assertEquals(expectedPShard, OperationRouting.generateShardId(metadata, "doc1", routing)); } @@ -75,6 +75,23 @@ public void testVirtualShardValidation() { ); assertTrue(e.getMessage().contains("must be >= number of shards")); + + int numVirtualShardsInvalid = 15; + + IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.builder("test2") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShardsInvalid) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build() + ); + + assertTrue(e2.getMessage().contains("must be a multiple of number of shards")); } public void testVirtualShardWithRoutingPartition() { @@ -99,7 +116,7 @@ public void testVirtualShardWithRoutingPartition() { int partitionOffset = Math.floorMod(Murmur3HashFunction.hash(id), partitionSize); int routingHash = Murmur3HashFunction.hash(routing) + partitionOffset; int expectedVShard = Math.floorMod(routingHash, numVirtualShards); - int expectedPShard = Math.floorMod(expectedVShard, numPhysicalShards); + int expectedPShard = expectedVShard / (numVirtualShards / numPhysicalShards); assertEquals(expectedPShard, OperationRouting.generateShardId(metadata, id, routing)); } From 8095046a5be90dbd2087c029310c15dd05df60de Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Wed, 11 Mar 2026 21:48:10 +0530 Subject: [PATCH 5/6] Add fail-safe to VirtualShardRoutingHelper and valid configuration tracking test Signed-off-by: Atri Sharma --- .../metadata/VirtualShardRoutingHelper.java | 11 +++++++- .../VirtualShardRoutingHelperTests.java | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java index d517b96ef5114..c5bd89d53a3a3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java @@ -52,7 +52,16 @@ public static int resolvePhysicalShardId(IndexMetadata indexMetadata, int vShard } } - int virtualShardsPerPhysical = indexMetadata.getNumberOfVirtualShards() / indexMetadata.getNumberOfShards(); + int numVirtualShards = indexMetadata.getNumberOfVirtualShards(); + int numPhysicalShards = indexMetadata.getNumberOfShards(); + + if (numVirtualShards < numPhysicalShards || numVirtualShards % numPhysicalShards != 0) { + throw new IllegalArgumentException( + "Virtual shards must be enabled and be a multiple of the number of physical shards to resolve routing." + ); + } + + int virtualShardsPerPhysical = numVirtualShards / numPhysicalShards; return vShardId / virtualShardsPerPhysical; } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java index 35ec78cb5a3e1..ebc5a7abc034f 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java @@ -89,4 +89,30 @@ public void testInvalidOverridesFallBackToRangeBased() { // vShard 19 -> 19 / 4 = 4 assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 19)); } + + public void testResolvePhysicalShardIdInvalidConfigurations() { + int numPhysicalShards = 5; + + // Disabled virtual shards + IndexMetadata metadataDisabled = org.mockito.Mockito.mock(IndexMetadata.class); + org.mockito.Mockito.when(metadataDisabled.getNumberOfVirtualShards()).thenReturn(-1); + org.mockito.Mockito.when(metadataDisabled.getNumberOfShards()).thenReturn(numPhysicalShards); + + IllegalArgumentException e1 = expectThrows( + IllegalArgumentException.class, + () -> VirtualShardRoutingHelper.resolvePhysicalShardId(metadataDisabled, 0) + ); + assertTrue(e1.getMessage().contains("must be enabled and be a multiple")); + + // Invalid multiple + IndexMetadata metadataInvalid = org.mockito.Mockito.mock(IndexMetadata.class); + org.mockito.Mockito.when(metadataInvalid.getNumberOfVirtualShards()).thenReturn(13); + org.mockito.Mockito.when(metadataInvalid.getNumberOfShards()).thenReturn(numPhysicalShards); + + IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> VirtualShardRoutingHelper.resolvePhysicalShardId(metadataInvalid, 0) + ); + assertTrue(e2.getMessage().contains("must be enabled and be a multiple")); + } } From d13a28c6dd8fab5960a42fa7b45be9c7b1f89955 Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Wed, 11 Mar 2026 22:56:55 +0530 Subject: [PATCH 6/6] Fixed review comments and move to range based sharding Signed-off-by: Atri Sharma --- .../cluster/metadata/IndexMetadata.java | 8 +++++ .../metadata/VirtualShardRoutingHelper.java | 31 +++++++++---------- .../VirtualShardRoutingHelperTests.java | 23 ++++++++------ 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 8bc56d7515402..5c08cf28d1285 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -268,6 +268,14 @@ static Setting buildNumberOfShardsSetting() { static final String DEFAULT_NUMBER_OF_SHARDS = "opensearch.index.default_number_of_shards"; static final String MAX_NUMBER_OF_SHARDS = "opensearch.index.max_number_of_shards"; public static final Setting INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting(); + /** + * Settings for configuring the number of virtual shards on an index. + *

+ * Note: Virtual Shards uses range-based mapping (e.g., {@code vShardId / (V / P)}) to route to + * physical shards instead of default modulo hashing. This ensures contiguous bucket mapping + * and safe physical shard merging operations later in the lifecycle. The number of virtual shards + * must be a multiple of the number of physical shards {@code (V % P == 0)}. + */ public static final Setting INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING = Setting.intSetting( SETTING_NUMBER_OF_VIRTUAL_SHARDS, -1, diff --git a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java index c5bd89d53a3a3..a3ff1bbaee8dd 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java @@ -10,13 +10,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.PublicApi; import java.util.Map; /** * Resolves virtual shard routing to physical shard IDs. */ - +@PublicApi(since = "3.6.0") public final class VirtualShardRoutingHelper { private VirtualShardRoutingHelper() {} @@ -29,20 +30,27 @@ private VirtualShardRoutingHelper() {} public static final String VIRTUAL_SHARDS_CUSTOM_METADATA_KEY = "virtual_shards_routing"; /** - * Resolves the Physical Shard ID for a given Virtual Shard ID. - * - * @param indexMetadata The index metadata. - * @param vShardId The virtual shard ID. - * @return The physical shard ID. + * Resolves the physical shard for a virtual shard id. */ public static int resolvePhysicalShardId(IndexMetadata indexMetadata, int vShardId) { + int numVirtualShards = indexMetadata.getNumberOfVirtualShards(); + int numPhysicalShards = indexMetadata.getNumberOfShards(); + + if (numVirtualShards < numPhysicalShards || numVirtualShards % numPhysicalShards != 0) { + throw new IllegalArgumentException( + "Virtual shards must be enabled and be a multiple of the number of physical shards to resolve routing." + ); + } + + vShardId = Math.floorMod(vShardId, numVirtualShards); + Map overrides = indexMetadata.getCustomData(VIRTUAL_SHARDS_CUSTOM_METADATA_KEY); if (overrides != null) { String pShardIdStr = overrides.get(String.valueOf(vShardId)); if (pShardIdStr != null) { try { int pShardId = Integer.parseInt(pShardIdStr); - if (pShardId >= 0 && pShardId < indexMetadata.getNumberOfShards()) { + if (pShardId >= 0 && pShardId < numPhysicalShards) { return pShardId; } logger.trace("Invalid override value [{}] for vShard [{}]: out of bounds", pShardId, vShardId); @@ -52,15 +60,6 @@ public static int resolvePhysicalShardId(IndexMetadata indexMetadata, int vShard } } - int numVirtualShards = indexMetadata.getNumberOfVirtualShards(); - int numPhysicalShards = indexMetadata.getNumberOfShards(); - - if (numVirtualShards < numPhysicalShards || numVirtualShards % numPhysicalShards != 0) { - throw new IllegalArgumentException( - "Virtual shards must be enabled and be a multiple of the number of physical shards to resolve routing." - ); - } - int virtualShardsPerPhysical = numVirtualShards / numPhysicalShards; return vShardId / virtualShardsPerPhysical; } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java index ebc5a7abc034f..93adb398cdaa1 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java @@ -39,8 +39,8 @@ public void testResolvePhysicalShardIdDefaultRangeBased() { public void testResolvePhysicalShardIdWithOverrides() { int numPhysicalShards = 5; Map overrides = new HashMap<>(); - overrides.put("7", "1"); // mapped out of standard routing - overrides.put("8", "2"); // mapped out of standard routing + overrides.put("7", "1"); + overrides.put("8", "2"); IndexMetadata.Builder builder = IndexMetadata.builder("test") .settings( @@ -57,7 +57,6 @@ public void testResolvePhysicalShardIdWithOverrides() { assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); - // Default falls back to range-based formula (20 / 5 = 4 vshards per pshard) assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 9)); } @@ -81,19 +80,14 @@ public void testInvalidOverridesFallBackToRangeBased() { IndexMetadata metadata = builder.build(); - // Standard range-based routing expects 4 vshards per physical shard - // vShard 7 -> 7 / 4 = 1 assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); - // vShard 8 -> 8 / 4 = 2 assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); - // vShard 19 -> 19 / 4 = 4 assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 19)); } public void testResolvePhysicalShardIdInvalidConfigurations() { int numPhysicalShards = 5; - // Disabled virtual shards IndexMetadata metadataDisabled = org.mockito.Mockito.mock(IndexMetadata.class); org.mockito.Mockito.when(metadataDisabled.getNumberOfVirtualShards()).thenReturn(-1); org.mockito.Mockito.when(metadataDisabled.getNumberOfShards()).thenReturn(numPhysicalShards); @@ -104,7 +98,6 @@ public void testResolvePhysicalShardIdInvalidConfigurations() { ); assertTrue(e1.getMessage().contains("must be enabled and be a multiple")); - // Invalid multiple IndexMetadata metadataInvalid = org.mockito.Mockito.mock(IndexMetadata.class); org.mockito.Mockito.when(metadataInvalid.getNumberOfVirtualShards()).thenReturn(13); org.mockito.Mockito.when(metadataInvalid.getNumberOfShards()).thenReturn(numPhysicalShards); @@ -115,4 +108,16 @@ public void testResolvePhysicalShardIdInvalidConfigurations() { ); assertTrue(e2.getMessage().contains("must be enabled and be a multiple")); } + + public void testResolvePhysicalShardIdOutOfBoundsNormalization() { + int numPhysicalShards = 5; + IndexMetadata metadata = org.mockito.Mockito.mock(IndexMetadata.class); + org.mockito.Mockito.when(metadata.getNumberOfVirtualShards()).thenReturn(20); + org.mockito.Mockito.when(metadata.getNumberOfShards()).thenReturn(numPhysicalShards); + + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, -1)); + + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 20)); + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 25)); + } }