diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 0df977f1ece12..5c08cf28d1285 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -264,9 +264,26 @@ static Setting buildNumberOfShardsSetting() { public static final String INDEX_SETTING_PREFIX = "index."; public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards"; + public static final String SETTING_NUMBER_OF_VIRTUAL_SHARDS = "index.number_of_virtual_shards"; static final String DEFAULT_NUMBER_OF_SHARDS = "opensearch.index.default_number_of_shards"; static final String MAX_NUMBER_OF_SHARDS = "opensearch.index.max_number_of_shards"; public static final Setting INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting(); + /** + * Settings for configuring the number of virtual shards on an index. + *

+ * Note: Virtual Shards uses range-based mapping (e.g., {@code vShardId / (V / P)}) to route to + * physical shards instead of default modulo hashing. This ensures contiguous bucket mapping + * and safe physical shard merging operations later in the lifecycle. The number of virtual shards + * must be a multiple of the number of physical shards {@code (V % P == 0)}. + */ + public static final Setting INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING = Setting.intSetting( + SETTING_NUMBER_OF_VIRTUAL_SHARDS, + -1, + -1, + Property.IndexScope, + Property.Final + ); + public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; public static final Setting INDEX_NUMBER_OF_REPLICAS_SETTING = Setting.intSetting( SETTING_NUMBER_OF_REPLICAS, @@ -1324,6 +1341,16 @@ public int getNumberOfShards() { return numberOfShards; } + /** + * Returns the number of virtual shards for this index. + * Returns -1 if virtual shards are disabled. + * + * @return the number of virtual shards or -1 + */ + public int getNumberOfVirtualShards() { + return settings.getAsInt(SETTING_NUMBER_OF_VIRTUAL_SHARDS, -1); + } + public int getNumberOfReplicas() { return numberOfReplicas; } @@ -2190,6 +2217,32 @@ public IndexMetadata build() { ); } + final int numberOfVirtualShards = INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING.get(settings); + if (numberOfVirtualShards != -1) { + if (numberOfVirtualShards < numberOfShards) { + throw new IllegalArgumentException( + "number of virtual shards [" + + numberOfVirtualShards + + "] must be >= number of shards [" + + numberOfShards + + "] for [" + + index + + "]" + ); + } + if (numberOfVirtualShards % numberOfShards != 0) { + throw new IllegalArgumentException( + "number of virtual shards [" + + numberOfVirtualShards + + "] must be a multiple of number of shards [" + + numberOfShards + + "] for [" + + index + + "]" + ); + } + } + // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable final Map> filledInSyncAllocationIds = new HashMap<>(); for (int i = 0; i < numberOfShards; i++) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java new file mode 100644 index 0000000000000..a3ff1bbaee8dd --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelper.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.PublicApi; + +import java.util.Map; + +/** + * Resolves virtual shard routing to physical shard IDs. + */ +@PublicApi(since = "3.6.0") +public final class VirtualShardRoutingHelper { + + private VirtualShardRoutingHelper() {} + + private static final Logger logger = LogManager.getLogger(VirtualShardRoutingHelper.class); + + /** + * Custom Metadata key for storing virtual shard routing overrides. + */ + public static final String VIRTUAL_SHARDS_CUSTOM_METADATA_KEY = "virtual_shards_routing"; + + /** + * Resolves the physical shard for a virtual shard id. + */ + public static int resolvePhysicalShardId(IndexMetadata indexMetadata, int vShardId) { + int numVirtualShards = indexMetadata.getNumberOfVirtualShards(); + int numPhysicalShards = indexMetadata.getNumberOfShards(); + + if (numVirtualShards < numPhysicalShards || numVirtualShards % numPhysicalShards != 0) { + throw new IllegalArgumentException( + "Virtual shards must be enabled and be a multiple of the number of physical shards to resolve routing." + ); + } + + vShardId = Math.floorMod(vShardId, numVirtualShards); + + Map overrides = indexMetadata.getCustomData(VIRTUAL_SHARDS_CUSTOM_METADATA_KEY); + if (overrides != null) { + String pShardIdStr = overrides.get(String.valueOf(vShardId)); + if (pShardIdStr != null) { + try { + int pShardId = Integer.parseInt(pShardIdStr); + if (pShardId >= 0 && pShardId < numPhysicalShards) { + return pShardId; + } + logger.trace("Invalid override value [{}] for vShard [{}]: out of bounds", pShardId, vShardId); + } catch (NumberFormatException e) { + logger.trace("Invalid override value [{}] for vShard [{}]: not a number", pShardIdStr, vShardId); + } + } + } + + int virtualShardsPerPhysical = numVirtualShards / numPhysicalShards; + return vShardId / virtualShardsPerPhysical; + } +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index 31d5c6a6e8bbe..a2cbf191572b6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -35,6 +35,7 @@ import org.apache.lucene.util.CollectionUtil; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.VirtualShardRoutingHelper; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; @@ -521,6 +522,13 @@ public static int generateShardId(IndexMetadata indexMetadata, @Nullable String partitionOffset = 0; } + int numVirtualShards = indexMetadata.getNumberOfVirtualShards(); + if (numVirtualShards != -1) { + final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset; + int vShardId = Math.floorMod(hash, numVirtualShards); + return VirtualShardRoutingHelper.resolvePhysicalShardId(indexMetadata, vShardId); + } + return calculateScaledShardId(indexMetadata, effectiveRouting, partitionOffset); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 5486f1c019695..ce63a66a4b27c 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -105,6 +105,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING, + IndexMetadata.INDEX_NUMBER_OF_VIRTUAL_SHARDS_SETTING, IndexMetadata.INDEX_READ_ONLY_SETTING, IndexMetadata.INDEX_BLOCKS_READ_SETTING, IndexMetadata.INDEX_BLOCKS_WRITE_SETTING, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java new file mode 100644 index 0000000000000..93adb398cdaa1 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/VirtualShardRoutingHelperTests.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class VirtualShardRoutingHelperTests extends OpenSearchTestCase { + + public void testResolvePhysicalShardIdDefaultRangeBased() { + int numPhysicalShards = 5; + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, 20) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 3)); + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 4)); + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 19)); + } + + public void testResolvePhysicalShardIdWithOverrides() { + int numPhysicalShards = 5; + Map overrides = new HashMap<>(); + overrides.put("7", "1"); + overrides.put("8", "2"); + + IndexMetadata.Builder builder = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, 20) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1); + builder.putCustom(VirtualShardRoutingHelper.VIRTUAL_SHARDS_CUSTOM_METADATA_KEY, overrides); + + IndexMetadata metadata = builder.build(); + + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); + + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 0)); + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 9)); + } + + public void testInvalidOverridesFallBackToRangeBased() { + int numPhysicalShards = 5; + Map overrides = new HashMap<>(); + overrides.put("7", "not_a_number"); + overrides.put("8", "-1"); + overrides.put("19", "5"); + + IndexMetadata.Builder builder = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, 20) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1); + builder.putCustom(VirtualShardRoutingHelper.VIRTUAL_SHARDS_CUSTOM_METADATA_KEY, overrides); + + IndexMetadata metadata = builder.build(); + + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 7)); + assertEquals(2, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 8)); + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 19)); + } + + public void testResolvePhysicalShardIdInvalidConfigurations() { + int numPhysicalShards = 5; + + IndexMetadata metadataDisabled = org.mockito.Mockito.mock(IndexMetadata.class); + org.mockito.Mockito.when(metadataDisabled.getNumberOfVirtualShards()).thenReturn(-1); + org.mockito.Mockito.when(metadataDisabled.getNumberOfShards()).thenReturn(numPhysicalShards); + + IllegalArgumentException e1 = expectThrows( + IllegalArgumentException.class, + () -> VirtualShardRoutingHelper.resolvePhysicalShardId(metadataDisabled, 0) + ); + assertTrue(e1.getMessage().contains("must be enabled and be a multiple")); + + IndexMetadata metadataInvalid = org.mockito.Mockito.mock(IndexMetadata.class); + org.mockito.Mockito.when(metadataInvalid.getNumberOfVirtualShards()).thenReturn(13); + org.mockito.Mockito.when(metadataInvalid.getNumberOfShards()).thenReturn(numPhysicalShards); + + IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> VirtualShardRoutingHelper.resolvePhysicalShardId(metadataInvalid, 0) + ); + assertTrue(e2.getMessage().contains("must be enabled and be a multiple")); + } + + public void testResolvePhysicalShardIdOutOfBoundsNormalization() { + int numPhysicalShards = 5; + IndexMetadata metadata = org.mockito.Mockito.mock(IndexMetadata.class); + org.mockito.Mockito.when(metadata.getNumberOfVirtualShards()).thenReturn(20); + org.mockito.Mockito.when(metadata.getNumberOfShards()).thenReturn(numPhysicalShards); + + assertEquals(4, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, -1)); + + assertEquals(0, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 20)); + assertEquals(1, VirtualShardRoutingHelper.resolvePhysicalShardId(metadata, 25)); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java new file mode 100644 index 0000000000000..3d3897955eb3d --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/VirtualShardOperationRoutingTests.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing; + +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +public class VirtualShardOperationRoutingTests extends OpenSearchTestCase { + + public void testGenerateVirtualShardId() { + int numPhysicalShards = 4; + int numVirtualShards = 16; + + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShards) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + String routing = "user1"; + int routingHash = Murmur3HashFunction.hash(routing); + int expectedVShard = Math.floorMod(routingHash, numVirtualShards); + int expectedPShard = expectedVShard / (numVirtualShards / numPhysicalShards); + + assertEquals(expectedPShard, OperationRouting.generateShardId(metadata, "doc1", routing)); + } + + public void testVirtualShardDisabledUsesLegacy() { + int numPhysicalShards = 4; + + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, -1) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + String routing = "user1"; + int legacyHash = Murmur3HashFunction.hash(routing); + int expectedLegacyShard = Math.floorMod(legacyHash, numPhysicalShards); + + assertEquals(expectedLegacyShard, OperationRouting.generateShardId(metadata, "doc1", routing)); + } + + public void testVirtualShardValidation() { + int numPhysicalShards = 10; + int numVirtualShards = 5; + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShards) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build() + ); + + assertTrue(e.getMessage().contains("must be >= number of shards")); + + int numVirtualShardsInvalid = 15; + + IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.builder("test2") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShardsInvalid) + ) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build() + ); + + assertTrue(e2.getMessage().contains("must be a multiple of number of shards")); + } + + public void testVirtualShardWithRoutingPartition() { + int numPhysicalShards = 4; + int numVirtualShards = 16; + int partitionSize = 2; + + IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_VIRTUAL_SHARDS, numVirtualShards) + ) + .routingPartitionSize(partitionSize) + .numberOfShards(numPhysicalShards) + .numberOfReplicas(1) + .build(); + + String id = "doc1"; + String routing = "user1"; + + int partitionOffset = Math.floorMod(Murmur3HashFunction.hash(id), partitionSize); + int routingHash = Murmur3HashFunction.hash(routing) + partitionOffset; + int expectedVShard = Math.floorMod(routingHash, numVirtualShards); + int expectedPShard = expectedVShard / (numVirtualShards / numPhysicalShards); + + assertEquals(expectedPShard, OperationRouting.generateShardId(metadata, id, routing)); + } +} diff --git a/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java b/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java index 1f63a3724f258..6b80fd635285b 100644 --- a/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java +++ b/server/src/test/java/org/opensearch/common/settings/ConsistentSettingsServiceTests.java @@ -36,8 +36,12 @@ import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.service.ClusterService; import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assume; import org.junit.Before; +import javax.crypto.SecretKeyFactory; + +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Locale; import java.util.concurrent.atomic.AtomicReference; @@ -55,6 +59,12 @@ public class ConsistentSettingsServiceTests extends OpenSearchTestCase { @Before public void init() throws Exception { + try { + SecretKeyFactory.getInstance("PBKDF2WithHmacSHA512"); + } catch (NoSuchAlgorithmException e) { + Assume.assumeNoException("PBKDF2WithHmacSHA512 algorithm is not available", e); + } + clusterState.set(ClusterState.EMPTY_STATE); clusterService = mock(ClusterService.class); Mockito.doAnswer((Answer) invocation -> { return clusterState.get(); }).when(clusterService).state();