Skip to content

Commit c5d26f7

Browse files
rgsrirammsfroh
rgsriram
and
msfroh
authored
Add support for custom remote store segment path prefix to support clusterless configurations (#18857)
--------- Signed-off-by: Sriram Ganesh <[email protected]> Signed-off-by: Michael Froh <[email protected]> Co-authored-by: Michael Froh <[email protected]>
1 parent 0c23ba7 commit c5d26f7

File tree

9 files changed

+286
-4
lines changed

9 files changed

+286
-4
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4646
- Expand fetch phase profiling to multi-shard queries ([#18887](https://github.com/opensearch-project/OpenSearch/pull/18887))
4747
- Prevent shard initialization failure due to streaming consumer errors ([#18877](https://github.com/opensearch-project/OpenSearch/pull/18877))
4848
- APIs for stream transport and new stream-based search api action ([#18722](https://github.com/opensearch-project/OpenSearch/pull/18722))
49+
- Add support for custom remote store segment path prefix to support clusterless configurations ([#18750](https://github.com/opensearch-project/OpenSearch/issues/18750))
4950
- Added the core process for warming merged segments in remote-store enabled domains ([#18683](https://github.com/opensearch-project/OpenSearch/pull/18683))
5051
- Streaming aggregation ([#18874](https://github.com/opensearch-project/OpenSearch/pull/18874))
5152
- Optimize Composite Aggregations by removing unnecessary object allocations ([#18531](https://github.com/opensearch-project/OpenSearch/pull/18531))

‎server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,57 @@ public Iterator<Setting<?>> settings() {
458458
Property.Dynamic
459459
);
460460

461+
/**
462+
* Used to specify a custom path prefix for remote store segments. This allows injecting a unique identifier
463+
* (e.g., writer node ID) into the remote store path to support clusterless configurations where multiple
464+
* writers may write to the same shard.
465+
*/
466+
public static final Setting<String> INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX = Setting.simpleString(
467+
"index.remote_store.segment.path_prefix",
468+
"",
469+
new Setting.Validator<>() {
470+
471+
@Override
472+
public void validate(final String value) {}
473+
474+
@Override
475+
public void validate(final String value, final Map<Setting<?>, Object> settings) {
476+
// Only validate if the value is not null and not empty
477+
if (value != null && !value.trim().isEmpty()) {
478+
// Validate that remote store is enabled when this setting is used
479+
final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
480+
if (isRemoteSegmentStoreEnabled == null || isRemoteSegmentStoreEnabled == false) {
481+
throw new IllegalArgumentException(
482+
"Setting "
483+
+ INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey()
484+
+ " can only be set when "
485+
+ INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
486+
+ " is set to true"
487+
);
488+
}
489+
490+
// Validate that the path prefix doesn't contain invalid characters for file paths
491+
if (value.contains("/") || value.contains("\\") || value.contains(":")) {
492+
throw new IllegalArgumentException(
493+
"Setting "
494+
+ INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey()
495+
+ " cannot contain path separators (/ or \\) or drive specifiers (:)"
496+
);
497+
}
498+
}
499+
}
500+
501+
@Override
502+
public Iterator<Setting<?>> settings() {
503+
final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
504+
return settings.iterator();
505+
}
506+
},
507+
Property.IndexScope,
508+
Property.PrivateIndex,
509+
Property.Dynamic
510+
);
511+
461512
private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, Setting<?> setting) {
462513
final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
463514
if (isRemoteSegmentStoreEnabled == false) {

‎server/src/main/java/org/opensearch/index/IndexService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -695,7 +695,8 @@ public synchronized IndexShard createShard(
695695
RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()),
696696
this.indexSettings.getUUID(),
697697
shardId,
698-
this.indexSettings.getRemoteStorePathStrategy()
698+
this.indexSettings.getRemoteStorePathStrategy(),
699+
this.indexSettings.getRemoteStoreSegmentPathPrefix()
699700
);
700701
}
701702
// When an instance of Store is created, a shardlock is created which is released on closing the instance of store.

‎server/src/main/java/org/opensearch/index/IndexSettings.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,7 @@ public static IndexMergePolicy fromString(String text) {
823823
private volatile TimeValue remoteTranslogUploadBufferInterval;
824824
private volatile String remoteStoreTranslogRepository;
825825
private volatile String remoteStoreRepository;
826+
private volatile String remoteStoreSegmentPathPrefix;
826827
private int remoteTranslogKeepExtraGen;
827828
private boolean autoForcemergeEnabled;
828829

@@ -1041,6 +1042,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
10411042
remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
10421043
remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY);
10431044
this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings);
1045+
String rawPrefix = IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(settings);
1046+
// Only set the prefix if it's explicitly set and not empty
1047+
this.remoteStoreSegmentPathPrefix = (rawPrefix != null && !rawPrefix.trim().isEmpty()) ? rawPrefix : null;
10441048
this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
10451049
this.shouldCleanupUnreferencedFiles = INDEX_UNREFERENCED_FILE_CLEANUP.get(settings);
10461050
this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -1445,6 +1449,13 @@ public String getRemoteStoreTranslogRepository() {
14451449
return remoteStoreTranslogRepository;
14461450
}
14471451

1452+
/**
1453+
* Returns the custom path prefix for remote store segments, if set.
1454+
*/
1455+
public String getRemoteStoreSegmentPathPrefix() {
1456+
return remoteStoreSegmentPathPrefix;
1457+
}
1458+
14481459
/**
14491460
* Returns true if the index is writable warm index and has partial store locality.
14501461
*/

‎server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,12 +230,14 @@ public static class ShardDataPathInput extends PathInput {
230230
private final String shardId;
231231
private final DataCategory dataCategory;
232232
private final DataType dataType;
233+
private final String indexFixedPrefix;
233234

234235
public ShardDataPathInput(Builder builder) {
235236
super(builder);
236237
this.shardId = Objects.requireNonNull(builder.shardId);
237238
this.dataCategory = Objects.requireNonNull(builder.dataCategory);
238239
this.dataType = Objects.requireNonNull(builder.dataType);
240+
this.indexFixedPrefix = builder.indexFixedPrefix; // Can be null
239241
assert dataCategory.isSupportedDataType(dataType) : "category:"
240242
+ dataCategory
241243
+ " type:"
@@ -258,7 +260,12 @@ DataType dataType() {
258260

259261
@Override
260262
BlobPath fixedSubPath() {
261-
return super.fixedSubPath().add(shardId).add(dataCategory.getName()).add(dataType.getName());
263+
BlobPath path = super.fixedSubPath().add(shardId);
264+
// Only add index fixed prefix if it's explicitly set and not empty
265+
if (indexFixedPrefix != null && !indexFixedPrefix.trim().isEmpty()) {
266+
path = path.add(indexFixedPrefix);
267+
}
268+
return path.add(dataCategory.getName()).add(dataType.getName());
262269
}
263270

264271
/**
@@ -279,6 +286,7 @@ public static class Builder extends PathInput.Builder<Builder> {
279286
private String shardId;
280287
private DataCategory dataCategory;
281288
private DataType dataType;
289+
private String indexFixedPrefix;
282290

283291
public Builder shardId(String shardId) {
284292
this.shardId = shardId;
@@ -295,6 +303,11 @@ public Builder dataType(DataType dataType) {
295303
return this;
296304
}
297305

306+
public Builder indexFixedPrefix(String indexFixedPrefix) {
307+
this.indexFixedPrefix = indexFixedPrefix;
308+
return this;
309+
}
310+
298311
@Override
299312
protected Builder self() {
300313
return this;

‎server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,16 @@ public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throw
6565

6666
public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId, RemoteStorePathStrategy pathStrategy)
6767
throws IOException {
68+
return newDirectory(repositoryName, indexUUID, shardId, pathStrategy, null);
69+
}
70+
71+
public Directory newDirectory(
72+
String repositoryName,
73+
String indexUUID,
74+
ShardId shardId,
75+
RemoteStorePathStrategy pathStrategy,
76+
String indexFixedPrefix
77+
) throws IOException {
6878
assert Objects.nonNull(pathStrategy);
6979
try (Repository repository = repositoriesService.get().repository(repositoryName)) {
7080

@@ -81,6 +91,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
8191
.dataCategory(SEGMENTS)
8292
.dataType(DATA)
8393
.fixedPrefix(segmentsPathFixedPrefix)
94+
.indexFixedPrefix(indexFixedPrefix)
8495
.build();
8596
// Derive the path for data directory of SEGMENTS
8697
BlobPath dataPath = pathStrategy.generatePath(dataPathInput);
@@ -100,6 +111,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
100111
.dataCategory(SEGMENTS)
101112
.dataType(METADATA)
102113
.fixedPrefix(segmentsPathFixedPrefix)
114+
.indexFixedPrefix(indexFixedPrefix)
103115
.build();
104116
// Derive the path for metadata directory of SEGMENTS
105117
BlobPath mdPath = pathStrategy.generatePath(mdPathInput);
@@ -112,7 +124,8 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
112124
indexUUID,
113125
shardIdStr,
114126
pathStrategy,
115-
segmentsPathFixedPrefix
127+
segmentsPathFixedPrefix,
128+
indexFixedPrefix
116129
);
117130

118131
return new RemoteSegmentStoreDirectory(

‎server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public RemoteStoreLockManager newLockManager(
4444
String shardId,
4545
RemoteStorePathStrategy pathStrategy
4646
) {
47-
return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix);
47+
return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix, null);
4848
}
4949

5050
public static RemoteStoreMetadataLockManager newLockManager(
@@ -54,6 +54,18 @@ public static RemoteStoreMetadataLockManager newLockManager(
5454
String shardId,
5555
RemoteStorePathStrategy pathStrategy,
5656
String segmentsPathFixedPrefix
57+
) {
58+
return newLockManager(repositoriesService, repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix, null);
59+
}
60+
61+
public static RemoteStoreMetadataLockManager newLockManager(
62+
RepositoriesService repositoriesService,
63+
String repositoryName,
64+
String indexUUID,
65+
String shardId,
66+
RemoteStorePathStrategy pathStrategy,
67+
String segmentsPathFixedPrefix,
68+
String indexFixedPrefix
5769
) {
5870
try (Repository repository = repositoriesService.repository(repositoryName)) {
5971
assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository";
@@ -66,6 +78,7 @@ public static RemoteStoreMetadataLockManager newLockManager(
6678
.dataCategory(SEGMENTS)
6779
.dataType(LOCK_FILES)
6880
.fixedPrefix(segmentsPathFixedPrefix)
81+
.indexFixedPrefix(indexFixedPrefix)
6982
.build();
7083
BlobPath lockDirectoryPath = pathStrategy.generatePath(lockFilesPathInput);
7184
BlobContainer lockDirectoryBlobContainer = ((BlobStoreRepository) repository).blobStore().blobContainer(lockDirectoryPath);

‎server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,4 +606,84 @@ public void testIndicesMetadataDiffSystemFlagFlipped() {
606606
assertThat(indexMetadataAfterDiffApplied.getVersion(), equalTo(nextIndexMetadata.getVersion()));
607607
}
608608

609+
/**
610+
* Test validation for remote store segment path prefix setting
611+
*/
612+
public void testRemoteStoreSegmentPathPrefixValidation() {
613+
// Test empty value (should be allowed)
614+
final Settings emptySettings = Settings.builder()
615+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
616+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "")
617+
.build();
618+
619+
IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(emptySettings);
620+
621+
final Settings whitespaceSettings = Settings.builder()
622+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
623+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), " ")
624+
.build();
625+
626+
IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(whitespaceSettings);
627+
628+
final Settings validSettings = Settings.builder()
629+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
630+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer-node-1")
631+
.build();
632+
633+
String value = IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(validSettings);
634+
assertEquals("writer-node-1", value);
635+
636+
final Settings disabledSettings = Settings.builder()
637+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), false)
638+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer-node-1")
639+
.build();
640+
641+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
642+
IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(disabledSettings);
643+
});
644+
assertTrue(e.getMessage().contains("can only be set when"));
645+
646+
final Settings noRemoteStoreSettings = Settings.builder()
647+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer-node-1")
648+
.build();
649+
650+
e = expectThrows(
651+
IllegalArgumentException.class,
652+
() -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(noRemoteStoreSettings); }
653+
);
654+
assertTrue(e.getMessage().contains("can only be set when"));
655+
656+
final Settings invalidPathSettings = Settings.builder()
657+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
658+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer/node")
659+
.build();
660+
661+
e = expectThrows(
662+
IllegalArgumentException.class,
663+
() -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(invalidPathSettings); }
664+
);
665+
assertTrue(e.getMessage().contains("cannot contain path separators"));
666+
667+
final Settings backslashSettings = Settings.builder()
668+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
669+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer\\node")
670+
.build();
671+
672+
e = expectThrows(
673+
IllegalArgumentException.class,
674+
() -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(backslashSettings); }
675+
);
676+
assertTrue(e.getMessage().contains("cannot contain path separators"));
677+
678+
final Settings colonSettings = Settings.builder()
679+
.put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
680+
.put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer:node")
681+
.build();
682+
683+
e = expectThrows(
684+
IllegalArgumentException.class,
685+
() -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(colonSettings); }
686+
);
687+
assertTrue(e.getMessage().contains("cannot contain path separators"));
688+
}
609689
}

0 commit comments

Comments
 (0)