Skip to content

Commit f4dfc64

Browse files
shiv0408shiv0408
committed
Merge branch '2.x' into backport/backport-15478-to-2.x
2 parents 034375c + 8a119fa commit f4dfc64

File tree

52 files changed

+2699
-232
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+2699
-232
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
55

66
## [Unreleased 2.x]
77
### Added
8+
- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
89

910
### Dependencies
1011
- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))

‎rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml

+36-34
Original file line numberDiff line numberDiff line change
@@ -25,38 +25,40 @@ setup:
2525
snapshot: test_snapshot
2626
wait_for_completion: true
2727

28-
- do:
29-
snapshot.status:
30-
repository: test_repo_status_1
31-
snapshot: test_snapshot
32-
33-
- is_true: snapshots
34-
- match: { snapshots.0.snapshot: test_snapshot }
35-
- match: { snapshots.0.state: SUCCESS }
36-
- gt: { snapshots.0.stats.incremental.file_count: 0 }
37-
- gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
38-
- gt: { snapshots.0.stats.total.file_count: 0 }
39-
- gt: { snapshots.0.stats.total.size_in_bytes: 0 }
40-
- is_true: snapshots.0.stats.start_time_in_millis
41-
## fast in memory snapshots can take less than one millisecond to complete.
42-
- gte: { snapshots.0.stats.time_in_millis: 0 }
43-
44-
---
45-
"Get missing snapshot status throws an exception":
46-
47-
- do:
48-
catch: /snapshot_missing_exception.+ is missing/
49-
snapshot.status:
50-
repository: test_repo_status_1
51-
snapshot: test_nonexistent_snapshot
52-
53-
---
54-
"Get missing snapshot status succeeds when ignoreUnavailable is true":
55-
56-
- do:
57-
snapshot.status:
58-
repository: test_repo_status_1
59-
snapshot: test_nonexistent_snapshot
60-
ignore_unavailable: true
28+
# TODO: fix and unmute tests
6129

62-
- is_true: snapshots
30+
# - do:
31+
# snapshot.status:
32+
# repository: test_repo_status_1
33+
# snapshot: test_snapshot
34+
#
35+
# - is_true: snapshots
36+
# - match: { snapshots.0.snapshot: test_snapshot }
37+
# - match: { snapshots.0.state: SUCCESS }
38+
# - gt: { snapshots.0.stats.incremental.file_count: 0 }
39+
# - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
40+
# - gt: { snapshots.0.stats.total.file_count: 0 }
41+
# - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
42+
# - is_true: snapshots.0.stats.start_time_in_millis
43+
### fast in memory snapshots can take less than one millisecond to complete.
44+
# - gte: { snapshots.0.stats.time_in_millis: 0 }
45+
#
46+
#---
47+
#"Get missing snapshot status throws an exception":
48+
#
49+
# - do:
50+
# catch: /snapshot_missing_exception.+ is missing/
51+
# snapshot.status:
52+
# repository: test_repo_status_1
53+
# snapshot: test_nonexistent_snapshot
54+
#
55+
#---
56+
#"Get missing snapshot status succeeds when ignoreUnavailable is true":
57+
#
58+
# - do:
59+
# snapshot.status:
60+
# repository: test_repo_status_1
61+
# snapshot: test_nonexistent_snapshot
62+
# ignore_unavailable: true
63+
#
64+
# - is_true: snapshots

‎server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java

+163
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
import org.opensearch.action.support.IndicesOptions;
3838
import org.opensearch.action.support.PlainActionFuture;
3939
import org.opensearch.cluster.health.ClusterHealthStatus;
40+
import org.opensearch.cluster.health.ClusterIndexHealth;
41+
import org.opensearch.cluster.health.ClusterShardHealth;
4042
import org.opensearch.cluster.metadata.IndexMetadata;
4143
import org.opensearch.cluster.routing.UnassignedInfo;
4244
import org.opensearch.cluster.service.ClusterService;
@@ -49,6 +51,7 @@
4951

5052
import java.util.ArrayList;
5153
import java.util.List;
54+
import java.util.Map;
5255
import java.util.concurrent.atomic.AtomicBoolean;
5356

5457
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -439,4 +442,164 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
439442
completionFuture.actionGet(TimeValue.timeValueSeconds(30));
440443
}
441444
}
445+
446+
public void testHealthWithClusterLevelAppliedAtTransportLayer() {
447+
createIndex(
448+
"test1",
449+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
450+
);
451+
ensureGreen();
452+
ClusterHealthResponse healthResponse = client().admin()
453+
.cluster()
454+
.prepareHealth()
455+
.setApplyLevelAtTransportLayer(true)
456+
.execute()
457+
.actionGet();
458+
assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
459+
assertTrue(healthResponse.getIndices().isEmpty());
460+
assertEquals(1, healthResponse.getActiveShards());
461+
assertEquals(1, healthResponse.getActivePrimaryShards());
462+
assertEquals(0, healthResponse.getUnassignedShards());
463+
assertEquals(0, healthResponse.getInitializingShards());
464+
assertEquals(0, healthResponse.getRelocatingShards());
465+
assertEquals(0, healthResponse.getDelayedUnassignedShards());
466+
}
467+
468+
public void testHealthWithIndicesLevelAppliedAtTransportLayer() {
469+
createIndex(
470+
"test1",
471+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
472+
);
473+
ensureGreen();
474+
ClusterHealthResponse healthResponse = client().admin()
475+
.cluster()
476+
.prepareHealth()
477+
.setLevel("indices")
478+
.setApplyLevelAtTransportLayer(true)
479+
.execute()
480+
.actionGet();
481+
assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
482+
483+
assertEquals(1, healthResponse.getActiveShards());
484+
assertEquals(1, healthResponse.getActivePrimaryShards());
485+
assertEquals(0, healthResponse.getUnassignedShards());
486+
assertEquals(0, healthResponse.getInitializingShards());
487+
assertEquals(0, healthResponse.getRelocatingShards());
488+
assertEquals(0, healthResponse.getDelayedUnassignedShards());
489+
490+
Map<String, ClusterIndexHealth> indices = healthResponse.getIndices();
491+
assertFalse(indices.isEmpty());
492+
assertEquals(1, indices.size());
493+
for (Map.Entry<String, ClusterIndexHealth> indicesHealth : indices.entrySet()) {
494+
String indexName = indicesHealth.getKey();
495+
assertEquals("test1", indexName);
496+
ClusterIndexHealth indicesHealthValue = indicesHealth.getValue();
497+
assertEquals(1, indicesHealthValue.getActiveShards());
498+
assertEquals(1, indicesHealthValue.getActivePrimaryShards());
499+
assertEquals(0, indicesHealthValue.getInitializingShards());
500+
assertEquals(0, indicesHealthValue.getUnassignedShards());
501+
assertEquals(0, indicesHealthValue.getDelayedUnassignedShards());
502+
assertEquals(0, indicesHealthValue.getRelocatingShards());
503+
assertEquals(ClusterHealthStatus.GREEN, indicesHealthValue.getStatus());
504+
assertTrue(indicesHealthValue.getShards().isEmpty());
505+
}
506+
}
507+
508+
public void testHealthWithShardLevelAppliedAtTransportLayer() {
509+
int dataNodes = internalCluster().getDataNodeNames().size();
510+
int greenClusterReplicaCount = dataNodes - 1;
511+
int yellowClusterReplicaCount = dataNodes;
512+
513+
createIndex(
514+
"test1",
515+
Settings.builder()
516+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
517+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount)
518+
.build()
519+
);
520+
ensureGreen(TimeValue.timeValueSeconds(120), "test1");
521+
createIndex(
522+
"test2",
523+
Settings.builder()
524+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
525+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount)
526+
.build()
527+
);
528+
ensureGreen(TimeValue.timeValueSeconds(120));
529+
client().admin()
530+
.indices()
531+
.prepareUpdateSettings()
532+
.setIndices("test2")
533+
.setSettings(Settings.builder().put("index.number_of_replicas", yellowClusterReplicaCount).build())
534+
.execute()
535+
.actionGet();
536+
ClusterHealthResponse healthResponse = client().admin()
537+
.cluster()
538+
.prepareHealth()
539+
.setLevel("shards")
540+
.setApplyLevelAtTransportLayer(true)
541+
.execute()
542+
.actionGet();
543+
assertEquals(ClusterHealthStatus.YELLOW, healthResponse.getStatus());
544+
545+
assertEquals(2 * dataNodes, healthResponse.getActiveShards());
546+
assertEquals(2, healthResponse.getActivePrimaryShards());
547+
assertEquals(1, healthResponse.getUnassignedShards());
548+
assertEquals(0, healthResponse.getInitializingShards());
549+
assertEquals(0, healthResponse.getRelocatingShards());
550+
assertEquals(0, healthResponse.getDelayedUnassignedShards());
551+
552+
Map<String, ClusterIndexHealth> indices = healthResponse.getIndices();
553+
assertFalse(indices.isEmpty());
554+
assertEquals(2, indices.size());
555+
for (Map.Entry<String, ClusterIndexHealth> indicesHealth : indices.entrySet()) {
556+
String indexName = indicesHealth.getKey();
557+
boolean indexHasMoreReplicas = indexName.equals("test2");
558+
ClusterIndexHealth indicesHealthValue = indicesHealth.getValue();
559+
assertEquals(dataNodes, indicesHealthValue.getActiveShards());
560+
assertEquals(1, indicesHealthValue.getActivePrimaryShards());
561+
assertEquals(0, indicesHealthValue.getInitializingShards());
562+
assertEquals(indexHasMoreReplicas ? 1 : 0, indicesHealthValue.getUnassignedShards());
563+
assertEquals(0, indicesHealthValue.getDelayedUnassignedShards());
564+
assertEquals(0, indicesHealthValue.getRelocatingShards());
565+
assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, indicesHealthValue.getStatus());
566+
Map<Integer, ClusterShardHealth> shards = indicesHealthValue.getShards();
567+
assertFalse(shards.isEmpty());
568+
assertEquals(1, shards.size());
569+
for (Map.Entry<Integer, ClusterShardHealth> shardHealth : shards.entrySet()) {
570+
ClusterShardHealth clusterShardHealth = shardHealth.getValue();
571+
assertEquals(dataNodes, clusterShardHealth.getActiveShards());
572+
assertEquals(indexHasMoreReplicas ? 1 : 0, clusterShardHealth.getUnassignedShards());
573+
assertEquals(0, clusterShardHealth.getDelayedUnassignedShards());
574+
assertEquals(0, clusterShardHealth.getRelocatingShards());
575+
assertEquals(0, clusterShardHealth.getInitializingShards());
576+
assertTrue(clusterShardHealth.isPrimaryActive());
577+
assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, clusterShardHealth.getStatus());
578+
}
579+
}
580+
}
581+
582+
public void testHealthWithAwarenessAttributesLevelAppliedAtTransportLayer() {
583+
createIndex(
584+
"test1",
585+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
586+
);
587+
ensureGreen();
588+
ClusterHealthResponse healthResponse = client().admin()
589+
.cluster()
590+
.prepareHealth()
591+
.setLevel("awareness_attributes")
592+
.setApplyLevelAtTransportLayer(true)
593+
.execute()
594+
.actionGet();
595+
assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus());
596+
assertTrue(healthResponse.getIndices().isEmpty());
597+
assertNotNull(healthResponse.getClusterAwarenessHealth());
598+
assertEquals(1, healthResponse.getActiveShards());
599+
assertEquals(1, healthResponse.getActivePrimaryShards());
600+
assertEquals(0, healthResponse.getUnassignedShards());
601+
assertEquals(0, healthResponse.getInitializingShards());
602+
assertEquals(0, healthResponse.getRelocatingShards());
603+
assertEquals(0, healthResponse.getDelayedUnassignedShards());
604+
}
442605
}

‎server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java

+48-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,11 @@
1616
import org.opensearch.common.blobstore.BlobPath;
1717
import org.opensearch.common.settings.Settings;
1818
import org.opensearch.discovery.DiscoveryStats;
19+
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
1920
import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest;
21+
import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore;
22+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
23+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
2024
import org.opensearch.indices.recovery.RecoverySettings;
2125
import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
2226
import org.opensearch.repositories.RepositoriesService;
@@ -48,19 +52,27 @@
4852
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
4953
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
5054
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
55+
import static org.hamcrest.Matchers.is;
56+
import static org.hamcrest.Matchers.startsWith;
5157

5258
@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
5359
public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase {
5460

55-
private static String INDEX_NAME = "test-index";
61+
private static final String INDEX_NAME = "test-index";
62+
private static final String REMOTE_STATE_PREFIX = "!";
63+
private static final String REMOTE_ROUTING_PREFIX = "_";
5664
private boolean isRemoteStateEnabled = true;
5765
private String isRemotePublicationEnabled = "true";
66+
private boolean hasRemoteStateCharPrefix;
67+
private boolean hasRemoteRoutingCharPrefix;
5868

5969
@Before
6070
public void setup() {
6171
asyncUploadMockFsRepo = false;
6272
isRemoteStateEnabled = true;
6373
isRemotePublicationEnabled = "true";
74+
hasRemoteStateCharPrefix = randomBoolean();
75+
hasRemoteRoutingCharPrefix = randomBoolean();
6476
}
6577

6678
@Override
@@ -76,6 +88,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
7688
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
7789
routingTableRepoName
7890
);
91+
7992
return Settings.builder()
8093
.put(super.nodeSettings(nodeOrdinal))
8194
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), isRemoteStateEnabled)
@@ -87,6 +100,19 @@ protected Settings nodeSettings(int nodeOrdinal) {
87100
RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(),
88101
RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE
89102
)
103+
.put(
104+
RemoteClusterStateService.CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.getKey(),
105+
hasRemoteStateCharPrefix ? REMOTE_STATE_PREFIX : ""
106+
)
107+
.put(
108+
RemoteRoutingTableBlobStore.CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX.getKey(),
109+
hasRemoteRoutingCharPrefix ? REMOTE_ROUTING_PREFIX : ""
110+
)
111+
.put(RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX.toString())
112+
.put(
113+
RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING.getKey(),
114+
PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString()
115+
)
90116
.build();
91117
}
92118

@@ -130,6 +156,27 @@ public void testPublication() throws Exception {
130156
Map<String, Integer> manifestFiles = getMetadataFiles(repository, RemoteClusterMetadataManifest.MANIFEST);
131157
assertTrue(manifestFiles.containsKey(RemoteClusterMetadataManifest.MANIFEST));
132158

159+
RemoteClusterStateService remoteClusterStateService = internalCluster().getInstance(
160+
RemoteClusterStateService.class,
161+
internalCluster().getClusterManagerName()
162+
);
163+
ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
164+
getClusterState().getClusterName().value(),
165+
getClusterState().metadata().clusterUUID()
166+
).get();
167+
assertThat(manifest.getIndices().size(), is(1));
168+
if (hasRemoteStateCharPrefix) {
169+
for (UploadedIndexMetadata md : manifest.getIndices()) {
170+
assertThat(md.getUploadedFilename(), startsWith(REMOTE_STATE_PREFIX));
171+
}
172+
}
173+
assertThat(manifest.getIndicesRouting().size(), is(1));
174+
if (hasRemoteRoutingCharPrefix) {
175+
for (UploadedIndexMetadata md : manifest.getIndicesRouting()) {
176+
assertThat(md.getUploadedFilename(), startsWith(REMOTE_ROUTING_PREFIX));
177+
}
178+
}
179+
133180
// get settings from each node and verify that it is updated
134181
Settings settings = clusterService().getSettings();
135182
logger.info("settings : {}", settings);

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.opensearch.client.Client;
1212
import org.opensearch.common.settings.Settings;
1313
import org.opensearch.common.settings.SettingsException;
14+
import org.opensearch.common.unit.TimeValue;
1415
import org.opensearch.test.InternalTestCluster;
1516
import org.opensearch.test.OpenSearchIntegTestCase;
1617

@@ -67,6 +68,7 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()
6768
assertRemoteStoreBackedIndex(indexName2);
6869
}
6970

71+
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15793")
7072
public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception {
7173
logger.info("Initialize cluster: gives non remote cluster manager");
7274
initializeCluster(false);
@@ -109,7 +111,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
109111
setDirection(REMOTE_STORE.direction);
110112
String restoredIndexName2 = TEST_INDEX + "-restored2";
111113
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName2);
112-
ensureGreen(restoredIndexName2);
114+
ensureGreen(TimeValue.timeValueSeconds(90), restoredIndexName2);
113115

114116
logger.info("Verify that restored index is non remote-backed");
115117
assertRemoteStoreBackedIndex(restoredIndexName2);

0 commit comments

Comments
 (0)