diff --git a/CHANGELOG.md b/CHANGELOG.md index df178d0101a14..d9cab177ba1fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support for HTTP/3 (server side) ([#20017](https://github.com/opensearch-project/OpenSearch/pull/20017)) - Add circuit breaker support for gRPC transport to prevent out-of-memory errors ([#20203](https://github.com/opensearch-project/OpenSearch/pull/20203)) - Add index-level-encryption support for snapshots and remote-store ([#20095](https://github.com/opensearch-project/OpenSearch/pull/20095)) +- Adding BackWardCompatibility test for remote publication enabled cluster ([#20221](https://github.com/opensearch-project/OpenSearch/pull/20221)) + ### Changed - Handle custom metadata files in subdirectory-store ([#20157](https://github.com/opensearch-project/OpenSearch/pull/20157)) diff --git a/TESTING.md b/TESTING.md index 6d51e64dc9b1c..14b6bb23d89a3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -395,6 +395,8 @@ Backwards compatibility tests exist to test upgrading from each supported versio The test can be run for any versions which the current version will be compatible with. Tests are run for released versions download the distributions from the artifact repository, see [DistributionDownloadPlugin](./buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java) for the repository location. Tests are run for versions that are not yet released automatically check out the branch and build from source to get the distributions, see [BwcVersions](./buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java) and [distribution/bwc/build.gradle](./distribution/bwc/build.gradle) for more information. +**Note:** Rolling upgrade tests run in two modes: with remote publication disabled (default) and with remote publication enabled (using a `-remote` suffix cluster). The remote publication tests verify cluster state persistence to remote storage during upgrades. See [Remote Cluster State documentation](https://docs.opensearch.org/latest/tuning-your-cluster/availability-and-recovery/remote-store/remote-cluster-state/) for more information. + The minimum JDK versions for runtime and compiling need to be installed, and environment variables `JAVAx_HOME`, such as `JAVA8_HOME`, pointing to the JDK installations are required to run the tests against unreleased versions, since the distributions are created by building from source. The required JDK versions for each branch are located at [.ci/java-versions.properties](.ci/java-versions.properties), see [BwcSetupExtension](./buildSrc/src/main/java/org/opensearch/gradle/internal/BwcSetupExtension.java) for more information. To run all the backwards compatibility tests use: diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index ffcf815bfa264..ae62c83ccd5b0 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -37,6 +37,23 @@ apply plugin: 'opensearch.standalone-test' apply from : "$rootDir/gradle/bwc-test.gradle" apply plugin: 'opensearch.rest-resources' +def configureTestTask(task, clusterName, bwcVersionStr, suite, excludeRemoteTest, extraConfig = {}) { + task.configure { + useCluster project, testClusters."${clusterName}" + systemProperty 'tests.upgrade_from_version', bwcVersionStr + systemProperty 'tests.rest.suite', suite + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${clusterName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${clusterName}".getName()}") + + if (excludeRemoteTest) { + exclude '**/RemotePublicationClusterStateIT.class' + } + + extraConfig.delegate = delegate + extraConfig() + } +} + for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { /* * The goal here is to: @@ -47,7 +64,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { *
  • run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' *
  • upgrade one more node *
  • run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again - *
  • updgrade the last node + *
  • upgrade the last node *
  • run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster' * */ @@ -58,7 +75,6 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { "${baseName}" { versions = [bwcVersionStr, project.version] numberOfNodes = 3 - setting 'repositories.url.allowed_urls', 'http://snapshot.test*' setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' @@ -67,57 +83,85 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { dependsOn processTestResources - useCluster project, testClusters."${baseName}" mustRunAfter(precommit) - doFirst { - delete("${buildDir}/cluster/shared/repo/${baseName}") + configureTestTask(it, baseName, bwcVersionStr, 'old_cluster', true) { + doFirst { delete("${buildDir}/cluster/shared/repo/${baseName}") } } - systemProperty 'tests.upgrade_from_version', bwcVersionStr - systemProperty 'tests.rest.suite', 'old_cluster' - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oldClusterTest" - useCluster project, testClusters."${baseName}" - doFirst { - testClusters."${baseName}".nextNodeToNextVersion() + configureTestTask(it, baseName, bwcVersionStr, 'mixed_cluster', true) { + systemProperty 'tests.first_round', 'true' + doFirst { testClusters."${baseName}".nextNodeToNextVersion() } } - systemProperty 'tests.rest.suite', 'mixed_cluster' - systemProperty 'tests.upgrade_from_version', bwcVersionStr - systemProperty 'tests.first_round', 'true' - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#oneThirdUpgradedTest" - useCluster project, testClusters."${baseName}" - doFirst { - testClusters."${baseName}".nextNodeToNextVersion() + configureTestTask(it, baseName, bwcVersionStr, 'mixed_cluster', true) { + systemProperty 'tests.first_round', 'false' + doFirst { testClusters."${baseName}".nextNodeToNextVersion() } } - systemProperty 'tests.rest.suite', 'mixed_cluster' - systemProperty 'tests.upgrade_from_version', bwcVersionStr - systemProperty 'tests.first_round', 'false' - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedTest" - doFirst { - testClusters."${baseName}".nextNodeToNextVersion() + configureTestTask(it, baseName, bwcVersionStr, 'upgraded_cluster', true) { + doFirst { testClusters."${baseName}".nextNodeToNextVersion() } } - useCluster project, testClusters."${baseName}" - systemProperty 'tests.rest.suite', 'upgraded_cluster' - systemProperty 'tests.upgrade_from_version', bwcVersionStr + } + + // Remote publication cluster + String remoteBaseName = "${baseName}-remote" + + testClusters { + "${remoteBaseName}" { + versions = [bwcVersionStr, project.version] + numberOfNodes = 3 + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' + setting 'path.repo', "${buildDir}/cluster/shared/repo/${remoteBaseName}" + setting 'http.content_type.required', 'true' + setting 'node.attr.remote_store.routing_table.repository', 'my-fs-repository' + setting 'node.attr.remote_store.state.repository', 'my-fs-repository' + setting 'node.attr.remote_store.repository.my-fs-repository.type', 'fs' + setting 'node.attr.remote_store.repository.my-fs-repository.settings.location', "${buildDir}/cluster/shared/repo/${remoteBaseName}/remote_state" + setting 'cluster.remote_store.state.enabled', 'true' + setting 'cluster.remote_store.publication.enabled', 'true' + } + } + + tasks.register("${remoteBaseName}#oldClusterTest", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#upgradedClusterTest" + configureTestTask(it, remoteBaseName, bwcVersionStr, 'old_cluster', false) { + doFirst { delete("${buildDir}/cluster/shared/repo/${remoteBaseName}") } + } + } + + tasks.register("${remoteBaseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { + dependsOn "${remoteBaseName}#oldClusterTest" + configureTestTask(it, remoteBaseName, bwcVersionStr, 'mixed_cluster', false) { + systemProperty 'tests.first_round', 'true' + doFirst { testClusters."${remoteBaseName}".nextNodeToNextVersion() } + } + } - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + tasks.register("${remoteBaseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { + dependsOn "${remoteBaseName}#oneThirdUpgradedTest" + configureTestTask(it, remoteBaseName, bwcVersionStr, 'mixed_cluster', false) { + systemProperty 'tests.first_round', 'false' + doFirst { testClusters."${remoteBaseName}".nextNodeToNextVersion() } + } + } + + tasks.register("${remoteBaseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { + dependsOn "${remoteBaseName}#twoThirdsUpgradedTest" + configureTestTask(it, remoteBaseName, bwcVersionStr, 'upgraded_cluster', false) { + doFirst { testClusters."${remoteBaseName}".nextNodeToNextVersion() } + } } tasks.register(bwcTaskName(bwcVersion)) { - dependsOn tasks.named("${baseName}#upgradedClusterTest") + dependsOn tasks.named("${remoteBaseName}#upgradedClusterTest") } } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RemotePublicationClusterStateIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RemotePublicationClusterStateIT.java new file mode 100644 index 0000000000000..f690b9707eb1f --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RemotePublicationClusterStateIT.java @@ -0,0 +1,268 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.upgrades; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Integration tests for remote publication enabled clusters during rolling upgrades. + *

    + * The test verifies that cluster state (indices, templates, settings) is correctly + * published and persisted to remote storage during version upgrades. + *

    + * For debugging failing tests: + *

    + *

    + * See: + * Remote Cluster State Documentation + */ +@SuppressWarnings("unchecked") +public class RemotePublicationClusterStateIT extends AbstractRollingTestCase { + private static final String templateName = "my_template"; + private static final String indexName = "test_cluster_state"; + private static final String componentTemplateName = "test_component_template"; + + public void testUpgradeWithRemotePublicationEnabled() throws Exception { + if (CLUSTER_TYPE == ClusterType.OLD) { + verifyRemotePublicationEnabled(); + Settings indexSettings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 1) + .put("index.refresh_interval", "5s").build(); + String indexMappings = "\"properties\": {\"name\": {\"type\":\"text\"}}"; + String aliases = "\"main\":{\"is_write_index\":true}"; + createIndex(indexName, indexSettings, indexMappings, aliases); + createIndexTemplate(); + createComponentTemplate(); + createComposableTemplate(); + addPersistentSettings(); + addTransientSettings(); + + Request request = new Request("GET", "_cluster/state"); + Response response = client().performRequest(request); + assertOK(response); + + verifyIndexInClusterState(response); + verifyTemplateMetadataInClusterState(response); + verifyComponentTemplateInClusterState(response); + verifyComposableTemplateInClusterState(response); + verifySettingsInClusterState(); + } else { + verifyRemotePublicationEnabled(); + + Request request = new Request("GET", "_cluster/state"); + Response response = client().performRequest(request); + assertOK(response); + + verifyIndexInClusterState(response); + verifyTemplateMetadataInClusterState(response); + verifyComponentTemplateInClusterState(response); + verifyComposableTemplateInClusterState(response); + } + } + + private static void createIndexTemplate() throws Exception { + Request putIndexTemplate = new Request("PUT", "_template/" + templateName); + String indexTemplateJson = """ + { + "index_patterns": ["pattern-*", "log-*"], + "settings": { + "number_of_shards": 3, + "number_of_replicas": 1, + "index.refresh_interval": "5s" + }, + "mappings": { + "properties": { + "timestamp": { + "type": "date" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "level": { + "type": "keyword" + } + } + }, + "aliases": { + "mydata": {} + }, + "version": 1 + }"""; + putIndexTemplate.setJsonEntity(indexTemplateJson); + client().performRequest(putIndexTemplate); + } + + private static void createComponentTemplate() throws Exception { + Request putComponentTemplate = new Request("PUT", "_component_template/" + componentTemplateName); + String componentTemplateJson = """ + { + "template": { + "mappings": { + "properties": { + "name": { + "type": "text" + } + } + } + }, + "version": 1 + }"""; + putComponentTemplate.setJsonEntity(componentTemplateJson); + client().performRequest(putComponentTemplate); + } + + private static void createComposableTemplate() throws Exception { + Request putComposableTemplate = new Request("PUT", "_index_template/composable_template"); + String composableTemplateJson = String.format(Locale.ROOT, """ + { + "index_patterns": ["te-*", "bar-*"], + "template": { + "mappings": { + "properties": { + "email": { + "type": "keyword" + } + } + } + }, + "version": 1, + "composed_of": ["%s"] + }""", componentTemplateName); + putComposableTemplate.setJsonEntity(composableTemplateJson); + client().performRequest(putComposableTemplate); + } + + private static void addPersistentSettings() throws Exception { + Request putSettings = new Request("PUT", "_cluster/settings"); + String settingsJson = """ + { + "persistent": { + "cluster": { + "remote": { + "cluster": { + "seeds": ["127.0.0.1:9300"] + } + } + } + } + }"""; + putSettings.setJsonEntity(settingsJson); + assertOK(client().performRequest(putSettings)); + } + + private static void addTransientSettings() throws Exception { + Request putSettings = new Request("PUT", "_cluster/settings"); + String settingsJson = """ + { + "transient": { + "cluster": { + "remote": { + "cluster": { + "seeds": ["127.0.0.1:9300"] + } + } + } + } + }"""; + putSettings.setJsonEntity(settingsJson); + assertOK(client().performRequest(putSettings)); + } + + private static void verifyIndexInClusterState(Response clusterStateResponse) throws Exception { + Map responseMap = entityAsMap(clusterStateResponse); + Map metadata = (Map) responseMap.get("metadata"); + assertNotNull("Metadata should exist in response", metadata); + Map indices = (Map) metadata.get("indices"); + assertNotNull("Indices should exist in metadata", indices); + Map index = (Map) indices.get(indexName); + assertNotNull("Index " + indexName + " should exist in cluster state", index); + Map settings = (Map) index.get("settings"); + assertNotNull("Settings should exist in index", settings); + Map mappings = (Map) index.get("mappings"); + assertNotNull("Mappings should exist in index", mappings); + } + + private static void verifyTemplateMetadataInClusterState(Response clusterStateResponse) throws Exception { + Map responseMap = entityAsMap(clusterStateResponse); + Map metadata = (Map) responseMap.get("metadata"); + assertNotNull("Metadata should exist in response", metadata); + Map templates = (Map) metadata.get("templates"); + assertNotNull("Templates should exist in metadata", templates); + Map templateMetadata = (Map) templates.get(templateName); + assertNotNull("Template " + templateName + " should exist in cluster state", templateMetadata); + List indexPatterns = (List) templateMetadata.get("index_patterns"); + assertEquals("Index patterns should match", Arrays.asList("pattern-*", "log-*"), indexPatterns); + } + + private static void verifyComponentTemplateInClusterState(Response clusterStateResponse) throws Exception { + Map responseMap = entityAsMap(clusterStateResponse); + Map metadata = (Map) responseMap.get("metadata"); + assertNotNull("Metadata should exist in response", metadata); + Map componentTemplate = (Map) getNestedValue(metadata, "component_template", "component_template", componentTemplateName); + assertNotNull("Component template should exist in metadata", componentTemplate); + } + + private static void verifyComposableTemplateInClusterState(Response clusterStateResponse) throws Exception { + Map responseMap = entityAsMap(clusterStateResponse); + Map metadata = (Map) responseMap.get("metadata"); + assertNotNull("Metadata should exist in response", metadata); + Map indexTemplates = (Map) getNestedValue(metadata, "index_template", "index_template"); + assertNotNull("Index templates should exist in metadata", indexTemplates); + Map composableTemplate = (Map) indexTemplates.get("composable_template"); + assertNotNull("Composable template should exist in metadata", composableTemplate); + } + + private static void verifySettingsInClusterState() throws Exception { + Request getSettingRequest = new Request("GET", "/_cluster/settings"); + Response response = client().performRequest(getSettingRequest); + assertOK(response); + Map responseMap = entityAsMap(response); + Map persistent = (Map) responseMap.get("persistent"); + assertNotNull("Persistent settings should exist in metadata", persistent); + } + + private static Object getNestedValue(Map map, String... keys) { + Object current = map; + for (String key : keys) { + if (!(current instanceof Map)) return null; + current = ((Map) current).get(key); + } + return current; + } + + private void verifyRemotePublicationEnabled() throws Exception { + Request getSettingRequest = new Request("GET", "/_cluster/settings"); + getSettingRequest.addParameter("include_defaults", "true"); + Response response = client().performRequest(getSettingRequest); + assertOK(response); + Map responseMap = entityAsMap(response); + Map defaultsSettings = (Map) responseMap.get("defaults"); + Object enabled = getNestedValue(defaultsSettings, "cluster", "remote_store", "publication", "enabled"); + assertEquals("true", enabled); + } +}